Skip to content

Commit 097c751

Browse files
robnlundman
robn
authored and
lundman
committed
ZTS: test response of various sync methods under different failmodes
These are all the same shape: set up the pool to suspend on first write, then perform some write+sync operation. The pool should suspend, and the sync operation should respond according to the failmode= property. We test fsync(), msync() and two forms of write() (open with O_SYNC, and async with sync=always), which all take slightly different paths to zil_commit() and back. A helper function is included to do the write+sync sequence with mmap() and msync(), since I didn't find a convenient tool to do that. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Reviewed-by: Alexander Motin <alexander.motin@TrueNAS.com> Signed-off-by: Rob Norris <rob.norris@klarasystems.com> Closes openzfs#17398
1 parent e775446 commit 097c751

15 files changed

Lines changed: 541 additions & 2 deletions

‎tests/runfiles/common.run‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -725,7 +725,11 @@ tests = ['fadvise_willneed']
725725
tags = ['functional', 'fadvise']
726726

727727
[tests/functional/failmode]
728-
tests = ['failmode_dmu_tx_wait', 'failmode_dmu_tx_continue']
728+
tests = ['failmode_dmu_tx_wait', 'failmode_dmu_tx_continue',
729+
'failmode_fsync_wait', 'failmode_fsync_continue',
730+
'failmode_msync_wait', 'failmode_msync_continue',
731+
'failmode_osync_wait', 'failmode_osync_continue',
732+
'failmode_syncalways_wait', 'failmode_syncalways_continue']
729733
tags = ['functional', 'failmode']
730734

731735
[tests/functional/fallocate]

‎tests/zfs-tests/cmd/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
/mmap_seek
2929
/mmap_sync
3030
/mmapwrite
31+
/mmap_write_sync
3132
/nvlist_to_lua
3233
/randfree_file
3334
/randwritecomp

‎tests/zfs-tests/cmd/Makefile.am‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ scripts_zfs_tests_bin_PROGRAMS += %D%/mkbusy %D%/mkfile %D%/mkfiles %D%/mktree
7474

7575
scripts_zfs_tests_bin_PROGRAMS += \
7676
%D%/mmap_exec %D%/mmap_ftruncate %D%/mmap_seek \
77-
%D%/mmap_sync %D%/mmapwrite %D%/readmmap
77+
%D%/mmap_sync %D%/mmapwrite %D%/readmmap %D%/mmap_write_sync
7878
%C%_mmapwrite_LDADD = -lpthread
7979

8080
if WANT_MMAP_LIBAIO

‎tests/zfs-tests/cmd/mmap_write_sync.c‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// SPDX-License-Identifier: CDDL-1.0
2+
/*
3+
* CDDL HEADER START
4+
*
5+
* The contents of this file are subject to the terms of the
6+
* Common Development and Distribution License (the "License").
7+
* You may not use this file except in compliance with the License.
8+
*
9+
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10+
* or https://opensource.org/licenses/CDDL-1.0.
11+
* See the License for the specific language governing permissions
12+
* and limitations under the License.
13+
*
14+
* When distributing Covered Code, include this CDDL HEADER in each
15+
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16+
* If applicable, add the following below this CDDL HEADER, with the
17+
* fields enclosed by brackets "[]" replaced with your own identifying
18+
* information: Portions Copyright [yyyy] [name of copyright owner]
19+
*
20+
* CDDL HEADER END
21+
*/
22+
23+
/*
24+
* Copyright (c) 2025, Klara, Inc.
25+
*/
26+
27+
#include <stdio.h>
28+
#include <unistd.h>
29+
#include <stdlib.h>
30+
#include <fcntl.h>
31+
#include <sys/stat.h>
32+
#include <sys/mman.h>
33+
34+
#define PAGES (8)
35+
36+
int
37+
main(int argc, char **argv)
38+
{
39+
if (argc != 2) {
40+
fprintf(stderr, "usage: %s <filename>\n", argv[0]);
41+
exit(1);
42+
}
43+
44+
long page_size = sysconf(_SC_PAGESIZE);
45+
if (page_size < 0) {
46+
perror("sysconf");
47+
exit(2);
48+
}
49+
size_t map_size = page_size * PAGES;
50+
51+
int fd = open(argv[1], O_CREAT|O_RDWR, S_IRWXU|S_IRWXG|S_IRWXO);
52+
if (fd < 0) {
53+
perror("open");
54+
exit(2);
55+
}
56+
57+
if (ftruncate(fd, map_size) < 0) {
58+
perror("ftruncate");
59+
close(fd);
60+
exit(2);
61+
}
62+
63+
uint64_t *p =
64+
mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
65+
if (p == MAP_FAILED) {
66+
perror("mmap");
67+
close(fd);
68+
exit(2);
69+
}
70+
71+
for (int i = 0; i < (map_size / sizeof (uint64_t)); i++)
72+
p[i] = 0x0123456789abcdef;
73+
74+
if (msync(p, map_size, MS_SYNC) < 0) {
75+
perror("msync");
76+
munmap(p, map_size);
77+
close(fd);
78+
exit(3);
79+
}
80+
81+
munmap(p, map_size);
82+
close(fd);
83+
exit(0);
84+
}

‎tests/zfs-tests/include/commands.cfg‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ export ZFSTEST_FILES='badsend
210210
mmap_seek
211211
mmap_sync
212212
mmapwrite
213+
mmap_write_sync
213214
nvlist_to_lua
214215
randfree_file
215216
randwritecomp

‎tests/zfs-tests/tests/Makefile.am‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,7 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \
276276
functional/direct/dio.kshlib \
277277
functional/events/events.cfg \
278278
functional/events/events_common.kshlib \
279+
functional/failmode/failmode.kshlib \
279280
functional/fault/fault.cfg \
280281
functional/gang_blocks/gang_blocks.kshlib \
281282
functional/grow/grow.cfg \
@@ -1541,6 +1542,14 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
15411542
functional/failmode/cleanup.ksh \
15421543
functional/failmode/failmode_dmu_tx_wait.ksh \
15431544
functional/failmode/failmode_dmu_tx_continue.ksh \
1545+
functional/failmode/failmode_fsync_wait.ksh \
1546+
functional/failmode/failmode_fsync_continue.ksh \
1547+
functional/failmode/failmode_msync_wait.ksh \
1548+
functional/failmode/failmode_msync_continue.ksh \
1549+
functional/failmode/failmode_osync_wait.ksh \
1550+
functional/failmode/failmode_osync_continue.ksh \
1551+
functional/failmode/failmode_syncalways_wait.ksh \
1552+
functional/failmode/failmode_syncalways_continue.ksh \
15441553
functional/failmode/setup.ksh \
15451554
functional/fallocate/cleanup.ksh \
15461555
functional/fallocate/fallocate_prealloc.ksh \

‎tests/zfs-tests/tests/functional/failmode/failmode.kshlib‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/bin/ksh -p
2+
# SPDX-License-Identifier: CDDL-1.0
3+
#
4+
# CDDL HEADER START
5+
#
6+
# The contents of this file are subject to the terms of the
7+
# Common Development and Distribution License (the "License").
8+
# You may not use this file except in compliance with the License.
9+
#
10+
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
11+
# or https://opensource.org/licenses/CDDL-1.0.
12+
# See the License for the specific language governing permissions
13+
# and limitations under the License.
14+
#
15+
# When distributing Covered Code, include this CDDL HEADER in each
16+
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
17+
# If applicable, add the following below this CDDL HEADER, with the
18+
# fields enclosed by brackets "[]" replaced with your own identifying
19+
# information: Portions Copyright [yyyy] [name of copyright owner]
20+
#
21+
# CDDL HEADER END
22+
#
23+
24+
#
25+
# Copyright (c) 2025, Klara, Inc.
26+
#
27+
28+
. $STF_SUITE/include/libtest.shlib
29+
30+
typeset -A failmode_sync_helper_cmd=(
31+
["fsync"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1 conv=fsync'
32+
["msync"]='mmap_write_sync DATAFILE'
33+
["osync"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1 oflag=sync'
34+
["syncalways"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1'
35+
)
36+
37+
typeset -A failmode_sync_helper_dsopts=(
38+
["syncalways"]="-o sync=always"
39+
)
40+
41+
function failmode_sync_cleanup
42+
{
43+
zinject -c all || true
44+
zpool clear $TESTPOOL || true
45+
destroy_pool $TESTPOOL
46+
}
47+
48+
#
49+
# failmode_sync_test <failmode> <helper>
50+
#
51+
# run a failmode sync test:
52+
# - failmode: wait|continue
53+
# - helper: fsync|msync|osync|syncalways
54+
#
55+
function failmode_sync_test
56+
{
57+
typeset failmode=$1
58+
typeset helper=$2
59+
60+
# we'll need two disks, one for the main pool, one for the log
61+
read -r DISK1 DISK2 _ <<<"$DISKS"
62+
63+
# file to write to the pool
64+
typeset datafile="/$TESTPOOL/$TESTFS/datafile"
65+
66+
# create a single-disk pool with a separate log and the wanted failmode
67+
log_must zpool create \
68+
-f -o failmode=$failmode $TESTPOOL $DISK1 log $DISK2
69+
70+
# create the test dataset. we bias the ZIL towards the log device to
71+
# try to ensure that the sync write never involves the main device
72+
log_must zfs create \
73+
-o recordsize=128k -o logbias=latency \
74+
${failmode_sync_helper_dsopts[$helper]} \
75+
$TESTPOOL/$TESTFS
76+
77+
# create the target file. the ZIL head structure is created on first
78+
# use, and does a full txg wait to finish, which we want to avoid
79+
log_must dd if=/dev/zero of=$datafile bs=128k count=1 conv=fsync
80+
log_must zpool sync
81+
82+
# inject errors. writes will fail, as will the followup probes
83+
zinject -d $DISK1 -e io -T write $TESTPOOL
84+
zinject -d $DISK1 -e nxio -T probe $TESTPOOL
85+
zinject -d $DISK2 -e io -T write $TESTPOOL
86+
zinject -d $DISK2 -e nxio -T probe $TESTPOOL
87+
88+
# run the helper program in the background. the pool should immediately
89+
# suspend, and the sync op block or fail based on the failmode
90+
typeset helper_cmd=${failmode_sync_helper_cmd[$helper]/DATAFILE/$datafile}
91+
log_note "running failmode sync helper: $helper_cmd"
92+
$helper_cmd &
93+
typeset -i pid=$!
94+
95+
# should only take a moment, but give it a chance
96+
log_note "waiting for pool to suspend"
97+
typeset -i tries=10
98+
until [[ $(kstat_pool $TESTPOOL state) == "SUSPENDED" ]] ; do
99+
if ((tries-- == 0)); then
100+
log_fail "pool didn't suspend"
101+
fi
102+
sleep 1
103+
done
104+
105+
# zil_commit() should have noticed the suspend by now
106+
typeset -i zilerr=$(kstat zil.zil_commit_error_count)
107+
108+
# see if the helper program blocked
109+
typeset -i blocked
110+
if kill -0 $pid ; then
111+
blocked=1
112+
log_note "$helper: blocked in the kernel"
113+
else
114+
blocked=0
115+
log_note "$helper: exited while pool suspended"
116+
fi
117+
118+
# bring the pool back online
119+
zinject -c all
120+
zpool clear $TESTPOOL
121+
122+
# program definitely exited now, get its return code
123+
wait $pid
124+
typeset -i rc=$?
125+
126+
failmode_sync_cleanup
127+
128+
log_note "$helper: zilerr=$zilerr blocked=$blocked rc=$rc"
129+
130+
# confirm expected results for the failmode
131+
if [[ $failmode = "wait" ]] ; then
132+
# - the ZIL saw an error, and fell back to a txg sync
133+
# - sync op blocked when the pool suspended
134+
# - after resume, sync op succeeded, helper returned success
135+
log_must test $zilerr -ne 0
136+
log_must test $blocked -eq 1
137+
log_must test $rc -eq 0
138+
elif [[ $failmode = "continue" ]] ; then
139+
# confirm expected results:
140+
# - the ZIL saw an error, and fell back to a txg sync
141+
# - helper exited when the pool suspended
142+
# - sync op returned an error, so helper returned failure
143+
log_must test $zilerr -ne 0
144+
log_must test $blocked -eq 0
145+
log_must test $rc -ne 0
146+
else
147+
log_fail "impossible failmode: $failmode"
148+
fi
149+
}

‎tests/zfs-tests/tests/functional/failmode/failmode_fsync_continue.ksh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/ksh -p
2+
# SPDX-License-Identifier: CDDL-1.0
3+
#
4+
# CDDL HEADER START
5+
#
6+
# The contents of this file are subject to the terms of the
7+
# Common Development and Distribution License (the "License").
8+
# You may not use this file except in compliance with the License.
9+
#
10+
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
11+
# or https://opensource.org/licenses/CDDL-1.0.
12+
# See the License for the specific language governing permissions
13+
# and limitations under the License.
14+
#
15+
# When distributing Covered Code, include this CDDL HEADER in each
16+
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
17+
# If applicable, add the following below this CDDL HEADER, with the
18+
# fields enclosed by brackets "[]" replaced with your own identifying
19+
# information: Portions Copyright [yyyy] [name of copyright owner]
20+
#
21+
# CDDL HEADER END
22+
#
23+
24+
#
25+
# Copyright (c) 2025, Klara, Inc.
26+
#
27+
28+
. $STF_SUITE/include/libtest.shlib
29+
. $STF_SUITE/tests/functional/failmode/failmode.kshlib
30+
31+
typeset desc="fsync() returns when pool suspends with failmode=continue"
32+
33+
log_assert $desc
34+
log_onexit failmode_sync_cleanup
35+
log_must failmode_sync_test continue fsync
36+
log_pass $desc

‎tests/zfs-tests/tests/functional/failmode/failmode_fsync_wait.ksh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/ksh -p
2+
# SPDX-License-Identifier: CDDL-1.0
3+
#
4+
# CDDL HEADER START
5+
#
6+
# The contents of this file are subject to the terms of the
7+
# Common Development and Distribution License (the "License").
8+
# You may not use this file except in compliance with the License.
9+
#
10+
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
11+
# or https://opensource.org/licenses/CDDL-1.0.
12+
# See the License for the specific language governing permissions
13+
# and limitations under the License.
14+
#
15+
# When distributing Covered Code, include this CDDL HEADER in each
16+
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
17+
# If applicable, add the following below this CDDL HEADER, with the
18+
# fields enclosed by brackets "[]" replaced with your own identifying
19+
# information: Portions Copyright [yyyy] [name of copyright owner]
20+
#
21+
# CDDL HEADER END
22+
#
23+
24+
#
25+
# Copyright (c) 2025, Klara, Inc.
26+
#
27+
28+
. $STF_SUITE/include/libtest.shlib
29+
. $STF_SUITE/tests/functional/failmode/failmode.kshlib
30+
31+
typeset desc="fsync() blocks when pool suspends with failmode=wait"
32+
33+
log_assert $desc
34+
log_onexit failmode_sync_cleanup
35+
log_must failmode_sync_test wait fsync
36+
log_pass $desc

0 commit comments

Comments
 (0)