git: ce7b20e5129c - main - stress2: Added more FFS tests

From: Peter Holm <pho_at_FreeBSD.org>
Date: Sun, 19 Mar 2023 08:55:27 UTC
The branch main has been updated by pho:

URL: https://cgit.FreeBSD.org/src/commit/?id=ce7b20e5129cf0f269951b313d336a9c7d54d790

commit ce7b20e5129cf0f269951b313d336a9c7d54d790
Author:     Peter Holm <pho@FreeBSD.org>
AuthorDate: 2023-03-19 08:55:00 +0000
Commit:     Peter Holm <pho@FreeBSD.org>
CommitDate: 2023-03-19 08:55:00 +0000

    stress2: Added more FFS tests
---
 tools/test/stress2/misc/all.exclude |  11 ++-
 tools/test/stress2/misc/force14.sh  | 119 ++++++++++++++++++++++++
 tools/test/stress2/misc/force15.sh  | 113 +++++++++++++++++++++++
 tools/test/stress2/misc/fsck10.sh   | 171 ++++++++++++++++++++++++++++++++++
 tools/test/stress2/misc/fsck11.sh   | 170 ++++++++++++++++++++++++++++++++++
 tools/test/stress2/misc/fsck12.sh   | 177 ++++++++++++++++++++++++++++++++++++
 tools/test/stress2/misc/gnop13.sh   | 138 ++++++++++++++++++++++++++++
 7 files changed, 897 insertions(+), 2 deletions(-)

diff --git a/tools/test/stress2/misc/all.exclude b/tools/test/stress2/misc/all.exclude
index 2f4e04473eff..022075e63cb4 100644
--- a/tools/test/stress2/misc/all.exclude
+++ b/tools/test/stress2/misc/all.exclude
@@ -5,9 +5,14 @@ backingstore.sh		g_vfs_done():md6a[WRITE(offset=...)]error = 28			20111220
 backingstore2.sh	panic: 43 vncache entries remaining				20111220
 backingstore3.sh	g_vfs_done():md6a[WRITE(offset=...)]error = 28			20111230
 dd.sh			CAM stuck in vmwait						20200116
+force13.sh		https://people.freebsd.org/~pho/stress/log/log0376.txt		20221113
+force14.sh		Waiting for fix							20230319
+force15.sh		Waiting for fix							20230319
 force4.sh		https://people.freebsd.org/~pho/stress/log/log0082.txt		20210328
 force7.sh		https://people.freebsd.org/~pho/stress/log/log0266.txt		20220207
-force13.sh		https://people.freebsd.org/~pho/stress/log/log0376.txt		20221113
+fsck10.sh		Waiting for fix							20230319
+fsck11.sh		Waiting for fix							20230319
+fsck12.sh		Waiting for fix							20230319
 fsync.sh		panic: Journal overflow						20190208
 fuse.sh			Memory corruption seen in log file kostik734.txt		20141114
 fuse2.sh		Deadlock seen							20121129
@@ -18,10 +23,11 @@ gjournal.sh		panic: Journal overflow						20190626
 gjournal2.sh		panic: Journal overflow						20180125
 gjournal3.sh		panic: Bio not on queue						20171225
 gjournal4.sh		CAM stuck in vmwait						20180517
+gnop10.sh		Waiting for fix							20230319
+gnop13.sh		https://people.freebsd.org/~pho/stress/log/log0386.txt		20221113
 gnop7.sh		Waiting for patch commit					20190820
 gnop8.sh		Waiting for patch commit					20201214
 gnop9.sh		Waiting for patch commit					20201214
-gnop13.sh		https://people.freebsd.org/~pho/stress/log/log0386.txt		20221113
 graid1_8.sh		Known issue							20170909
 graid1_9.sh		panic: Bad effnlink						20180212
 ifconfig.sh		Bug 253824							20210322
@@ -49,6 +55,7 @@ sctp3.sh		panic: Queues are not empty when handling SHUTDOWN-COMPLETE	20210211
 sendfile25.sh		WiP								20200611
 signal.sh		Timing issues. Needs fixing					20171116
 snap8.sh		https://people.freebsd.org/~pho/stress/log/log0123.txt		20211008
+suj12.sh		Waiting for fix							20230319
 suj19.sh		https://people.freebsd.org/~pho/stress/log/log0378.txt		20221113
 suj27.sh		https://people.freebsd.org/~pho/stress/log/log0387.txt		20221113
 suj36.sh		https://people.freebsd.org/~pho/stress/log/log0392.txt		20221114
diff --git a/tools/test/stress2/misc/force14.sh b/tools/test/stress2/misc/force14.sh
new file mode 100755
index 000000000000..eec850c50891
--- /dev/null
+++ b/tools/test/stress2/misc/force14.sh
@@ -0,0 +1,119 @@
+#!/bin/sh
+
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2023 Peter Holm <pho@FreeBSD.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+# SU+J Snapshot test with a 50G file system
+
+# "panic: handle_jwork: Unknown type jnewblk" seen:
+# https://people.freebsd.org/~pho/stress/log/log0422.txt
+
+# "panic: flush_pagedep_deps: failed to flush inodedep..." seen:
+# https://people.freebsd.org/~pho/stress/log/log0423.txt
+
+# "panic: softdep_deallocate_dependencies: dangling deps" seen:
+# https://people.freebsd.org/~pho/stress/log/log0424.txt
+
+[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
+. ../default.cfg
+
+set -u
+log=/tmp/force14.sh.log
+mount | grep $mntpoint | grep -q /dev/md && umount -f $mntpoint
+mdconfig -l | grep -q md$mdstart && mdconfig -d -u $mdstart
+truncate -s 50g $diskimage
+mdconfig -a -t vnode -f $diskimage -u $mdstart
+flags="-j"
+
+newfs $flags md$mdstart > /dev/null 2>&1
+
+# Exclude rename for now due to log0374.txt
+export TESTPROGS=`cd ..; find testcases/ -perm -1 -type f | \
+    egrep -Ev "/run/|/badcode/|/pty/|/shm/|/socket/|sysctl|tcp|thr|udp|rename"`
+export runRUNTIME=3m
+export RUNDIR=$mntpoint/stressX
+export CTRLDIR=$mntpoint/stressX.control
+start=`date +%s`
+while [ $((`date +%s` - start)) -lt $((15 * 60)) ]; do
+	mount /dev/md$mdstart $mntpoint
+	rm -fr $mntpoint/lost+found
+	chmod 777 $mntpoint
+
+	su $testuser -c 'cd ..; ./testcases/run/run $TESTPROGS' > \
+	    /dev/null 2>&1 &
+
+	t=`jot -r 1 60 180`
+	st=`date +%s`
+	mkdir -p $mntpoint/.snap
+	for i in `jot 10`; do
+		rm -f $mntpoint/.snap/$i
+		mksnap_ffs $mntpoint $mntpoint/.snap/$i ||
+		    { ../tools/killall.sh; break; }
+		sleep `jot -r 1 1 5`
+		[ $((`date +%s` - st)) -ge $t ] && break
+	done &
+	sleep `jot -r 1 60 180`
+	while mdconfig -l | grep -q md$mdstart; do
+		mdconfig -d -u $mdstart -o force || sleep 1
+	done
+	sleep 1
+	../tools/killall.sh
+	wait
+	n=0
+	while mount | grep -q "on $mntpoint "; do
+		umount $mntpoint || sleep 1
+		[ $((n += 1)) -gt 300 ] && { echo FAIL; exit 1; }
+	done
+	mdconfig -a -t vnode -f $diskimage -u $mdstart
+	c=0
+	# Process the journal
+	fsck_ffs -fy /dev/md$mdstart > $log 2>&1; s=$?
+	grep 'INTERNAL ERROR: GOT TO reply' $log
+	for i in `jot 4`; do
+		[ $i -ne 1 ]  &&
+		    echo "fsck_ffs -fy /dev/md$mdstart"
+		fsck_ffs -fy /dev/md$mdstart > $log 2>&1; s=$?
+		grep 'INTERNAL ERROR: GOT TO reply' $log
+		grep -q CLEAN $log && c=$((c+=1))
+		grep -Eq "WAS MODIFIED" $log || break
+	done
+	[ $c -gt 1 ] &&
+	    { echo "Note: FS marked clean+modified $c times out of $i fsck runs"; s=101; }
+	[ $s -ne 0 ] && break
+	grep -Eq "IS CLEAN|MARKED CLEAN" $log || { s=102; break; }
+done
+if [ $s -eq 0 ]; then
+	mount /dev/md$mdstart $mntpoint
+	cp -R /usr/include $mntpoint
+	dd if=/dev/zero of=$mntpoint/big bs=1m count=10 status=none
+	find $mntpoint/* -delete
+
+	umount $mntpoint
+	mdconfig -d -u $mdstart
+	rm -f $diskimage $log
+fi
+exit $s
diff --git a/tools/test/stress2/misc/force15.sh b/tools/test/stress2/misc/force15.sh
new file mode 100755
index 000000000000..955025cd98c6
--- /dev/null
+++ b/tools/test/stress2/misc/force15.sh
@@ -0,0 +1,113 @@
+#!/bin/sh
+
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2023 Peter Holm <pho@FreeBSD.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+# FFS+SU with snapshots and forced unmounts
+
+# "panic: flush_pagedep_deps: failed to flush inodedep ..." seen
+# https://people.freebsd.org/~pho/stress/log/log0427.txt
+
+[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
+
+prog=$(basename "$0" .sh)
+diskimage=/tmp/diskimage
+log=/tmp/$prog.log
+mdstart=10
+mntpoint=/mnt
+newfs_flags="-U"
+
+set -u
+mount | grep $mntpoint | grep -q /dev/md && umount -f $mntpoint
+mdconfig -l | grep -q md$mdstart && mdconfig -d -u $mdstart
+dd if=/dev/zero of=$diskimage bs=1m count=1k status=none
+mdconfig -a -t vnode -f $diskimage -u $mdstart
+flags="-U"
+
+newfs $flags md$mdstart > /dev/null 2>&1
+
+start=`date +%s`
+while [ $((`date +%s` - start)) -lt $((5 * 60)) ]; do
+	mount /dev/md$mdstart $mntpoint
+	rm -fr $mntpoint/lost+found
+
+	jot 10 | xargs -I% mkdir -p $mntpoint/%
+	n=5000
+	for j in `jot 10`; do
+		(
+			jot $n | xargs -P0 -I% touch $mntpoint/$j/%
+			jot $n | xargs -P0 -I% rm    $mntpoint/$j/%
+		) > /dev/null 2>&1 &
+	done 
+
+	sleep `jot -r 1 5 20`
+	t=`jot -r 1 60 180`
+	st=`date +%s`
+	mkdir -p $mntpoint/.snap
+	for i in `jot 10`; do
+		rm -f $mntpoint/.snap/$i
+		mksnap_ffs $mntpoint $mntpoint/.snap/$i > /dev/null 2>&1 || break
+		sleep `jot -r 1 1 5`
+		[ $((`date +%s` - st)) -ge $t ] && break
+	done &
+	sleep `jot -r 1 2 5`
+	while mdconfig -l | grep -q md$mdstart; do
+		mdconfig -d -u $mdstart -o force || sleep 1
+	done
+	sleep 1
+	wait
+	n=0
+	while mount | grep -q "on $mntpoint "; do
+		umount $mntpoint || sleep 1
+		[ $((n += 1)) -gt 300 ] && { echo FAIL; exit 1; }
+	done
+	mdconfig -a -t vnode -f $diskimage -u $mdstart
+	c=0
+	for i in `jot 5`; do
+		[ $i -ne 1 ] &&
+		    echo "$i: fsck_ffs -fy /dev/md$mdstart"
+		fsck_ffs -fy /dev/md$mdstart > $log 2>&1; s=$?
+		grep 'INTERNAL ERROR: GOT TO reply' $log
+		grep -q CLEAN $log && grep -q MODIFIED && c=$((c+=1))
+		grep -Eq "WAS MODIFIED" $log || break
+	done
+	[ $c -gt 1 ] &&
+	    { echo "Note: FS marked clean+modified $c times out of $i fsck runs"; s=101; }
+	[ $s -ne 0 ] && break
+	grep -Eq "IS CLEAN|MARKED CLEAN" $log || { s=102; break; }
+done
+if [ $s -eq 0 ]; then
+	mount /dev/md$mdstart $mntpoint
+	cp -R /usr/include $mntpoint
+	dd if=/dev/zero of=$mntpoint/big bs=1m count=10 status=none
+	find $mntpoint/* -delete
+
+	umount $mntpoint
+	mdconfig -d -u $mdstart
+	rm -f $diskimage $log
+fi
+exit $s
diff --git a/tools/test/stress2/misc/fsck10.sh b/tools/test/stress2/misc/fsck10.sh
new file mode 100755
index 000000000000..30479e0dfebb
--- /dev/null
+++ b/tools/test/stress2/misc/fsck10.sh
@@ -0,0 +1,171 @@
+#!/bin/sh
+
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2023 Peter Holm <pho@FreeBSD.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+# Variation of fsck9.sh with smaller disk size and newfs options '', 'U' and 'O1'
+# fsck_ffs core dump seen
+
+. ../default.cfg
+
+[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
+
+cc -o /tmp/flip -Wall -Wextra -O2 ../tools/flip.c || exit 1
+
+set -e
+prog=$(basename "$0" .sh)
+u1=$mdstart
+u2=$((mdstart + 1))
+mp1=${mntpoint}$u1
+mp2=${mntpoint}$u2
+mkdir -p $mp1 $mp2
+log=$mp1/$prog.sh.log
+diskimage=$mp1/$prog.sh.diskimage
+backup=/tmp/$prog.sh.diskimage.`date +%Y%m%dT%H%M%S`.gz
+cleans=0
+reruns=0
+
+# Pick a random newfs flag
+newfs_flags=$(echo "" "-U" "-O1" | awk -v N=`jot -r 1 1 3` '{print $N}')
+[ $# -eq 1 ] && newfs_flags="$1" # or use script argument
+max=$((512 * 1024))
+
+mount | grep "on $mp1 " | grep -q /dev/md && umount -f $mp1
+[ -c /dev/md$u1 ] && mdconfig -d -u $u1
+mdconfig -a -t swap -s 1g -u $u1
+newfs $newfs_flags -n /dev/md$u1 > /dev/null
+mount /dev/md$u1 $mp1
+
+[ -c /dev/md$u2 ] && mdconfig -d -u $u2
+dd if=/dev/zero of=$diskimage bs=$max count=1 status=none
+mdconfig -a -t vnode -f $diskimage -u $u2
+backups=`newfs -N $newfs_flags md$u2 | grep -A1 "super-block backups" | \
+    tail -1 | sed 's/,//g'`
+echo "newfs $newfs_flags -n md$u2"
+newfs $newfs_flags -n md$u2 > /dev/null
+set +e
+
+chk() {
+	local i
+
+	clean=0
+	rerun=0
+	fsck_ffs -fy $1 > $log 2>&1
+	r=$?
+	if grep -qiE "super-?block.*failed" $log; then
+		for b in $backups; do
+			echo "fsck_ffs -b $b -fy $1"
+			fsck_ffs -b $b -fy $1 > $log 2>&1
+			r=$?
+			grep -qiE "super-?block.*failed" $log ||
+			   break
+			echo "Checking next SB"
+		done
+		usedasb=1
+	else
+		usedasb=0
+	fi
+	LANG=C egrep -q "[A-Z][A-Z]" $log && clean=0
+	grep -Eq "IS CLEAN|MARKED CLEAN" $log && clean=1
+	grep -q RERUN $log && rerun=1
+	[ $r -ne 0 -a $clean -eq 1 ] && echo "Exit code $r w/ clean == 1"
+}
+
+cd $mp1
+clean=0
+errors=0
+s=0
+start=`date +%s`
+while [ $((`date +%s` - start)) -lt 300 ]; do
+	mount /dev/md$u2 $mp2 || break
+	if ! ls -lR $mp2 > /dev/null; then
+		s=102
+		echo "ls failed"; grep "core dumped" /var/log/messages | tail -1
+		break
+	fi
+	rm -f $mp2/????????
+	touch $mp2/`jot -rc 8 a z | tr -d '\n'`
+	mkdir -p $mp2/dir/dir/dir/dir
+	echo "abc" > $mp2/dir/dir/dir/dir/f
+	while mount | grep -q "on $mp2 "; do umount $mp2; done
+	echo * | grep -q core && break
+	mdconfig -d -u $u2
+
+	# Introduce 5 random single bit errors to the file system image
+	/tmp/flip -n 5 $diskimage
+
+	sync; sleep .1
+	if [ `stat -f%z $diskimage` -gt $max ]; then
+		ls -lh $diskimage
+		truncate -s $max $diskimage
+	else
+		gzip < $diskimage > $backup
+	fi
+	fsync $backup
+	sync; sleep .1
+
+	for i in `jot 5`; do
+		[ $i -gt 2 ] && echo "fsck run #$i"
+		chk $diskimage
+		[ $rerun -eq 1 ] && { reruns=$((reruns + 1)); continue; }
+		[ $clean -eq 1 ] && { cleans=$((cleans + 1)); break; }
+		if [ -f fsck_ffs.core ]; then
+			tstamp=`date +%Y%m%dT%H%M%S`
+			gzip < $backup > /tmp/fsck_ffs.core.diskimage.$tstamp
+			gzip < fsck_ffs.core > /tmp/fsck_ffs.core.$tstamp
+			break 2
+		fi
+	done
+	if [ $clean -eq 1 ]; then
+		fsck_ffs -fy $diskimage > $log 2>&1
+		if grep -q MODIFIED $log; then
+			echo "*** fsck of \"clean\" FS found more issues:"
+			cat $log
+			errors=$((errors + 1))
+			s=1
+			break
+		fi
+	fi
+	[ $clean -ne 1 ] && break
+	mdconfig -a -t vnode -f $diskimage -u $u2
+done
+for i in `jot 5`; do
+	mount | grep -q "on $mp2 " || break
+	umount $mp2 && break
+	sleep 2
+done
+mdconfig -l | grep -q $u2 && mdconfig -d -u $u2
+
+[ $s -eq 0 ] && rm -f $backup || echo "Preserved $backup due to status code $s"
+cd /tmp
+for i in `jot 5`; do
+	umount $mp1 && break
+	sleep 2
+done
+mdconfig -d -u $u1
+rm -f /tmp/flip
+exit $s
diff --git a/tools/test/stress2/misc/fsck11.sh b/tools/test/stress2/misc/fsck11.sh
new file mode 100755
index 000000000000..cb5d1d256be6
--- /dev/null
+++ b/tools/test/stress2/misc/fsck11.sh
@@ -0,0 +1,170 @@
+#!/bin/sh
+
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2023 Peter Holm <pho@FreeBSD.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+# Demonstrate how the "CLEAN" message and the exit code can be misleading
+
+# "panic: softdep_update_inodeblock inconsistent ip ..." seen:
+# https://people.freebsd.org/~pho/stress/log/log0421.txt
+# https://people.freebsd.org/~pho/fsck11.sh.diskimage.20230228T064402.gz
+
+. ../default.cfg
+
+[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
+
+cc -o /tmp/flip -Wall -Wextra -O2 ../tools/flip.c || exit 1
+
+set -e
+prog=$(basename "$0" .sh)
+u1=$mdstart
+u2=$((mdstart + 1))
+mp1=${mntpoint}$u1
+mp2=${mntpoint}$u2
+mkdir -p $mp1 $mp2
+log=$mp1/$prog.sh.log
+diskimage=$mp1/$prog.sh.diskimage
+backup=/tmp/$prog.sh.diskimage.`date +%Y%m%dT%H%M%S`.gz
+cleans=0
+reruns=0
+
+# Pick a random newfs flag
+newfs_flags=$(echo "" "-U" "-O1" | awk -v N=`jot -r 1 1 3` '{print $N}')
+[ $# -eq 1 ] && newfs_flags="$1" # or use script argument
+max=$((2 * 1024 * 1024))
+[ "$newfs_flags" == "-j" ] && max=$((20 * 1024 * 1024)) # Make room for the journal file
+
+mount | grep "on $mp1 " | grep -q /dev/md && umount -f $mp1
+[ -c /dev/md$u1 ] && mdconfig -d -u $u1
+mdconfig -a -t swap -s 1g -u $u1
+newfs $newfs_flags -n /dev/md$u1 > /dev/null
+mount /dev/md$u1 $mp1
+
+[ -c /dev/md$u2 ] && mdconfig -d -u $u2
+dd if=/dev/zero of=$diskimage bs=$max count=1 status=none
+mdconfig -a -t vnode -f $diskimage -u $u2
+backups=`newfs -N $newfs_flags md$u2 | grep -A1 "super-block backups" | \
+    tail -1 | sed 's/,//g'`
+echo "newfs $newfs_flags -n md$u2"
+newfs $newfs_flags -n md$u2 > /dev/null
+set +e
+
+chk() {
+	local i
+
+	clean=0
+	rerun=0
+	fsck_ffs -fy $1 > $log 2>&1
+	r=$?
+	if grep -qiE "super-?block.*failed" $log; then
+		for b in $backups; do
+			echo "fsck_ffs -b $b -fy $1"
+			fsck_ffs -b $b -fy $1 > $log 2>&1
+			r=$?
+			grep -qiE "super-?block.*failed" $log ||
+			   break
+			echo "Checking next SB"
+		done
+		usedasb=1
+	else
+		usedasb=0
+	fi
+	LANG=C egrep -q "[A-Z][A-Z]" $log && clean=0
+	grep -Eq "IS CLEAN|MARKED CLEAN" $log && clean=1
+	grep -q RERUN $log && rerun=1
+	[ $r -ne 0 -a $clean -eq 1 ] && echo "Exit code $r w/ clean == 1"
+}
+
+cd $mp1
+clean=0
+errors=0
+s=0
+start=`date +%s`
+while [ $((`date +%s` - start)) -lt 300 ]; do
+	mount /dev/md$u2 $mp2 || break
+	if ! ls -lR $mp2 > /dev/null; then
+		s=102
+		echo "ls failed"; grep "core dumped" /var/log/messages | tail -1
+		break
+	fi
+
+	find $mp2 -type f | xargs cat > /dev/null
+	for j in `jot 9`; do
+		rm -rf $mp2/$j
+		mkdir $mp2/$j
+		jot 10 | xargs -P0 -I% cp /etc/group $mp2/$j/%
+	done 2>/dev/null
+
+	while mount | grep -q "on $mp2 "; do umount $mp2; done
+	echo * | grep -q core && break
+	mdconfig -d -u $u2
+
+	# Introduce 5 random single bit errors to the file system image
+	/tmp/flip -n 5 $diskimage
+
+	if [ `stat -f%z $diskimage` -gt $max ]; then
+		ls -lh $diskimage
+		truncate -s $max $diskimage
+	else
+		gzip < $diskimage > $backup
+	fi
+	fsync $backup
+	sync; sleep 1
+
+	[ $newfs_flags == "-j" ] &&
+		fsck -fy $diskimage > $log 2>&1	# process the journal file
+	for i in `jot 5`; do
+		[ $i -gt 2 ] && echo "fsck run #$i"
+		chk $diskimage
+		[ $rerun -eq 1 ] && { reruns=$((reruns + 1)); continue; }
+		[ $clean -eq 1 ] && { cleans=$((cleans + 1)); break; }
+		if [ -f fsck_ffs.core ]; then
+			tstamp=`date +%Y%m%dT%H%M%S`
+			gzip < $backup > /tmp/fsck_ffs.core.diskimage.$tstamp
+			gzip < fsck_ffs.core > /tmp/fsck_ffs.core.$tstamp
+			break 2
+		fi
+	done
+	[ $clean -ne 1 ] && { s=99; break; }	# broken image?
+	mdconfig -a -t vnode -f $diskimage -u $u2
+done
+for i in `jot 5`; do
+	mount | grep -q "on $mp2 " || break
+	umount $mp2 && break
+	sleep 2
+done
+mdconfig -l | grep -q $u2 && mdconfig -d -u $u2
+
+[ $s -eq 0 ] && rm -f $backup || echo "Preserved $backup due to status code $s"
+cd /tmp
+for i in `jot 5`; do
+	umount $mp1 && break
+	sleep 2
+done
+mdconfig -d -u $u1
+rm -f /tmp/flip
+exit $s
diff --git a/tools/test/stress2/misc/fsck12.sh b/tools/test/stress2/misc/fsck12.sh
new file mode 100755
index 000000000000..d2787e962927
--- /dev/null
+++ b/tools/test/stress2/misc/fsck12.sh
@@ -0,0 +1,177 @@
+#!/bin/sh
+
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2023 Peter Holm <pho@FreeBSD.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+# "panic: ffs_copyonwrite: bad copy block" seen:
+# Hunt for fsck_ffs's INTERNAL ERROR message
+
+# https://people.freebsd.org/~pho/stress/log/log0426.txt
+
+. ../default.cfg
+
+[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
+
+cc -o /tmp/flip -Wall -Wextra -O2 ../tools/flip.c || exit 1
+
+set -e
+prog=$(basename "$0" .sh)
+u1=$mdstart
+u2=$((mdstart + 1))
+mp1=${mntpoint}$u1
+mp2=${mntpoint}$u2
+mkdir -p $mp1 $mp2
+log=$mp1/$prog.sh.log
+diskimage=$mp1/$prog.sh.diskimage
+backup=/tmp/$prog.sh.diskimage.`date +%Y%m%dT%H%M%S`.gz
+cleans=0
+reruns=0
+
+mount | grep "on $mp1 " | grep -q /dev/md && umount -f $mp1
+[ -c /dev/md$u1 ] && mdconfig -d -u $u1
+mdconfig -a -t swap -s 1g -u $u1
+newfs $newfs_flags -n /dev/md$u1 > /dev/null
+mount /dev/md$u1 $mp1
+
+newfs_flags='-j'
+[ $# -eq 1 ] && newfs_flags="$1" # or use script argument
+max=$((2 * 1024 * 1024))
+[ "$newfs_flags" == "-j" ] && max=$((20 * 1024 * 1024)) # Make room for the journal file
+
+[ -c /dev/md$u2 ] && mdconfig -d -u $u2
+dd if=/dev/zero of=$diskimage bs=$max count=1 status=none
+mdconfig -a -t vnode -f $diskimage -u $u2
+backups=`newfs -N $newfs_flags md$u2 | grep -A1 "super-block backups" | \
+    tail -1 | sed 's/,//g'`
+echo "newfs $newfs_flags md$u2"
+newfs $newfs_flags md$u2 > /dev/null
+set +e
+
+chk() {
+	local i
+
+	clean=0
+	rerun=0
+	fsck_ffs -fy $1 > $log 2>&1
+	r=$?
+	grep "GOT TO reply" $log && exit 1
+	if grep -qiE "super-?block.*failed" $log; then
+		for b in $backups; do
+			echo "fsck_ffs -b $b -fy $1"
+			fsck_ffs -b $b -fy $1 > $log 2>&1
+			r=$?
+			grep "GOT TO reply" $log && exit 1
+			grep -qiE "super-?block.*failed" $log ||
+			   break
+			echo "Checking next SB"
+		done
+		usedasb=1
+	else
+		usedasb=0
+	fi
+	LANG=C egrep -q "[A-Z][A-Z]" $log && clean=0
+	grep -Eq "IS CLEAN|MARKED CLEAN" $log && clean=1
+	grep -q RERUN $log && rerun=1
+	[ $r -ne 0 -a $clean -eq 1 ] && echo "Exit code $r w/ clean == 1"
+}
+
+cd $mp1
+clean=0
+errors=0
+s=0
+sno=0
+start=`date +%s`
+while [ $((`date +%s` - start)) -lt 300 ]; do
+	mount /dev/md$u2 $mp2 || break
+	if ! ls -lR $mp2 > /dev/null; then
+		s=102
+		echo "ls failed"; grep "core dumped" /var/log/messages | tail -1
+		break
+	fi
+	mkdir -p $mp2/d/d/d/d/d
+	cp /etc/group $mp2/d/d/d/d/d/f
+	for i  in `jot 20`; do
+		cp -r $mp2/d $i
+	done
+
+	rm -f $mp2/.snap/$sno
+	mksnap_ffs $mp2 $mp2/.snap/$sno &
+	sno=$(((sno + 1) % 10))
+		
+	sleep 1
+	rm -rf $mp/?
+	wait
+	while mount | grep -q "on $mp2 "; do umount $mp2; done
+	echo * | grep -q core && break
+	mdconfig -d -u $u2
+
+	# Introduce 5 random single bit errors to the file system image
+	/tmp/flip -n 5 $diskimage
+
+	if [ `stat -f%z $diskimage` -gt $max ]; then
+		ls -lh $diskimage
+		truncate -s $max $diskimage
+	else
+		gzip < $diskimage > $backup
+	fi
+	fsync $backup
+	sync; sleep 1
+
+	[ $newfs_flags == "-j" ] &&
+		fsck -fy $diskimage > $log 2>&1	# process the journal file
+	for i in `jot 5`; do
+		[ $i -gt 2 ] && echo "fsck run #$i"
+		chk $diskimage
+		[ $rerun -eq 1 ] && { reruns=$((reruns + 1)); continue; }
+		grep -q "MODIFIED" $log && continue # For now, do not trust CLEAN
+		[ $clean -eq 1 ] && { cleans=$((cleans + 1)); break; }
+		if [ -f fsck_ffs.core ]; then
+			tstamp=`date +%Y%m%dT%H%M%S`
+			gzip < $backup > /tmp/fsck_ffs.core.diskimage.$tstamp.gz
+			gzip < fsck_ffs.core > /tmp/fsck_ffs.core.$tstamp.gz
+			break 2
+		fi
+	done
+	[ $clean -ne 1 ] && { s=99; break; }	# broken image?
+	mdconfig -a -t vnode -f $diskimage -u $u2
+done
+for i in `jot 5`; do
+	mount | grep -q "on $mp2 " || break
+	umount $mp2 && break
+	sleep 2
+done
+mdconfig -l | grep -q $u2 && mdconfig -d -u $u2
+
+[ $s -eq 0 ] && rm -f $backup || echo "Preserved $backup due to status code $s"
+cd /tmp
+for i in `jot 5`; do
+	umount $mp1 && break
+	sleep 2
+done
+mdconfig -d -u $u1
+rm -f /tmp/flip
+exit $s
diff --git a/tools/test/stress2/misc/gnop13.sh b/tools/test/stress2/misc/gnop13.sh
new file mode 100755
index 000000000000..45668b9d8e82
--- /dev/null
+++ b/tools/test/stress2/misc/gnop13.sh
@@ -0,0 +1,138 @@
+#!/bin/sh
+
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2023 Peter Holm <pho@FreeBSD.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+# Snapshot test
+# Variation of force13.sh, but uses "gnop -f destroy"
+
+# Seen:
+# UFS: forcibly unmounting /dev/md10.nop from /mnt
+# panic: flush_pagedep_deps: failed to flush inodedep
+
+[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
+. ../default.cfg
+
+set -u
+log=/tmp/gnop13.sh.log
+mount | grep $mntpoint | grep -q /dev/md && umount -f $mntpoint
+mdconfig -l | grep -q md$mdstart && mdconfig -d -u $mdstart
+truncate -s 10g $diskimage
+mdconfig -a -t vnode -f $diskimage -u $mdstart
+flags=$newfs_flags
+[ `jot -r 1 0 1` -eq 1 ] && flags="-j"
+
+echo "newfs $flags md$mdstart"
+newfs $flags md$mdstart > /dev/null 2>&1
+
+# Exclude rename for now due to log0374.txt
+export TESTPROGS=`cd ..; find testcases/ -perm -1 -type f | \
+    egrep -Ev "/run/|/badcode/|/pty/|/shm/|/socket/|sysctl|tcp|thr|udp|rename"`
+export runRUNTIME=3m
+export RUNDIR=$mntpoint/stressX
+export CTRLDIR=$mntpoint/stressX.control
+start=`date +%s`
+while [ $((`date +%s` - start)) -lt $((15 * 60)) ]; do
+	gnop create /dev/md$mdstart || exit 1
+	mount /dev/md$mdstart.nop $mntpoint || exit 1
+	rm -fr $mntpoint/lost+found
+	chmod 777 $mntpoint
+
+	su $testuser -c 'cd ..; ./testcases/run/run $TESTPROGS' > \
+	    /dev/null 2>&1 &
+
+	t=`jot -r 1 60 120`
+	st=`date +%s`
+	for i in `jot 10`; do
+		[ -d $mntpoint/.snap ] || break
+		rm -f $mntpoint/.snap/$i
+		echo "mksnap_ffs $mntpoint $mntpoint/.snap/$i"
+		mksnap_ffs $mntpoint $mntpoint/.snap/$i
+		sleep `jot -r 1 1 5`
+		[ $((`date +%s` - st)) -ge $t ] && break
+	done &
+	sleep `jot -r 1 60 120`
+	while [ -c /dev/md$mdstart.nop ]; do
+		echo "gnop destroy -f /dev/md$mdstart.nop"
+		gnop destroy -f /dev/md$mdstart.nop || sleep 1
+	done
+	n=0
+	st=`date +%s`
+	while mount | grep -q "on $mntpoint "; do
+		[ $n -eq 0 ] && /bin/echo -n "Waiting for $mntpoint to force umount ..."
+		n=$((n + 1))
+		sleep 2
+		if [ $((`date +%s` - st)) -ge 180 ]; then
+			echo "Giving up on waiting for umount of $mntpoint"
+			umount $mntpoint || umount -f $mntpoint
+			break
+		fi
+	done
+	[ $n -ne 0 ] && echo
+
+	../tools/killall.sh
+	wait
+	mount | grep -q "on $mntpoint " && umount -f $mntpoint
+	c=0
+	# Run fsck minimum two times
+	for i in `jot 5`; do
+		fsck_ffs -fy /dev/md$mdstart > $log 2>&1; s=$?
+		grep -q CLEAN $log && grep -q "MODIFIED" $log && c=$((c+=1))
+		grep -Eq "FILE SYSTEM WAS MODIFIED" $log || break
+	done
+	[ $c -gt 1 ] &&
+	    { echo "Note: FS marked clean+modified $c times out of $i fsck runs"; s=101; }
+	[ $s -ne 0 ] && break
+	grep -Eq "IS CLEAN|MARKED CLEAN" $log || { s=102; break; }
+done
+[ $s -eq 101 ] && s=0 # Ignore CLEANish problem for now
+if [ $s -eq 0 ]; then
+	mount /dev/md$mdstart.nop $mntpoint
+	cp -R /usr/include $mntpoint
+	dd if=/dev/zero of=$mntpoint/big bs=1m count=10 status=none
+	find $mntpoint/* -delete
+
+	# Check the RO snapshots
+	for f in $mntpoint/.snap/*; do
+		c=0
+		for i in `jot 5`; do
+			echo "fsck_ffs -fy $f"
+			fsck_ffs -fy $f > $log 2>&1; s=$?
+			grep -q CLEAN $log && grep -q "MODIFIED" $log && c=$((c+=1))
+			grep -Eq "FILE SYSTEM WAS MODIFIED" $log || break
+		done
+		[ $c -gt 1 ] &&
+		    { echo "Note: snapshot $i marked clean+modified $c times out of $i fsck runs"; s=201; }
+		[ $s -ne 0 ] && break
+		grep -Eq "IS CLEAN|MARKED CLEAN" $log ||
+		    { s=202; tail -10 $log; break; }
+	done
+	umount $mntpoint
+	mdconfig -d -u $mdstart
+	rm -f $diskimage $log
+fi
+exit $s