Re: ZFS deadlock in 14

Reply: Kevin Bowling : "Re: ZFS deadlock in 14"
In reply to: Kevin Bowling : "Re: ZFS deadlock in 14"
Go to: [ bottom of page ] [ top of archives ] [ this month ]
From: Cy Schubert <Cy.Schubert_at_cschubert.com>
Date: Thu, 10 Aug 2023 13:37:45 UTC
In message <CAK7dMtDJeuf8rjWbsNEZABUfeqpjUyCHzuOL9AAhKk93sy+PKg@mail.gmail.c
om>
, Kevin Bowling writes:
> Possibly https://github.com/openzfs/zfs/commit/2cb992a99ccadb78d97049b40bd4=
> 42eb4fdc549d
>
> On Tue, Aug 8, 2023 at 10:08=E2=80=AFAM Dag-Erling Sm=C3=B8rgrav <des@freeb=
> sd.org> wrote:
> >
> > At some point between 42d088299c (4 May) and f0c9703301 (26 June), a
> > deadlock was introduced in ZFS.  It is still present as of 9c2823bae9 (4
> > August) and is 100% reproducable just by starting poudriere bulk in a
> > 16-core VM and waiting a few hours until deadlkres kicks in.  In the
> > latest instance, deadlkres complained about a bash process:
> >
> >     #0  sched_switch (td=3Dtd@entry=3D0xfffffe02fb1d8000, flags=3Dflags@e=
> ntry=3D259) at /usr/src/sys/kern/sched_ule.c:2299
> >     #1  0xffffffff80b5a0a3 in mi_switch (flags=3Dflags@entry=3D259) at /u=
> sr/src/sys/kern/kern_synch.c:550
> >     #2  0xffffffff80babcb4 in sleepq_switch (wchan=3D0xfffff818543a9e70, =
> pri=3D64) at /usr/src/sys/kern/subr_sleepqueue.c:609
> >     #3  0xffffffff80babb8c in sleepq_wait (wchan=3D<unavailable>, pri=3D<=
> unavailable>) at /usr/src/sys/kern/subr_sleepqueue.c:660
> >     #4  0xffffffff80b1c1b0 in sleeplk (lk=3Dlk@entry=3D0xfffff818543a9e70=
> , flags=3Dflags@entry=3D2121728, ilk=3Dilk@entry=3D0x0, wmesg=3Dwmesg@entry=
> =3D0xffffffff8222a054 "zfs", pri=3D<optimized out>, pri@entry=3D64, timo=3D=
> timo@entry=3D6, queue=3D1) at /usr/src/sys/kern/kern_lock.c:310
> >     #5  0xffffffff80b1a23f in lockmgr_slock_hard (lk=3D0xfffff818543a9e70=
> , flags=3D2121728, ilk=3D<optimized out>, file=3D0xffffffff812544fb "/usr/s=
> rc/sys/kern/vfs_subr.c", line=3D3057, lwa=3D0x0) at /usr/src/sys/kern/kern_=
> lock.c:705
> >     #6  0xffffffff80c59ec3 in VOP_LOCK1 (vp=3D0xfffff818543a9e00, flags=
> =3D2105344, file=3D0xffffffff812544fb "/usr/src/sys/kern/vfs_subr.c", line=
> =3D3057) at ./vnode_if.h:1120
> >     #7  _vn_lock (vp=3Dvp@entry=3D0xfffff818543a9e00, flags=3D2105344, fi=
> le=3D<unavailable>, line=3D<unavailable>, line@entry=3D3057) at /usr/src/sy=
> s/kern/vfs_vnops.c:1815
> >     #8  0xffffffff80c4173d in vget_finish (vp=3D0xfffff818543a9e00, flags=
> =3D<unavailable>, vs=3Dvs@entry=3DVGET_USECOUNT) at /usr/src/sys/kern/vfs_s=
> ubr.c:3057
> >     #9  0xffffffff80c1c9b7 in cache_lookup (dvp=3Ddvp@entry=3D0xfffff802c=
> d02ac40, vpp=3Dvpp@entry=3D0xfffffe046b20ac30, cnp=3Dcnp@entry=3D0xfffffe04=
> 6b20ac58, tsp=3Dtsp@entry=3D0x0, ticksp=3Dticksp@entry=3D0x0) at /usr/src/s=
> ys/kern/vfs_cache.c:2086
> >     #10 0xffffffff80c2150c in vfs_cache_lookup (ap=3D<optimized out>) at =
> /usr/src/sys/kern/vfs_cache.c:3068
> >     #11 0xffffffff80c32c37 in VOP_LOOKUP (dvp=3D0xfffff802cd02ac40, vpp=
> =3D0xfffffe046b20ac30, cnp=3D0xfffffe046b20ac58) at ./vnode_if.h:69
> >     #12 vfs_lookup (ndp=3Dndp@entry=3D0xfffffe046b20abd8) at /usr/src/sys=
> /kern/vfs_lookup.c:1266
> >     #13 0xffffffff80c31ce1 in namei (ndp=3Dndp@entry=3D0xfffffe046b20abd8=
> ) at /usr/src/sys/kern/vfs_lookup.c:689
> >     #14 0xffffffff80c52090 in kern_statat (td=3D0xfffffe02fb1d8000, flag=
> =3D<optimized out>, fd=3D-100, path=3D0xa75b480e070 <error: Cannot access m=
> emory at address 0xa75b480e070>, pathseg=3Dpathseg@entry=3DUIO_USERSPACE, s=
> bp=3Dsbp@entry=3D0xfffffe046b20ad18)
> >         at /usr/src/sys/kern/vfs_syscalls.c:2441
> >     #15 0xffffffff80c52797 in sys_fstatat (td=3D<unavailable>, uap=3D0xff=
> fffe02fb1d8400) at /usr/src/sys/kern/vfs_syscalls.c:2419
> >     #16 0xffffffff81049398 in syscallenter (td=3D<optimized out>) at /usr=
> /src/sys/amd64/amd64/../../kern/subr_syscall.c:190
> >     #17 amd64_syscall (td=3D0xfffffe02fb1d8000, traced=3D0) at /usr/src/s=
> ys/amd64/amd64/trap.c:1199
> >     #18 <signal handler called>
> >
> > The lock it is trying to acquire in frame 5 belongs to another bash
> > process which is in the process of creating a fifo:
> >
> >     #0  sched_switch (td=3Dtd@entry=3D0xfffffe046acd8e40, flags=3Dflags@e=
> ntry=3D259) at /usr/src/sys/kern/sched_ule.c:2299
> >     #1  0xffffffff80b5a0a3 in mi_switch (flags=3Dflags@entry=3D259) at /u=
> sr/src/sys/kern/kern_synch.c:550
> >     #2  0xffffffff80babcb4 in sleepq_switch (wchan=3D0xfffff8018acbf154, =
> pri=3D87) at /usr/src/sys/kern/subr_sleepqueue.c:609
> >     #3  0xffffffff80babb8c in sleepq_wait (wchan=3D<unavailable>, pri=3D<=
> unavailable>) at /usr/src/sys/kern/subr_sleepqueue.c:660
> >     #4  0xffffffff80b59606 in _sleep (ident=3Dident@entry=3D0xfffff8018ac=
> bf154, lock=3Dlock@entry=3D0xfffff8018acbf120, priority=3Dpriority@entry=3D=
> 87, wmesg=3D0xffffffff8223af0e "zfs teardown inactive", sbt=3Dsbt@entry=3D0=
> , pr=3Dpr@entry=3D0, flags=3D256)
> >         at /usr/src/sys/kern/kern_synch.c:225
> >     #5  0xffffffff80b45dc0 in rms_rlock_fallback (rms=3D0xfffff8018acbf12=
> 0) at /usr/src/sys/kern/kern_rmlock.c:1015
> >     #6  0xffffffff80b45c93 in rms_rlock (rms=3D<unavailable>, rms@entry=
> =3D0xfffff8018acbf120) at /usr/src/sys/kern/kern_rmlock.c:1036
> >     #7  0xffffffff81fb147b in zfs_freebsd_reclaim (ap=3D<optimized out>) =
> at /usr/src/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c:5164
> >     #8  0xffffffff8111d245 in VOP_RECLAIM_APV (vop=3D0xffffffff822e71a0 <=
> zfs_vnodeops>, a=3Da@entry=3D0xfffffe0410f1c9c8) at vnode_if.c:2180
> >     #9  0xffffffff80c43569 in VOP_RECLAIM (vp=3D0xfffff802cdbaca80) at ./=
> vnode_if.h:1084
> >     #10 vgonel (vp=3Dvp@entry=3D0xfffff802cdbaca80) at /usr/src/sys/kern/=
> vfs_subr.c:4143
> >     #11 0xffffffff80c3ef61 in vtryrecycle (vp=3D0xfffff802cdbaca80) at /u=
> sr/src/sys/kern/vfs_subr.c:1693
> >     #12 vnlru_free_impl (count=3Dcount@entry=3D1, mnt_op=3Dmnt_op@entry=
> =3D0x0, mvp=3D0xfffff8010864da00) at /usr/src/sys/kern/vfs_subr.c:1344
> >     #13 0xffffffff80c49553 in vnlru_free_locked (count=3D1) at /usr/src/s=
> ys/kern/vfs_subr.c:1357
> >     #14 vn_alloc_hard (mp=3Dmp@entry=3D0x0) at /usr/src/sys/kern/vfs_subr=
> .c:1744
> >     #15 0xffffffff80c3f6f0 in vn_alloc (mp=3D0x0) at /usr/src/sys/amd64/i=
> nclude/atomic.h:375
> >     #16 getnewvnode_reserve () at /usr/src/sys/kern/vfs_subr.c:1888
> >     #17 0xffffffff81faa072 in zfs_create (dzp=3D0xfffff812200261d0, name=
> =3D0xfffff8011b8ac805 "sh-np.yPbxoo", vap=3D0xfffffe0410f1cc20, excl=3D<opt=
> imized out>, mode=3D<optimized out>, zpp=3Dzpp@entry=3D0xfffffe0410f1cbc8, =
> cr=3D0xfffff80140fb1100, flag=3D<optimized out>, vsecp=3D0x0, mnt_ns=3D0x0)
> >         at /usr/src/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_o=
> s.c:1146
> >     #18 0xffffffff81faea57 in zfs_freebsd_create (ap=3D0xfffffe0410f1cda0=
> ) at /usr/src/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c:4618
> >     #19 0xffffffff8111aa9a in VOP_MKNOD_APV (vop=3D0xffffffff822e71a0 <zf=
> s_vnodeops>, a=3Da@entry=3D0xfffffe0410f1cda0) at vnode_if.c:372
> >     #20 0xffffffff80c50207 in VOP_MKNOD (dvp=3D<unavailable>, cnp=3D0xfff=
> ffe0410f1cd50, vap=3D0xfffffe0410f1cc20, vpp=3D<optimized out>) at ./vnode_=
> if.h:188
> >     #21 kern_mkfifoat (td=3D0xfffffe046acd8e40, fd=3D-100, path=3D0x12772=
> f073500 <error: Cannot access memory at address 0x12772f073500>, pathseg=3D=
> UIO_USERSPACE, mode=3D<optimized out>) at /usr/src/sys/kern/vfs_syscalls.c:=
> 1492
> >     #22 0xffffffff81049398 in syscallenter (td=3D<optimized out>) at /usr=
> /src/sys/amd64/amd64/../../kern/subr_syscall.c:190
> >     #23 amd64_