Locked up nfsd after avg@ sendfile patch

[Andriy Gapon]

on 13/10/2010 18:13 Sam Fourman Jr. said the following:
> 
> FNFS# uname -a
> FreeBSD FNFS.PuffyBSD.Com <http://FNFS.PuffyBSD.Com> 9.0-CURRENT FreeBSD
> 9.0-CURRENT #23: Wed Oct 13 08:07:13 CDT 2010    
> root at FNFS.PuffyBSD.Com:/usr/obj/usr/src/sys/FNFS  amd64
> FNFS#
> 
> 
> running CURRENT as of - r213742
> 
> FNFS# top -PS
> 
> last pid: 65634;  load averages:  3.39,  2.81, 
> 1.62                                                                                                    
> up 0+01:47:43  10:07:35
> 322 processes: 6 running, 295 sleeping, 21 waiting
> CPU 0: 74.8% user,  0.0% nice,  7.5% system,  0.0% interrupt, 17.7% idle
> CPU 1: 85.4% user,  0.0% nice,  4.3% system,  0.0% interrupt, 10.2% idle
> CPU 2: 83.1% user,  0.0% nice,  8.7% system,  0.0% interrupt,  8.3% idle
> CPU 3: 82.3% user,  0.0% nice,  4.7% system,  0.0% interrupt, 13.0% idle
> Mem: 145M Active, 17M Inact, 6101M Wired, 1352K Cache, 1402M Free
> Swap: 63G Total, 63G Free
> 
>   PID USERNAME       THR PRI NICE   SIZE    RES STATE   C   TIME   WCPU COMMAND
>    11 root             4 171 ki31     0K    64K RUN     0 409:24 203.32% idle
>     0 root           164  -8    0     0K  2608K -       0   1:54  0.00% kernel
>    12 root            21 -60    -     0K   336K WAIT    0   0:15  0.00% intr
>  1081 root             4  44    0 10052K  1080K zfs     0   0:14  0.00% nfsd
>     7 root             1  76    -     0K    16K ccb_sc  2   0:05  0.00% xpt_thrd
>     4 root             1  -8    -     0K    16K -       2   0:02  0.00% g_down
>     3 root             1  -8    -     0K    16K -       1   0:02  0.00% g_up
>    18 root             1  44    -     0K    16K zio->i  1   0:02  0.00% syncer
>     5 root             6  -8    -     0K   128K tx->tx  0   0:01  0.00% zfskern
>  2485 root             1  44    0 42480K  4232K select  0   0:01  0.00% sshd
>    13 root             1  44    -     0K    16K -       0   0:01  0.00% yarrow
>  2425     70           1  44    0 53492K  5584K select  3   0:00  0.00% postgres
>  2419     70           1  44    0 60780K  5804K select  2   0:00  0.00% postgres
>  2426     70           1  44    0 53492K  5584K select  0   0:00  0.00% postgres
>  2420     70           1  44    0 60780K  5788K select  1   0:00  0.00% postgres
>     2 root             1  -8    -     0K    16K -       1   0:00  0.00% g_event
>    14 root            28 -64    -     0K   448K -       0   0:00  0.00% usb
>  1916     70           1  44    0 53492K  5484K select  0   0:00  0.00% postgres
>  1594     70           1  44    0 60780K  5736K select  2   0:00  0.00% postgres
> 65625 root             1  96    0 26484K 18328K CPU2    2   0:00  0.00% cc1
> 44389 root             1  44    0  7296K   852K select  3   0:00  0.00% make
> 44471 root             1  45    0  7296K  1348K select  3   0:00  0.00% make
>  2857 sfourman         1  44    0 42480K  4456K select  0   0:00  0.00% sshd
>    17 root             1 -16    -     0K    16K vlruwt  2   0:00  0.00% vnlru
>  2427     70           1  44    0 53492K  5564K select  0   0:00  0.00% postgres
>  2421     70           1  44    0 60780K  5812K select  2   0:00  0.00% postgres
>  1066 root             1  44    0 11280K  1924K select  1   0:00  0.00% mountd
>  2858 sfourman         1  44    0 25060K  3400K select  1   0:00  0.00% sftp-server
>  3065 pixi             1  46    0  8096K  3716K tx->tx  0   0:00  0.00% cvsup
>  1625 root             1  44    0 11976K  3356K select  0   0:00  0.00% sendmail
>  2257 root             1  44    0 11976K  2900K select  3   0:00  0.00% sendmail
>  1955 root             1  44    0 11976K  3096K select  0   0:00  0.00% sendmail
>  1257 root             1  44    0 16408K  3392K select  1   0:00  0.00% sendmail
> FNFS# procstat -k -k 1081
>   PID    TID COMM             TDNAME           KSTACK                      
>  1081 100284 nfsd             nfsd: master     mi_switch+0x176 sleepq_wait+0x42
> _cv_wait+0x129 zil_commit+0x7a zfs_freebsd_fsync+0xd6 VOP_FSYNC_APV+0x6c
> nfsvno_fsync+0xb7 nfsrvd_commit+0xe3 nfsrvd_dorpc+0x39d nfssvc_program+0x447
> svc_run_internal+0x6e9 svc_run+0x8f nfsrvd_nfsd+0x92 nfssvc_nfsd+0x9b nfssvc+0xb6
> syscallenter+0x331 syscall+0x4b Xfast_syscall+0xe2
>  1081 100291 nfsd             nfsd: service    mi_switch+0x176 sleepq_wait+0x42
> _cv_wait+0x129 zil_commit+0x7a zfs_freebsd_fsync+0xd6 VOP_FSYNC_APV+0x6c
> nfsvno_fsync+0xb7 nfsrvd_commit+0xe3 nfsrvd_dorpc+0x39d nfssvc_program+0x447
> svc_run_internal+0x6e9 svc_thread_start+0xb fork_exit+0x11b fork_trampoline+0xe
>  1081 100292 nfsd             nfsd: service    mi_switch+0x176 sleepq_wait+0x42
> _cv_wait+0x129 zil_commit+0x7a zfs_freebsd_fsync+0xd6 VOP_FSYNC_APV+0x6c
> nfsvno_fsync+0xb7 nfsrvd_commit+0xe3 nfsrvd_dorpc+0x39d nfssvc_program+0x447
> svc_run_internal+0x6e9 svc_thread_start+0xb fork_exit+0x11b fork_trampoline+0xe
>  1081 100293 nfsd             nfsd: service    mi_switch+0x176 sleepq_wait+0x42
> __lockmgr_args+0x7a3 vop_stdlock+0x39 VOP_LOCK1_APV+0x52 _vn_lock+0x47
> zfs_fhtovp+0x30e nfsvno_fhtovp+0x4f nfsd_fhtovp+0xa6 nfsrvd_dorpc+0x9d0
> nfssvc_program+0x447 svc_run_internal+0x6e9 svc_thread_start+0xb fork_exit+0x11b
> fork_trampoline+0xe
> FNFS#

procstat -kk -a | fgrep zil_commit

-- 
Andriy Gapon