Process in T state does not want to die.....

Willem Jan Withagen wjw at digiware.nl
Thu Nov 28 20:53:03 UTC 2019


  # ps -o pid,lwp,flags,flags2,state,tracer,command -p 3532
  PID    LWP        F       F2 STAT TRACER COMMAND
3532 103955 11080081 00000000 TsJ       0 ceph-osd -i 5

# procstat -kk 3532
   PID    TID COMM                TDNAME              KSTACK
  3532 103166 ceph-osd            log                 mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103167 ceph-osd            service             mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103168 ceph-osd            admin_socket        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103169 ceph-osd            msgr-worker-0       mi_switch+0xe2 
thread_suspend_switch+0x140 thread_single+0x47b sigexit+0x53 
postsig+0x304 ast+0x327 fast_syscall_common+0x198
  3532 103170 ceph-osd            msgr-worker-1       mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103171 ceph-osd            msgr-worker-2       mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103172 ceph-osd            signal_handler      mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103406 ceph-osd            OpHistorySvc        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103407 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103418 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103419 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103421 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103427 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103663 ceph-osd            fn_anonymous        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103675 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103677 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103678 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103679 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103680 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103681 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103682 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103683 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103684 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103685 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103955 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104621 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104826 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104827 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104828 ceph-osd            wb_throttle         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104829 ceph-osd            filestore_sync      mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104830 ceph-osd            journal_write       mi_switch+0xe2 
sleepq_wait+0x2c _sleep+0x247 bwillwrite+0x97 dofilewrite+0x93 
sys_writev+0x6e amd64_syscall+0x364 fast_syscall_common+0x101
  3532 104831 ceph-osd            fn_jrn_objstore     mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104832 ceph-osd            tp_fstore_op        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104833 ceph-osd            tp_fstore_op        mi_switch+0xe2 
sleepq_wait+0x2c _sleep+0x247 bwillwrite+0x97 vn_open_cred+0xc8 
zfs_setextattr+0x216 VOP_SETEXTATTR_APV+0x7c extattr_set_vp+0x11d 
sys_extattr_set_fd+0xee amd64_syscall+0x364 fast_syscall_common+0x101
  3532 104834 ceph-osd            fn_odsk_fstore      mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104835 ceph-osd            fn_appl_fstore      mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104836 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104838 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104839 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104841 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104843 ceph-osd            fn_anonymous        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105398 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105399 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105400 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105401 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105402 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105403 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105404 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105405 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105406 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105407 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105408 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105409 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105410 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105411 ceph-osd            fn_anonymous        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105412 ceph-osd            finisher            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105413 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105414 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105415 ceph-osd            osd_srv_agent       mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105416 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105417 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105418 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105419 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105420 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105421 ceph-osd            tp_osd_cmd          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105422 ceph-osd            osd_srv_heartbt     mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
root at cephtest:~ # procstat -kk 3532 | less
   PID    TID COMM                TDNAME              KSTACK
  3532 103166 ceph-osd            log                 mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103167 ceph-osd            service             mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103168 ceph-osd            admin_socket        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103169 ceph-osd            msgr-worker-0       mi_switch+0xe2 
thread_suspend_switch+0x140 thread_single+0x47b sigexit+0x53 
postsig+0x304 ast+0x327 fast_syscall_common+0x198
  3532 103170 ceph-osd            msgr-worker-1       mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103171 ceph-osd            msgr-worker-2       mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103172 ceph-osd            signal_handler      mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103406 ceph-osd            OpHistorySvc        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103407 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103418 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103419 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103421 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103427 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103663 ceph-osd            fn_anonymous        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103675 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103677 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103678 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103679 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103680 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103681 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103682 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103683 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103684 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103685 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 103955 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104621 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104826 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104827 ceph-osd            -                   mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104828 ceph-osd            wb_throttle         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104829 ceph-osd            filestore_sync      mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104830 ceph-osd            journal_write       mi_switch+0xe2 
sleepq_wait+0x2c _sleep+0x247 bwillwrite+0x97 dofilewrite+0x93 
sys_writev+0x6e amd64_syscall+0x364 fast_syscall_common+0x101
  3532 104831 ceph-osd            fn_jrn_objstore     mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104832 ceph-osd            tp_fstore_op        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104833 ceph-osd            tp_fstore_op        mi_switch+0xe2 
sleepq_wait+0x2c _sleep+0x247 bwillwrite+0x97 vn_open_cred+0xc8 
zfs_setextattr+0x216 VOP_SETEXTATTR_APV+0x7c extattr_set_vp+0x11d 
sys_extattr_set_fd+0xee amd64_syscall+0x364 fast_syscall_common+0x101
  3532 104834 ceph-osd            fn_odsk_fstore      mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104835 ceph-osd            fn_appl_fstore      mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104836 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104838 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104839 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104841 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 104843 ceph-osd            fn_anonymous        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105398 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105399 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105400 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105401 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105402 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105403 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105404 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105405 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105406 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105407 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105408 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105409 ceph-osd            ms_dispatch         mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105410 ceph-osd            ms_local            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105411 ceph-osd            fn_anonymous        mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105412 ceph-osd            finisher            mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105413 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105414 ceph-osd            safe_timer          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105415 ceph-osd            osd_srv_agent       mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105416 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105417 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105418 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105419 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105420 ceph-osd            tp_osd_tp           mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105421 ceph-osd            tp_osd_cmd          mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
  3532 105422 ceph-osd            osd_srv_heartbt     mi_switch+0xe2 
thread_suspend_check+0x297 ast+0x3b9 doreti_ast+0x1f
root at cephtest:~ #



More information about the freebsd-hackers mailing list