panic(s) in ZFS on CURRENT

  Hi,

I got several panics on my desktop running eb2b00da564 which is
after the latest OpenZFS merge.

#1 (couple cores with this backtrace)

--- trap 0x9, rip = 0xffffffff803ab94b, rsp = 0xfffffe022e45ed30, rbp = 0xfffffe022e45ed50 ---
buf_hash_insert() at buf_hash_insert+0xab/frame 0xfffffe022e45ed50
arc_write_done() at arc_write_done+0xfa/frame 0xfffffe022e45ed90
zio_done() at zio_done+0xf0b/frame 0xfffffe022e45ee00
zio_execute() at zio_execute+0x9f/frame 0xfffffe022e45ee40
taskqueue_run_locked() at taskqueue_run_locked+0x181/frame 0xfffffe022e45eec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xc3/frame 0xfffffe022e45eef0
fork_exit() at fork_exit+0x7d/frame 0xfffffe022e45ef30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe022e45ef30
--- trap 0, rip = 0, rsp = 0, rbp = 0 ---
(kgdb) frame 7
#7  buf_hash_insert (hdr=hdr@entry=0xfffff8001b21fa28, lockp=lockp@entry=0xfffffe022e45ed60)
    at /usr/src/FreeBSD/sys/contrib/openzfs/module/zfs/arc.c:1062
1062                    if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
(kgdb) p hdr
$1 = (arc_buf_hdr_t *) 0xfffff8001b21fa28
(kgdb) p *hdr
$2 = {b_dva = {dva_word = {16, 20406677952}}, b_birth = 38447120, b_type = ARC_BUFC_METADATA, b_complevel = 255 '\377', b_reserved1 = 0 '\000', 
  b_reserved2 = 0, b_hash_next = 0x0, 
  b_flags = (ARC_FLAG_L2CACHE | ARC_FLAG_IO_IN_PROGRESS | ARC_FLAG_BUFC_METADATA | ARC_FLAG_HAS_L1HDR | ARC_FLAG_COMPRESSED_ARC | ARC_FLAG_COMPRESS_0 | ARC_FLAG_COMPRESS_1 | ARC_FLAG_COMPRESS_2 | ARC_FLAG_COMPRESS_3), b_psize = 8, b_lsize = 32, b_spa = 1230587331341359116, b_l2hdr = {
    b_dev = 0x0, b_daddr = 0, b_hits = 0, b_arcs_state = ARC_STATE_ANON, b_l2node = {list_next = 0x0, list_prev = 0x0}}, b_l1hdr = {b_cv = {
      cv_description = 0xffffffff80bb5b02 "hdr->b_l1hdr.b_cv", cv_waiters = 0}, b_byteswap = 10 '\n', b_state = 0xffffffff80ef23c0 <ARC_anon>, 
    b_arc_node = {list_next = 0x0, list_prev = 0x0}, b_arc_access = 0, b_mru_hits = 0, b_mru_ghost_hits = 0, b_mfu_hits = 0, 
    b_mfu_ghost_hits = 0, b_bufcnt = 1, b_buf = 0xfffff80003139d80, b_refcnt = {rc_count = 2}, b_acb = 0x0, b_pabd = 0xfffff80a35dc6480}, 
  b_crypt_hdr = {b_rabd = 0x10, b_ot = 2744191968, b_ebufcnt = 4, b_dsobj = 38340866, b_salt = "\001\000\000\000\000\000\000", 
    b_iv = "\000\000\000\000\000\000\000\000\220\000\026\017", b_mac = "\b\000 \000\f\230\262m\250\354\023\021\000\000\000"}}

#2 (single core)

--- trap 0x9, rip = 0xffffffff803ab94b, rsp = 0xfffffe0256158780, rbp = 0xfffffe02561587a0 ---
buf_hash_insert() at buf_hash_insert+0xab/frame 0xfffffe02561587a0
arc_hdr_realloc() at arc_hdr_realloc+0x138/frame 0xfffffe0256158800
arc_read() at arc_read+0x2dc/frame 0xfffffe02561588b0
dbuf_read() at dbuf_read+0xb3e/frame 0xfffffe02561589f0
dmu_buf_hold() at dmu_buf_hold+0x46/frame 0xfffffe0256158a30
zap_cursor_retrieve() at zap_cursor_retrieve+0x167/frame 0xfffffe0256158a90
zfs_freebsd_readdir() at zfs_freebsd_readdir+0x383/frame 0xfffffe0256158cc0
VOP_READDIR_APV() at VOP_READDIR_APV+0x1f/frame 0xfffffe0256158ce0
kern_getdirentries() at kern_getdirentries+0x186/frame 0xfffffe0256158dd0
sys_getdirentries() at sys_getdirentries+0x29/frame 0xfffffe0256158e00
amd64_syscall() at amd64_syscall+0x100/frame 0xfffffe0256158f30
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0256158f30
(kgdb) frame 7
#7  buf_hash_insert (hdr=0xfffff80c906b03e8, lockp=lockp@entry=0x0) at /usr/src/FreeBSD/sys/contrib/openzfs/module/zfs/arc.c:1062
1062                    if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
(kgdb) p *hdr
$1 = {b_dva = {dva_word = {16, 19965896928}}, b_birth = 36629088, b_type = ARC_BUFC_METADATA, b_complevel = 0 '\000', b_reserved1 = 0 '\000', 
  b_reserved2 = 0, b_hash_next = 0x0, 
  b_flags = (ARC_FLAG_BUFC_METADATA | ARC_FLAG_HAS_L1HDR | ARC_FLAG_HAS_L2HDR | ARC_FLAG_COMPRESSED_ARC | ARC_FLAG_COMPRESS_1), b_psize = 5, 
  b_lsize = 5, b_spa = 3583499065027950438, b_l2hdr = {b_dev = 0xfffffe02306c8000, b_daddr = 4917395456, b_hits = 0, 
    b_arcs_state = ARC_STATE_MRU, b_l2node = {list_next = 0xfffff801313fc9b0, list_prev = 0xfffff801313fca70}}, b_l1hdr = {b_cv = {
      cv_description = 0xffffffff80bb5b02 "hdr->b_l1hdr.b_cv", cv_waiters = 0}, b_byteswap = 10 '\n', 
    b_state = 0xffffffff80f02900 <ARC_l2c_only>, b_arc_node = {list_next = 0x0, list_prev = 0x0}, b_arc_access = 0, b_mru_hits = 0, 
    b_mru_ghost_hits = 0, b_mfu_hits = 0, b_mfu_ghost_hits = 0, b_bufcnt = 0, b_buf = 0x0, b_refcnt = {rc_count = 0}, b_acb = 0x0, b_pabd = 0x0}, 
  b_crypt_hdr = {b_rabd = 0x10, b_ot = 2786027712, b_ebufcnt = 4, b_dsobj = 36629088, b_salt = "\001\000\000\000\000\000\000", 
    b_iv = "\240\2769$\001\370\377\377\220\000\036\017", b_mac = "\b\000 \000fw\357\327i%\2731\000\200l0"}}

#3 (not ZFS, but VFS, could be related?)

--- trap 0x9, rip = 0xffffffff80801408, rsp = 0xfffffe02348cbcc0, rbp = 0xfffffe02348cbcf0 ---
pwd_chdir() at pwd_chdir+0x28/frame 0xfffffe02348cbcf0
kern_chdir() at kern_chdir+0x169/frame 0xfffffe02348cbe00
amd64_syscall() at amd64_syscall+0x100/frame 0xfffffe02348cbf30
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe02348cbf30

(kgdb) frame 7
#7  pwd_alloc () at /usr/src/FreeBSD/sys/kern/kern_descrip.c:3920
warning: Source file is more recent than executable.
3920            bzero(pwd, sizeof(*pwd));
(kgdb) p pwd
$1 = (struct pwd *) 0xfff2fff0fff1ffed
(kgdb) p *pwd
Cannot access memory at address 0xfff2fff0fff1ffed

I'm switching to INVARIANTS kernel right now and will see if that panics earlier.

-- 
Gleb Smirnoff