Panic when removing a SCSI device entry

[Joerg Wunsch]

As Kostik Belousov wrote:

> Please do "p *(struct cdev_priv *)0xe98804f4" and
> "p *(struct cdev_priv *)0xce0dc900" from kgdb.

Well, that kernel unfortunately lacked debugging symbols, and while
I've still been thinking about the best way to recompile an exact
same kernel with them ...

> I am pretty much sure that INVARIANTS kernel would hit the assert
> about SI_NAMED flag being clear on destroy_devl() invocation.
> We would have catched the issue earlier, with less interesting data
> destroyed.

... the now INVARIANTS kernel panicked again last night.  This time,
I've got debugging symbols.  So, as you expected, the corruption had
been caught earliere now.  The panic message is:

(kgdb) p panicstr
$1 = 0xc088dca0 "Bad link elm 0xc81cc200 prev->next != elm"

Here is the stack trace:

(kgdb) bt
#0  doadump () at pcpu.h:231
#1  0xc057943e in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:419
#2  0xc0579710 in panic (fmt=Variable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:592
#3  0xc044b707 in camperiphfree (periph=0xc81cc200) at /usr/src/sys/cam/cam_periph.c:550
#4  0xc044b8e5 in cam_periph_release_locked (periph=0xc81cc200) at /usr/src/sys/cam/cam_periph.c:336
#5  0xc044ba74 in cam_periph_release (periph=0xc81cc200) at /usr/src/sys/cam/cam_periph.c:352
#6  0xc046eea2 in saopen (dev=0xc8d9ba00, flags=1, fmt=8192, td=0xc93d15c0)
    at /usr/src/sys/cam/scsi/scsi_sa.c:499
#7  0xc053833e in giant_open (dev=0xc8d9ba00, oflags=1, devtype=8192, td=0xc93d15c0)
    at /usr/src/sys/kern/kern_conf.c:361
#8  0xc05177b2 in devfs_open (ap=0xe98b3b00) at /usr/src/sys/fs/devfs/devfs_vnops.c:992
#9  0xc07b9a95 in VOP_OPEN_APV (vop=0xc08403c0, a=0xe98b3b00) at vnode_if.c:445
#10 0xc06143d6 in vn_open_cred (ndp=0xe98b3b78, flagp=0xe98b3c2c, cmode=0, vn_open_flags=0, 
    cred=0xc807a780, fp=0xc90f6d58) at vnode_if.h:196
#11 0xc06144db in vn_open (ndp=0xe98b3b78, flagp=0xe98b3c2c, cmode=0, fp=0xc90f6d58)
    at /usr/src/sys/kern/vfs_vnops.c:94
#12 0xc06133fc in kern_openat (td=0xc93d15c0, fd=-100, path=0x804a0bb <Address 0x804a0bb out of bounds>, 
    pathseg=UIO_USERSPACE, flags=1, mode=6) at /usr/src/sys/kern/vfs_syscalls.c:1083
#13 0xc0613845 in kern_open (td=0xc93d15c0, path=0x804a0bb <Address 0x804a0bb out of bounds>, 
    pathseg=UIO_USERSPACE, flags=0, mode=6) at /usr/src/sys/kern/vfs_syscalls.c:1039
#14 0xc06138c0 in open (td=0xc93d15c0, uap=0xe98b3cec) at /usr/src/sys/kern/vfs_syscalls.c:1015
#15 0xc05b6276 in syscallenter (td=0xc93d15c0, sa=0xe98b3ce4) at /usr/src/sys/kern/subr_trap.c:326
#16 0xc0799b54 in syscall (frame=0xe98b3d28) at /usr/src/sys/i386/i386/trap.c:1086
#17 0xc077fd21 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:266
#18 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)

The dev node in question (I think, it's the "dev" argument in stack
frame #6) is:
(kgdb) p dev
$5 = (struct cdev *) 0xc8d9ba00
(kgdb) p *dev
$6 = {__si_reserved = 0x0, si_flags = 4, si_atime = {tv_sec = 1305772805, tv_nsec = 0}, si_ctime = {
    tv_sec = 1305773153, tv_nsec = 0}, si_mtime = {tv_sec = 1305773153, tv_nsec = 0}, si_uid = 0, 
  si_gid = 5, si_mode = 432, si_cred = 0x0, si_drv0 = 1, si_refcount = 2, si_list = {
    le_next = 0xc7f9ed00, le_prev = 0xc8d43538}, si_clone = {le_next = 0x0, le_prev = 0x0}, 
  si_children = {lh_first = 0xc8d9f100}, si_siblings = {le_next = 0x0, le_prev = 0x0}, si_parent = 0x0, 
  si_name = 0xc8d9ba78 "nsa0.0", si_drv1 = 0xc81cc200, si_drv2 = 0x0, si_devsw = 0xc0833a40, 
  si_iosize_max = 65536, si_usecount = 1, si_threadcount = 2, __si_u = {__sid_snapdata = 0x0}, 
  __si_namebuf = "nsa0.0", '\0' <repeats 57 times>}

The contents of the "periph" object as seen in the various CAM layer
functions is:

(kgdb) p *periph
$1 = {pinfo = {priority = 1, generation = 87063, index = -1}, periph_start = 0xc046ba80 <sastart>, 
  periph_oninval = 0xc046bcf0 <saoninvalidate>, periph_dtor = 0xc046f2d0 <sacleanup>, 
  periph_name = 0xc07e07bc "sa", path = 0xc8dd31b0, softc = 0xc8387800, sim = 0xc6e83c80, 
  unit_number = 0, type = CAM_PERIPH_BIO, flags = 8, immediate_priority = 4294967295, refcount = 0, 
  ccb_list = {slh_first = 0x0}, periph_links = {sle_next = 0x0}, unit_links = {tqe_next = 0x0, 
    tqe_prev = 0xc0833b30}, deferred_callback = 0, deferred_ac = 0}

Finally, here's the "show cdev" command from DDB:

db> show cdev
geom.ctl 0xc6d1a100
devctl 0xc6ccc700
console 0xc6ccc600
sndstat 0xc6ccc500
ptmx 0xc6ccc400
ctty 0xc6ccc300
mem 0xc6ccc200
kmem 0xc6db3800
audit 0xc6db3700
bpf 0xc6db3600
bpf0 0xc6db3500
null 0xc6db3400
zero 0xc6db3300
fd/0 0xc6db3200
stdin 0xc6db3100
fd/1 0xc6db3000
stdout 0xc6db2e00
fd/2 0xc6db2d00
stderr 0xc6db2c00
klog 0xc6db2b00
pci 0xc6db2a00
midistat 0xc6db2900
kbdmux0 0xc6db2700
kbd0 0xc6db2600
random 0xc6db2400
urandom 0xc6db2300
sysmouse 0xc6db2200
io 0xc6db2100
speaker 0xc6db2000
fido 0xc6d1be00
ata 0xc6d1bd00
acpi 0xc6d1b800
ttyu2 0xc6e7dd00
ttyu2.init 0xc6e7d800
ttyu2.lock 0xc6e7d700
cuau2 0xc6e7d600
cuau2.init 0xc6e7d500
cuau2.lock 0xc6e7d400
ttyu3 0xc6e7d000
ttyu3.init 0xc6e7ce00
ttyu3.lock 0xc6e7cd00
cuau3 0xc6e7cc00
cuau3.init 0xc6e7cb00
cuau3.lock 0xc6e7ca00
ttyu4 0xc6e7c600
ttyu4.init 0xc6e7c500
ttyu4.lock 0xc6e7c800
cuau4 0xc6e7c900
cuau4.init 0xc6e7d200
cuau4.lock 0xc6e7d300
ttyu5 0xc6e7e400
ttyu5.init 0xc6e7e500
ttyu5.lock 0xc6e7e600
cuau5 0xc6e7e700
cuau5.init 0xc6e7e800
cuau5.lock 0xc6e7e900
ttyu6 0xc6e7e000
ttyu6.init 0xc6f01e00
ttyu6.lock 0xc6f01d00
cuau6 0xc6f01c00
cuau6.init 0xc6f01b00
cuau6.lock 0xc6f01a00
ttyu7 0xc6f01600
ttyu7.init 0xc6f01500
ttyu7.lock 0xc6f01400
cuau7 0xc6f01300
cuau7.init 0xc6f01200
cuau7.lock 0xc6f01100
ttyu8 0xc6f00c00
ttyu8.init 0xc6f00b00
ttyu8.lock 0xc6f00a00
cuau8 0xc6f00900
cuau8.init 0xc6f00800
cuau8.lock 0xc6f00700
ttyu9 0xc6f00300
ttyu9.init 0xc6f00200
ttyu9.lock 0xc6f00100
cuau9 0xc6f00000
cuau9.init 0xc6e7fe00
cuau9.lock 0xc6e7fd00
ttyv0 0xc6f01000
ttyv1 0xc6f01800
ttyv2 0xc6fa5d00
ttyv3 0xc6fa5c00
ttyv4 0xc6fa5b00
ttyv5 0xc6fa5a00
ttyv6 0xc6fa5900
ttyv7 0xc6fa5800
ttyv8 0xc6fa5700
ttyv9 0xc6fa5600
ttyva 0xc6fa5500
ttyvb 0xc6fa5400
ttyvc 0xc6fa5300
ttyvd 0xc6fa5200
ttyve 0xc6fa5100
ttyvf 0xc6fa5000
consolectl 0xc6fa4e00
lpt0 0xc6fa4b00
lpt0.ctl 0xc6fa4a00
ppi0 0xc6fa4900
ttyu0 0xc6fa4600
ttyu0.init 0xc6fa4500
ttyu0.lock 0xc6fa4400
cuau0 0xc6fa4300
cuau0.init 0xc6fa4200
cuau0.lock 0xc6fa4100
usbctl 0xc71d6d00
mdctl 0xc71d6b00
devstat 0xc71d6a00
fd0 0xc71d6900
usb/0.1.0 0xc71d6700
ugen0.1 0xc71d6600
usb/1.1.0 0xc71d6500
ugen1.1 0xc71d6400
usb/0.1.1 0xc71d6300
usb/1.1.1 0xc71d5d00
xpt0 0xc71d5800
mixer0 0xc71d4a00
mixer1 0xc71d4000
mixer2 0xc7216a00
acd0 0xc7216100
ad4 0xc7216000
ad4s1 0xc7215e00
ad4s1b 0xc7215d00
ad4s1h 0xc7215c00
gvinum/ports 0xc728e100
gvinum/src 0xc728e000
gvinum/news 0xc728de00
gvinum/distfiles 0xc728dd00
gvinum/pdf 0xc728dc00
gvinum/mysql 0xc728db00
gvinum/upload 0xc728da00
gvinum/obj 0xc728d900
gvinum/root 0xc728d800
gvinum/local 0xc728d700
gvinum/usr 0xc728d600
gvinum/var 0xc728d500
gvinum/home_cvs 0xc728d400
gvinum/home 0xc728d300
gvinum/junk 0xc728d200
gvinum/bacula_db 0xc728d100
gvinum/dump 0xc728d000
gvinum/tmp 0xc72a4400
gvinum/camel 0xc72a4300
gvinum/squid 0xc72a4200
gvinum/sound 0xc72a4100
usb/1.2.0 0xc72a2b00
ugen1.2 0xc72a2a00
cd0 0xc7290c00
usb/1.2.1 0xc7290b00
pass0 0xc7290400
pass1 0xc7290300
pass2 0xc7290200
da0 0xc72e8800
da0a 0xc72a2700
da0h 0xc72a2800
da1 0xc72a2e00
ufsid/4856d98a00081994 0xc72a3000
da1a 0xc72a3100
da1h 0xc72a3200
usb/0.2.0 0xc72e6900
ugen0.2 0xc72e6a00
usb/0.2.1 0xc72e6b00
usb/0.3.0 0xc72e7400
ugen0.3 0xc72e7500
usb/0.3.1 0xc72e7600
ukbd0 0xc72e7e00
kbd1 0xc72e8000
usb/0.4.0 0xc72e8100
ugen0.4 0xc72e8200
usb/0.4.1 0xc72e8300
ums0 0xc72e7300
usb/0.5.0 0xc72e6e00
ugen0.5 0xc72e6c00
usb/0.5.1 0xc72e6100
usb/0.6.0 0xc72a4900
ugen0.6 0xc72a4800
usb/0.6.2 0xc72a4700
pf 0xc7449700
nfslock 0xc7e39100
tap0 0xc7e95b00
apm0 0xc7fa0100
dsp2.0 0xc7446100
dsp1.0 0xc7e37300
dsp0.0 0xc7e37100
pts/1 0xc8230600
pts/2 0xc8230900
ptyp0 0xc7e36d00
ttyp0 0xc8230700
ptyp1 0xc8230b00
ttyp1 0xc82eb800
pts/3 0xc8d9ae00
pts/0 0xc8047800
pts/4 0xc82c9400
pts/5 0xc8da1400
pts/6 0xc8e02700
usb/0.6.0 0xc8497a00
ugen0.6 0xc7e97500
usb/0.6.2 0xc82ed400
pass3 0xc8d9d700
ch0 0xc8046200
nsa0.0 0xc8d9ba00
esa0.0 0xc8d43500
nsa0 0xc8d9f100
esa0 0xc8d43c00
sa0.1 0xc8497e00
nsa0.1 0xc8048800
esa0.1 0xc8e00100
sa0.2 0xc8e01b00
nsa0.2 0xc8da0c00
esa0.2 0xc8d9d900
sa0.3 0xc8d9de00
nsa0.3 0xc8d9f900
esa0.3 0xc8d42600
usb/0.7.0 0xcecdad00
ugen0.7 0xcecd4a00
usb/0.7.2 0xcecd7800
pts/7 0xc8046100
ptyp2 0xc822da00
ttyp2 0xc8230300
pass4 0xce291100
sa0.ctl 0xcecfca00
sa0.0 0xcecfcd00
nsa0.0 0xc7f9e900
esa0.0 0xcecf3000
sa0 0xc8d42b00
nsa0 0xcecd7b00
esa0 0xc90a7000
sa0.1 0xcbda4c00
nsa0.1 0xcecfcc00
esa0.1 0xce21a400
sa0.2 0xceccb600
nsa0.2 0xcece3b00
esa0.2 0xc8e01d00
sa0.3 0xcecd4600
nsa0.3 0xceccb300
esa0.3 0xcecfc500

I think that's all I could tell by now ...
-- 
cheers, J"org               .-.-.   --... ...--   -.. .  DL8DTL

http://www.sax.de/~joerg/                        NIC: JW11-RIPE
Never trust an operating system you don't have sources for. ;-)