Kernel panic: Page fault when loading kernel native iSCSI target (FreeBSD 10.0-STABLE #0 r268091)

Fri Jul 4 13:53:36 UTC 2014

Hello All,

We are getting kernel panics while reading and writing to an iSCSI target. It is the kernel implementation of iSCSI and we are running the initiators in Windows 2012R2 with load on two 10 GE links. The problem is repeatable, but occurs what appears to be within a random period from when the load is initiated. No obvious useful info in dmesg or syslog.

Backtrace from the kernel dump below.

I will try to collect additional information upon request.

Best Regards
Christer Eriksson

INFO -------------------------------------------------------------
Dump header from device /dev/ada1s1
Architecture: amd64
Architecture Version: 2
Dump Length: 2995580928B (2856 MB)
Blocksize: 512
Dumptime: Fri Jul 4 14:50:34 2014
Hostname: TestArray1.
Magic: FreeBSD Kernel Dump
Version String: FreeBSD 10.0-STABLE #0 r268091: Tue Jul 1 15:40:42 CEST 2014
root at TestArray1.:/usr/obj/usr/src/sys/GENERIC
Panic String: page fault
Dump Parity: 455418556
Bounds: 3
Dump Status: good

KGDB -------------------------------------------------------------
#kgdb /usr/obj/usr/src/sys/GENERIC/kernel.debug /var/crash/vmcore.3
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...

Unread portion of the kernel message buffer:

Fatal trap 12: page fault while in kernel mode
cpuid = 6; apic id = 06
fault virtual address= 0x0
fault code= supervisor write data, page not present
instruction pointer= 0x20:0xffffffff80ce2766
stack pointer = 0x28:0xfffffe1049ba38f0
frame pointer = 0x28:0xfffffe1049ba3940
code segment= base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags= interrupt enabled, resume, IOPL = 0
current process= 0 (cfiscsirx)
trap number= 12
panic: page fault
cpuid = 6
KDB: stack backtrace:
#0 0xffffffff8092a270 at kdb_backtrace+0x60
#1 0xffffffff808ef7c5 at panic+0x155
#2 0xffffffff80ce4a5f at trap_fatal+0x38f
#3 0xffffffff80ce4d78 at trap_pfault+0x308
#4 0xffffffff80ce4430 at trap+0x4a0
#5 0xffffffff80ccae32 at calltrap+0x8
#6 0xffffffff81c304c0 at cfiscsi_handle_data_segment+0xf0
#7 0xffffffff81c30eda at cfiscsi_receive_callback+0x5ea
#8 0xffffffff81c4f5bb at icl_receive_thread+0x11b
#9 0xffffffff808c037a at fork_exit+0x9a
#10 0xffffffff80ccb36e at fork_trampoline+0xe
Uptime: 9m51s
Dumping 2856 out of 65476 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%

Reading symbols from /boot/kernel/zfs.ko.symbols...done.
Loaded symbols for /boot/kernel/zfs.ko.symbols
Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
Loaded symbols for /boot/kernel/opensolaris.ko.symbols
Reading symbols from /boot/kernel/ums.ko.symbols...done.
Loaded symbols for /boot/kernel/ums.ko.symbols
Reading symbols from /boot/kernel/uhid.ko.symbols...done.
Loaded symbols for /boot/kernel/uhid.ko.symbols
Reading symbols from /boot/kernel/ctl.ko.symbols...done.
Loaded symbols for /boot/kernel/ctl.ko.symbols
Reading symbols from /boot/kernel/iscsi.ko.symbols...done.
Loaded symbols for /boot/kernel/iscsi.ko.symbols
Reading symbols from /boot/kernel/dtraceall.ko.symbols...done.
Loaded symbols for /boot/kernel/dtraceall.ko.symbols
Reading symbols from /boot/kernel/cyclic.ko.symbols...done.
Loaded symbols for /boot/kernel/cyclic.ko.symbols
Reading symbols from /boot/kernel/dtrace.ko.symbols...done.
Loaded symbols for /boot/kernel/dtrace.ko.symbols
Reading symbols from /boot/kernel/dtmalloc.ko.symbols...done.
Loaded symbols for /boot/kernel/dtmalloc.ko.symbols
Reading symbols from /boot/kernel/dtnfscl.ko.symbols...done.
Loaded symbols for /boot/kernel/dtnfscl.ko.symbols
Reading symbols from /boot/kernel/fbt.ko.symbols...done.
Loaded symbols for /boot/kernel/fbt.ko.symbols
Reading symbols from /boot/kernel/fasttrap.ko.symbols...done.
Loaded symbols for /boot/kernel/fasttrap.ko.symbols
Reading symbols from /boot/kernel/lockstat.ko.symbols...done.
Loaded symbols for /boot/kernel/lockstat.ko.symbols
Reading symbols from /boot/kernel/sdt.ko.symbols...done.
Loaded symbols for /boot/kernel/sdt.ko.symbols
Reading symbols from /boot/kernel/systrace.ko.symbols...done.
Loaded symbols for /boot/kernel/systrace.ko.symbols
Reading symbols from /boot/kernel/systrace_freebsd32.ko.symbols...done.
Loaded symbols for /boot/kernel/systrace_freebsd32.ko.symbols
Reading symbols from /boot/kernel/profile.ko.symbols...done.
Loaded symbols for /boot/kernel/profile.ko.symbols
#0 doadump (textdump=<value optimized out>) at pcpu.h:219
219pcpu.h: No such file or directory.
in pcpu.h
(kgdb) bt
#0 doadump (textdump=<value optimized out>) at pcpu.h:219
#1 0xffffffff808ef442 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:452
#2 0xffffffff808ef804 in panic (fmt=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:759
#3 0xffffffff80ce4a5f in trap_fatal (frame=<value optimized out>, eva=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:881
#4 0xffffffff80ce4d78 in trap_pfault (frame=0xfffffe1049ba3840, usermode=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:692
#5 0xffffffff80ce4430 in trap (frame=0xfffffe1049ba3840) at /usr/src/sys/amd64/amd64/trap.c:456
#6 0xffffffff80ccae32 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:232
#7 0xffffffff80ce2766 in bcopy () at /usr/src/sys/amd64/amd64/support.S:112
#8 0xffffffff8095be82 in m_copydata (m=<value optimized out>, off=<value optimized out>, len=<value optimized out>, cp=<value optimized out>)
at /usr/src/sys/kern/uipc_mbuf.c:887
#9 0xffffffff81c304c0 in cfiscsi_handle_data_segment (request=0xfffff80248460eb0, cdw=0xfffff80248484540)
at /usr/src/sys/modules/ctl/../../cam/ctl/ctl_frontend_iscsi.c:782
#10 0xffffffff81c30eda in cfiscsi_receive_callback (request=0xfffff80248460eb0) at /usr/src/sys/modules/ctl/../../cam/ctl/ctl_frontend_iscsi.c:916
#11 0xffffffff81c4f5bb in icl_receive_thread (arg=0xfffff80248a16980) at /usr/src/sys/modules/iscsi/../../dev/iscsi/icl.c:730
#12 0xffffffff808c037a in fork_exit (callout=0xffffffff81c4f4a0 <icl_receive_thread>, arg=0xfffff80248a16980, frame=0xfffffe1049ba3ac0)
at /usr/src/sys/kern/kern_fork.c:995
#13 0xffffffff80ccb36e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:606
#14 0x0000000000000000 in ?? ()
Current language: auto; currently minimal
(kgdb) bt full
#0 doadump (textdump=<value optimized out>) at pcpu.h:219
No locals.
#1 0xffffffff808ef442 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:452
No locals.
#2 0xffffffff808ef804 in panic (fmt=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:759
ap = {{gp_offset = 16, fp_offset = 48, overflow_arg_area = 0xfffffe1049ba3530, reg_save_area = 0xfffffe1049ba34b0}}
#3 0xffffffff80ce4a5f in trap_fatal (frame=<value optimized out>, eva=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:881
softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27, ssd_dpl = 0, ssd_p = 1, ssd_long = 1, ssd_def32 = 0, ssd_gran = 1}
#4 0xffffffff80ce4d78 in trap_pfault (frame=0xfffffe1049ba3840, usermode=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:692
rv = Cannot access memory at address 0x0
(kgdb) list *0xffffffff80ce2766
0xffffffff80ce2766 is at /usr/src/sys/amd64/amd64/support.S:113.
108cmpq%rcx,%rax/* overlapping && src < dst? */
109jb1f
110
111shrq$3,%rcx/* copy by 64-bit words */
112cld/* nope, copy forwards */
113rep
114movsq
115movq%rdx,%rcx
116andq$7,%rcx/* any bytes left? */
117rep
(kgdb) up
#1 0xffffffff808ef442 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:452
452doadump(TRUE);
(kgdb) up
#2 0xffffffff808ef804 in panic (fmt=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:759
759kern_reboot(bootopt);
(kgdb) up
#3 0xffffffff80ce4a5f in trap_fatal (frame=<value optimized out>, eva=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:881
881panic("%s", trap_msg[type]);
(kgdb) up
#4 0xffffffff80ce4d78 in trap_pfault (frame=0xfffffe1049ba3840, usermode=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:692
692trap_fatal(frame, eva);
(kgdb) up
#5 0xffffffff80ce4430 in trap (frame=0xfffffe1049ba3840) at /usr/src/sys/amd64/amd64/trap.c:456
456(void) trap_pfault(frame, FALSE);
(kgdb) up
#6 0xffffffff80ccae32 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:232
232calltrap
Current language: auto; currently asm
(kgdb) up
#7 0xffffffff80ce2766 in bcopy () at /usr/src/sys/amd64/amd64/support.S:112
112cld/* nope, copy forwards */
(kgdb) up
#8 0xffffffff8095be82 in m_copydata (m=<value optimized out>, off=<value optimized out>, len=<value optimized out>, cp=<value optimized out>)
at /usr/src/sys/kern/uipc_mbuf.c:887
887bcopy(mtod(m, caddr_t) + off, cp, count);
Current language: auto; currently minimal
(kgdb) up
#9 0xffffffff81c304c0 in cfiscsi_handle_data_segment (request=0xfffff80248460eb0, cdw=0xfffff80248484540)
at /usr/src/sys/modules/ctl/../../cam/ctl/ctl_frontend_iscsi.c:782
782icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
(kgdb) up
#10 0xffffffff81c30eda in cfiscsi_receive_callback (request=0xfffff80248460eb0) at /usr/src/sys/modules/ctl/../../cam/ctl/ctl_frontend_iscsi.c:916
916done = cfiscsi_handle_data_segment(request, cdw);
(kgdb) list
911
912io = cdw->cdw_ctl_io;
913KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
914 ("CTL_FLAG_DATA_IN"));
915
916done = cfiscsi_handle_data_segment(request, cdw);
917if (done) {
918CFISCSI_SESSION_LOCK(cs);
919TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
920CFISCSI_SESSION_UNLOCK(cs);
(kgdb) up
#11 0xffffffff81c4f5bb in icl_receive_thread (arg=0xfffff80248a16980) at /usr/src/sys/modules/iscsi/../../dev/iscsi/icl.c:730
730(ic->ic_receive)(response);
(kgdb) list
725icl_pdu_free(response);
726icl_conn_fail(ic);
727return;
728}
729
730(ic->ic_receive)(response);
731}
732}
733
734static void
(kgdb) up
#12 0xffffffff808c037a in fork_exit (callout=0xffffffff81c4f4a0 <icl_receive_thread>, arg=0xfffff80248a16980, frame=0xfffffe1049ba3ac0)
at /usr/src/sys/kern/kern_fork.c:995
995callout(arg, frame);
(kgdb) list
990 * cpu_set_fork_handler intercepts this function call to
991 * have this call a non-return function to stay in kernel mode.
992 * initproc has its own fork handler, but it does return.
993 */
994KASSERT(callout != NULL, ("NULL callout in fork_exit"));
995callout(arg, frame);
996
997/*
998 * Check if a kernel thread misbehaved and returned from its main
999 * function.
(kgdb) up
#13 0xffffffff80ccb36e in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:606
606callfork_exit
Current language: auto; currently asm
(kgdb) list
601
602ENTRY(fork_trampoline)
603movq%r12,%rdi/* function */
604movq%rbx,%rsi/* arg1 */
605movq%rsp,%rdx/* trapframe pointer */
606callfork_exit
607MEXITCOUNT
608jmpdoreti/* Handle any ASTs */
609
610/*