[Bug 234026] [panic] [dummynet] Repeatable panic in dummynet due to locking issues and use-after-free
bugzilla-noreply at freebsd.org
bugzilla-noreply at freebsd.org
Fri Dec 14 23:51:43 UTC 2018
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234026
Bug ID: 234026
Summary: [panic] [dummynet] Repeatable panic in dummynet due to
locking issues and use-after-free
Product: Base System
Version: 11.2-STABLE
Hardware: Any
OS: Any
Status: New
Keywords: crash
Severity: Affects Some People
Priority: ---
Component: kern
Assignee: net at FreeBSD.org
Reporter: eugen at freebsd.org
Hi!
I run multiple routers using FreeBSD 11.2-STABLE/amd64 r336962, ipfw+dummynet
and net/mpd5 daemon that dynamically creates/destroys ngXXX interfaces for
multiple PPPoE clients. If an interface ngXXX is destroyed while dummynet
pipe/queue keeps mbuf with m_pkthdr.rcvif pointing to freed struct ifnet,
kernel panices when taskqueue runs
dummynet_task/dummynet_send/netisr_dispatch_src/ip_input sequence and I have
crashdump.
kgdb session follows:
Script started on Sat Dec 15 06:47:49 2018
Command: kgdb kernel.debug /home/nanobsd/pppoe/crash/vmcore.0
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...
Unread portion of the kernel message buffer:
stack pointer = 0x28:0xfffffe01244bb920
frame pointer = 0x28:0xfffffe01244bb9a0
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 0 (dummynet)
trap number = 12
panic: page fault
cpuid = 0
KDB: stack backtrace:
db_trace_self_wrapper() at 0xffffffff802fc89b =
db_trace_self_wrapper+0x2b/frame 0xfffffe01244bb5d0
vpanic() at 0xffffffff804f0ac7 = vpanic+0x177/frame 0xfffffe01244bb630
panic() at 0xffffffff804f0943 = panic+0x43/frame 0xfffffe01244bb690
trap_fatal() at 0xffffffff8076f2af = trap_fatal+0x35f/frame 0xfffffe01244bb6e0
trap_pfault() at 0xffffffff8076f309 = trap_pfault+0x49/frame 0xfffffe01244bb740
trap() at 0xffffffff8076eae4 = trap+0x2d4/frame 0xfffffe01244bb850
calltrap() at 0xffffffff8074ff3c = calltrap+0x8/frame 0xfffffe01244bb850
--- trap 0xc, rip = 0xffffffff804ec893, rsp = 0xfffffe01244bb920, rbp =
0xfffffe01244bb9a0 ---
__rw_rlock_hard() at 0xffffffff804ec893 = __rw_rlock_hard+0xf3/frame
0xfffffe01244bb9a0
ip_input() at 0xffffffff806444ca = ip_input+0x53a/frame 0xfffffe01244bba30
netisr_dispatch_src() at 0xffffffff8060ebe8 = netisr_dispatch_src+0xa8/frame
0xfffffe01244bba80
dummynet_send() at 0xffffffff806723dd = dummynet_send+0x10d/frame
0xfffffe01244bbab0
dummynet_task() at 0xffffffff80671e1c = dummynet_task+0x2ec/frame
0xfffffe01244bbb20
taskqueue_run_locked() at 0xffffffff80548a54 = taskqueue_run_locked+0x154/frame
0xfffffe01244bbb80
taskqueue_thread_loop() at 0xffffffff80549bb8 =
taskqueue_thread_loop+0x98/frame 0xfffffe01244bbbb0
fork_exit() at 0xffffffff804ba803 = fork_exit+0x83/frame 0xfffffe01244bbbf0
fork_trampoline() at 0xffffffff80750eee = fork_trampoline+0xe/frame
0xfffffe01244bbbf0
--- trap 0, rip = 0, rsp = 0, rbp = 0 ---
Uptime: 57d17h28m40s
Dumping 467 out of 4073 MB:..4%..11%..21%..31%..42%..52%..62%..72%..83%..93%
Reading symbols from /boot/modules/tmpfs.ko...done.
Loaded symbols for /boot/modules/tmpfs.ko
#0 doadump (textdump=1) at pcpu.h:230
230 __asm("movq %%gs:%1,%0" : "=r" (td)
(kgdb) bt
#0 doadump (textdump=1) at pcpu.h:230
#1 0xffffffff804f06c0 in kern_reboot (howto=260) at
/home/src/sys/kern/kern_shutdown.c:383
#2 0xffffffff804f0b01 in vpanic (fmt=<value optimized out>, ap=<value
optimized out>)
at /home/src/sys/kern/kern_shutdown.c:776
#3 0xffffffff804f0943 in panic (fmt=<value optimized out>)
at /home/src/sys/kern/kern_shutdown.c:707
#4 0xffffffff8076f2af in trap_fatal (frame=0xfffffe01244bb860,
eva=274877908504)
at /home/src/sys/amd64/amd64/trap.c:877
#5 0xffffffff8076f309 in trap_pfault (frame=0xfffffe01244bb860, usermode=0) at
pcpu.h:230
#6 0xffffffff8076eae4 in trap (frame=0xfffffe01244bb860) at
/home/src/sys/amd64/amd64/trap.c:415
#7 0xffffffff8074ff3c in calltrap () at
/home/src/sys/amd64/amd64/exception.S:231
#8 0xffffffff804ec893 in __rw_rlock_hard (rw=0xfffff80092e78190,
td=0xfffff80001d02620,
v=<value optimized out>) at /home/src/sys/kern/kern_rwlock.c:493
#9 0xffffffff806444ca in ip_input (m=<value optimized out>)
at /home/src/sys/netinet/ip_input.c:795
#10 0xffffffff8060ebe8 in netisr_dispatch_src (proto=1, source=<value optimized
out>,
m=<value optimized out>) at /home/src/sys/net/netisr.c:1120
#11 0xffffffff806723dd in dummynet_send (m=0x0) at
/home/src/sys/netpfil/ipfw/ip_dn_io.c:774
#12 0xffffffff80671e1c in dummynet_task (context=<value optimized out>,
pending=<value optimized out>) at /home/src/sys/netpfil/ipfw/ip_dn_io.c:729
#13 0xffffffff80548a54 in taskqueue_run_locked (queue=0xfffff80006085e00)
at /home/src/sys/kern/subr_taskqueue.c:463
#14 0xffffffff80549bb8 in taskqueue_thread_loop (arg=<value optimized out>)
at /home/src/sys/kern/subr_taskqueue.c:755
#15 0xffffffff804ba803 in fork_exit (callout=0xffffffff80549b20
<taskqueue_thread_loop>,
arg=0xffffffff80c82c38, frame=0xfffffe01244bbc00) at
/home/src/sys/kern/kern_fork.c:1072
#16 0xffffffff80750eee in fork_trampoline () at
/home/src/sys/amd64/amd64/exception.S:972
---Type <return> to continue, or q <return> to quit---
#17 0x0000000000000000 in ?? ()
Current language: auto; currently minimal
(kgdb) frame 9
#9 0xffffffff806444ca in ip_input (m=<value optimized out>)
at /home/src/sys/netinet/ip_input.c:795
795 IF_ADDR_RLOCK(ifp);
(kgdb) l
790 * interface. Reception of forwarded directed broadcasts would
791 * be handled via ip_forward() and ether_output() with the
loopback
792 * into the stack for SIMPLEX interfaces handled by
ether_output().
793 */
794 if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) {
795 IF_ADDR_RLOCK(ifp);
796 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
797 if (ifa->ifa_addr->sa_family != AF_INET)
798 continue;
799 ia = ifatoia(ifa);
(kgdb) p *ifp
$1 = {if_link = {tqe_next = 0x4000000004, tqe_prev = 0x4000000006}, if_clones =
{
le_next = 0x4000000007, le_prev = 0x4000000009}, if_groups = {tqh_first =
0x400000000a,
tqh_last = 0x4000000011}, if_alloctype = 250 'З', if_softc = 0x4000000104,
if_llsoftc = 0x40000004d0, if_l2com = 0x40000004d4,
if_dname = 0x4000000184 <Address 0x4000000184 out of bounds>, if_dunit = 218,
if_index = 64,
if_index_reserved = 0, if_xname = 0xfffff80092e78060 "\220\001",
if_description = 0x400000035e <Address 0x400000035e out of bounds>, if_flags
= 1050,
if_drv_flags = 64, if_capabilities = 454, if_capenable = 64, if_linkmib =
0x4000000386,
if_linkmiblen = 274877907462, if_refcount = 682, if_type = 64 '@', if_addrlen
= 0 '\0',
if_hdrlen = 0 '\0', if_link_state = 0 '\0', if_mtu = 522, if_metric = 64,
if_baudrate = 274877907476, if_hwassist = 274877907488, if_epoch =
274877907500,
if_lastchange = {tv_sec = 274877908294, tv_usec = 274877907730}, if_snd = {
ifq_head = 0x40000002e0, ifq_tail = 0x4000000334, ifq_len = 824, ifq_maxlen
= 64, ifq_mtx = {
lock_object = {lo_name = 0x40000003c6 <Address 0x40000003c6 out of
bounds>,
lo_flags = 1298, lo_data = 64, lo_witness = 0x4000000332}, mtx_lock =
274877907950},
ifq_drv_head = 0x40000002ae, ifq_drv_tail = 0x40000000fc, ifq_drv_len =
858,
ifq_drv_maxlen = 64, altq_type = 870, altq_flags = 64, altq_disc =
0x400000036a,
altq_ifp = 0x4000000124, altq_enqueue = 0x4000000318, altq_dequeue =
0x400000030a,
altq_request = 0x400000036c, altq_clfier = 0x4000000188, altq_classify =
0x400000058d,
altq_tbr = 0x400000058f, altq_cdnr = 0x4000000376}, if_linktask = {ta_link
= {
stqe_next = 0x4000000262}, ta_pending = 460, ta_priority = 0, ta_func =
0x4000000264,
ta_context = 0x40000001b6}, if_addr_lock = {lock_object = {
lo_name = 0x40000001b8 <Address 0x40000001b8 out of bounds>, lo_flags =
1072, lo_data = 64,
lo_witness = 0x400000026a}, rw_lock = 274877907356}, if_addrhead = {
tqh_first = 0x4000000382, tqh_last = 0x4000000196}, if_multiaddrs = {
tqh_first = 0x4000000120, tqh_last = 0x4000000218}, if_amcount = 294,
if_addr = 0x40000001be,
if_broadcastaddr = 0x4000000064 <Address 0x4000000064 out of bounds>,
if_afdata_lock = {
---Type <return> to continue, or q <return> to quit---
lock_object = {lo_name = 0x4000000192 <Address 0x4000000192 out of bounds>,
lo_flags = 810,
lo_data = 64, lo_witness = 0x40000002de}, rw_lock = 274877907684},
if_afdata = 0xfffff80092e78208, if_afdata_initialized = 441, if_fib = 64,
if_vnet = 0x40000000db, if_home_vnet = 0x4000000411, if_vlantrunk =
0x40000001bf,
if_bpf = 0x40000001c1, if_pcount = 1051, if_bridge = 0x40000001c7, if_lagg =
0x40000003ef,
if_pf_kif = 0x4000000207, if_carp = 0x400000020b, if_label = 0x40000002ab,
if_netmap = 0x4000000215, if_output = 0x4000000219, if_input = 0x40000002af,
if_start = 0x4000000221, if_ioctl = 0x400000022d, if_init = 0x40000002e1,
if_resolvemulti = 0x40000002e5, if_qflush = 0x4000000305, if_transmit =
0x4000000263,
if_reassign = 0x4000000265, if_get_counter = 0x400000030b, if_requestencap =
0x400000026b,
if_counters = 0xfffff80092e78410, if_hw_tsomax = 999, if_hw_tsomaxsegcount =
64,
if_hw_tsomaxsegsize = 735, if_pspare = 0xfffff80092e78480, if_hw_addr =
0x4000000039,
if_pcp = 101 'e', if_bspare = 0xfffff80092e784a1 "", if_ispare =
0xfffff80092e784a4}
(kgdb) frame 11
#11 0xffffffff806723dd in dummynet_send (m=0x0) at
/home/src/sys/netpfil/ipfw/ip_dn_io.c:774
774 netisr_dispatch(NETISR_IP, m);
(kgdb) p m
$2 = (struct mbuf *) 0x0
(kgdb) l
769 case DIR_OUT:
770 ip_output(m, NULL, NULL, IP_FORWARDING, NULL,
NULL);
771 break ;
772
773 case DIR_IN :
774 netisr_dispatch(NETISR_IP, m);
775 break;
776
777 #ifdef INET6
778 case DIR_IN | PROTO_IPV6:
(kgdb) quit
--
You are receiving this mail because:
You are the assignee for the bug.
More information about the freebsd-net
mailing list