[Bug 271991] Crash on some network packets with fresh stable
- In reply to: bugzilla-noreply_a_freebsd.org: "[Bug 271991] Crash on some network packets with fresh stable"
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 19 Jun 2023 04:25:23 UTC
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271991 --- Comment #9 from Ivan Rozhuk <rozhuk.im@gmail.com> --- This is full list from tcpdump: [155] Fatal trap 12: page fault while in kernel mode [155] cpuid = 1; apic id = 01 [155] fault virtual address = 0x2dd [155] fault code = supervisor read data, page not present [155] instruction pointer = 0x20:0xffffffff807246d3 [155] stack pointer = 0x28:0xfffffe015c814250 [155] frame pointer = 0x28:0xfffffe015c8142c0 [155] code segment = base 0x0, limit 0xfffff, type 0x1b [155] = DPL 0, pres 1, long 1, def32 0, gran 1 [155] processor eflags = interrupt enabled, resume, IOPL = 0 [155] current process = 54569 (nginx) [155] trap number = 12 [155] panic: page fault [155] cpuid = 1 [155] time = 1687145826 [155] KDB: stack backtrace: [155] #0 0xffffffff80665e1b at kdb_backtrace+0x6b [155] #1 0xffffffff8061bff2 at vpanic+0x152 [155] #2 0xffffffff8061be93 at panic+0x43 [155] #3 0xffffffff8093b2a7 at trap_fatal+0x387 [155] #4 0xffffffff8093b2ff at trap_pfault+0x4f [155] #5 0xffffffff809121ce at calltrap+0x8 [155] #6 0xffffffff8071fdba at bpf_mtap+0x10a [155] #7 0xffffffff807511d4 at iflib_txq_drain+0x3c4 [155] #8 0xffffffff80756303 at drain_ring_lockless+0x63 [155] #9 0xffffffff807561ea at ifmp_ring_enqueue+0x29a [155] #10 0xffffffff80754409 at iflib_if_transmit+0x239 [155] #11 0xffffffff80737b0b at ether_output_frame+0x9b [155] #12 0xffffffff8073797d at ether_output+0x6ed [155] #13 0xffffffff80785106 at ip_output_send+0xe6 [155] #14 0xffffffff80784e33 at ip_output+0xff3 [155] #15 0xffffffff811ac339 at rack_output+0x3ee9 [155] #16 0xffffffff807aeb3f at tcp_usr_send+0x2af [155] #17 0xffffffff80619902 at vn_sendfile+0x1222 [155] Uptime: 2m35s [155] Dumping 660 out of 8171 MB:..3%..13%..22%..32%..42%..51%..61%..71%..83%..93% __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 55 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu, (kgdb) #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 #1 doadump (textdump=<optimized out>) at ../../../sys/kern/kern_shutdown.c:396 #2 0xffffffff8061bbe8 in kern_reboot (howto=260) at ../../../sys/kern/kern_shutdown.c:484 #3 0xffffffff8061c05f in vpanic (fmt=<optimized out>, ap=ap@entry=0xfffffe015c8140a0) at ../../../sys/kern/kern_shutdown.c:923 #4 0xffffffff8061be93 in panic (fmt=<unavailable>) at ../../../sys/kern/kern_shutdown.c:847 #5 0xffffffff8093b2a7 in trap_fatal (frame=0xfffffe015c814190, eva=733) at ../../../sys/amd64/amd64/trap.c:942 #6 0xffffffff8093b2ff in trap_pfault (frame=0xfffffe015c814190, usermode=false, signo=<optimized out>, ucode=<optimized out>) at ../../../sys/amd64/amd64/trap.c:761 #7 <signal handler called> #8 0xffffffff807246d3 in bpf_filter (pc=0xfffff80070d49448, p=p@entry=0xfffff800159d5500 "", wirelen=wirelen@entry=1514, buflen=buflen@entry=0) at ../../../sys/net/bpf_filter.c:261 #9 0xffffffff8071fdba in bpf_mtap (bp=0xfffff800049c3a00, m=0xfffff800159d5500) at ../../../sys/net/bpf.c:2348 #10 0xffffffff807511d4 in iflib_txq_drain (r=0xfffffe01305b3000, cidx=123, pidx=<optimized out>) at ../../../sys/net/iflib.c:3989 #11 0xffffffff80756303 in drain_ring_lockless (r=<optimized out>, os=..., prev=0, budget=32) at ../../../sys/net/mp_ring.c:187 #12 0xffffffff807561ea in ifmp_ring_enqueue (r=0xfffffe01305b3000, items=<optimized out>, items@entry=0xfffffe015c814538, n=n@entry=1, budget=budget@entry=32, abdicate=abdicate@entry=0) at ../../../sys/net/mp_ring.c:470 #13 0xffffffff80754409 in iflib_if_transmit (ifp=<optimized out>, m=0xfffff800159d5500) at ../../../sys/net/iflib.c:4327 #14 0xffffffff80737b0b in ether_output_frame (ifp=0xfffff80004ec2000, m=0x50, m@entry=0xfffff800159d5500) at ../../../sys/net/if_ethersubr.c:514 #15 0xffffffff8073797d in ether_output (ifp=<optimized out>, m=0xfffff800159d5500, dst=<optimized out>, ro=<optimized out>) at ../../../sys/net/if_ethersubr.c:441 #16 0xffffffff80785106 in ip_output_send (inp=inp@entry=0xfffff801242bad90, ifp=0x14, m=0x50, gw=0xfffffe38, gw@entry=0xfffff801242baf40, ro=0x0, ro@entry=0xfffff801242baf20, stamp_tag=<optimized out>) at ../../../sys/netinet/ip_output.c:277 #17 0xffffffff80784e33 in ip_output (m=<optimized out>, m@entry=0xfffff800159d5500, opt=opt@entry=0x0, ro=<optimized out>, ro@entry=0xfffff801242baf20, flags=0, imo=imo@entry=0x0, inp=0xfffff801242bad90) at ../../../sys/netinet/ip_output.c:799 #18 0xffffffff811ac339 in rack_output (tp=0xfffffe015c6e60e0) at ../../../../../../../../../../sys/netinet/tcp_stacks/rack.c:18270 #19 0xffffffff807aeb3f in tcp_usr_send (so=0xfffff800156ddb10, flags=0, m=<optimized out>, nam=0x0, control=<optimized out>, td=0xfffffe015c5c0e40) at ../../../sys/netinet/tcp_usrreq.c:1178 #20 0xffffffff80619902 in vn_sendfile (fp=<optimized out>, sockfd=9, hdr_uio=0x0, trl_uio=0x0, offset=<optimized out>, nbytes=4194304, sent=0xfffffe015c814dd0, flags=1, td=0xfffffe015c5c0e40) at ../../../sys/kern/kern_sendfile.c:1188 #21 0xffffffff8061a766 in fo_sendfile (fp=0x14, sockfd=80, hdr_uio=0xfffffe38, trl_uio=0x0, offset=0, nbytes=18446741880533238416, sent=0xfffffe015c814dd0, flags=733, td=0xfffffe015c5c0e40) at ../../../sys/sys/file.h:416 #22 sendfile (uap=0xfffffe015c5c1228, td=<optimized out>, compat=<optimized out>) at ../../../sys/kern/kern_sendfile.c:1326 #23 sys_sendfile (td=0xfffffe015c5c0e40, uap=0xfffffe015c5c1228) at ../../../sys/kern/kern_sendfile.c:1354 #24 0xffffffff8093bb50 in syscallenter (td=0xfffffe015c5c0e40) at ../../../sys/amd64/amd64/../../kern/subr_syscall.c:190 #25 amd64_syscall (td=0xfffffe015c5c0e40, traced=0) at ../../../sys/amd64/amd64/trap.c:1183 #26 <signal handler called> #27 0x000000080081e2ba in ?? () Backtrace stopped: Cannot access memory at address 0x7fffffffdaf8 (kgdb) This is full list from: ng_bpf_enable="YES" ng_bpf_profiles="lan0" ng_bpf_lan0_in="ip and tcp and tcp port 80 and tcp[1024] != 0" ng_bpf_lan0_out="ip and tcp and tcp port 80 and tcp[1024] != 0" My original config was: ng_bpf_enable="YES" ng_bpf_profiles="vlan886" ng_bpf_vlan886_in="ip and tcp and ip[6] & 64 = 0 and ip[8]>128 and tcp[tcpflags] == tcp-rst" it filters only incoming from internet packets from intel i211 adapter. [235] Fatal trap 12: page fault while in kernel mode [235] cpuid = 0; apic id = 00 [235] fault virtual address = 0x2dd [235] fault code = supervisor read data, page not present [235] instruction pointer = 0x20:0xffffffff818f7e93 [235] stack pointer = 0x28:0xfffffe015b3e13e0 [235] frame pointer = 0x28:0xfffffe015b3e1450 [235] code segment = base 0x0, limit 0xfffff, type 0x1b [235] = DPL 0, pres 1, long 1, def32 0, gran 1 [235] processor eflags = interrupt enabled, resume, IOPL = 0 [235] current process = 66911 (nginx) [235] trap number = 12 [235] panic: page fault [235] cpuid = 0 [235] time = 1687147722 [235] KDB: stack backtrace: [235] #0 0xffffffff80665e1b at kdb_backtrace+0x6b [235] #1 0xffffffff8061bff2 at vpanic+0x152 [235] #2 0xffffffff8061be93 at panic+0x43 [235] #3 0xffffffff8093b2a7 at trap_fatal+0x387 [235] #4 0xffffffff8093b2ff at trap_pfault+0x4f [235] #5 0xffffffff809121ce at calltrap+0x8 [235] #6 0xffffffff818f756a at ng_bpf_rcvdata+0xda [235] #7 0xffffffff818ee777 at ng_apply_item+0x207 [235] #8 0xffffffff818ee25c at ng_snd_item+0x1cc [235] #9 0xffffffff818e8bdd at ng_ether_output+0x5d [235] #10 0xffffffff80737957 at ether_output+0x6c7 [235] #11 0xffffffff80785106 at ip_output_send+0xe6 [235] #12 0xffffffff80784e33 at ip_output+0xff3 [235] #13 0xffffffff811af339 at rack_output+0x3ee9 [235] #14 0xffffffff807aeb3f at tcp_usr_send+0x2af [235] #15 0xffffffff80619902 at vn_sendfile+0x1222 [235] #16 0xffffffff8061a766 at sys_sendfile+0xe6 [235] #17 0xffffffff8093bb50 at amd64_syscall+0xd0 [235] Uptime: 3m55s [235] Dumping 646 out of 8171 MB:..3%..13%..23%..33%..43%..52%..62%..72%..82%..92% __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 55 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu, (kgdb) #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 #1 doadump (textdump=<optimized out>) at ../../../sys/kern/kern_shutdown.c:396 #2 0xffffffff8061bbe8 in kern_reboot (howto=260) at ../../../sys/kern/kern_shutdown.c:484 #3 0xffffffff8061c05f in vpanic (fmt=<optimized out>, ap=ap@entry=0xfffffe015b3e1230) at ../../../sys/kern/kern_shutdown.c:923 #4 0xffffffff8061be93 in panic (fmt=<unavailable>) at ../../../sys/kern/kern_shutdown.c:847 #5 0xffffffff8093b2a7 in trap_fatal (frame=0xfffffe015b3e1320, eva=733) at ../../../sys/amd64/amd64/trap.c:942 #6 0xffffffff8093b2ff in trap_pfault (frame=0xfffffe015b3e1320, usermode=false, signo=<optimized out>, ucode=<optimized out>) at ../../../sys/amd64/amd64/trap.c:761 #7 <signal handler called> #8 0xffffffff818f7e93 in bpf_filter (pc=0xfffff80004e3c1bc, p=0xfffff800a9815500 "", wirelen=1514, buflen=0) at ../../../../../../../../../../sys/net/bpf_filter.c:261 #9 0xffffffff818f756a in ng_bpf_rcvdata (hook=<optimized out>, item=0xfffff800a9239980) at ../../../../../../../../../../sys/netgraph/ng_bpf.c:475 #10 0xffffffff818ee777 in ng_apply_item (node=node@entry=0xfffff80004e3c200, item=item@entry=0xfffff800a9239980, rw=-456) at ../../../../../../../../../../sys/netgraph/ng_base.c:2406 #11 0xffffffff818ee25c in ng_snd_item (item=item@entry=0xfffff800a9239980, flags=flags@entry=0) at ../../../../../../../../../../sys/netgraph/ng_base.c:2323 #12 0xffffffff818e8bdd in ng_ether_output (ifp=<optimized out>, mp=0xfffffe015b3e15c8) at ../../../../../../../../../../sys/netgraph/ng_ether.c:294 #13 0xffffffff80737957 in ether_output (ifp=<optimized out>, m=0xfffff800a9815500, dst=<optimized out>, ro=<optimized out>) at ../../../sys/net/if_ethersubr.c:431 #14 0xffffffff80785106 in ip_output_send (inp=inp@entry=0xfffff800a96ffba0, ifp=0x14, m=0x50, gw=0xfffffe38, gw@entry=0xfffff800a96ffd50, ro=0x0, ro@entry=0xfffff800a96ffd30, stamp_tag=<optimized out>) at ../../../sys/netinet/ip_output.c:277 #15 0xffffffff80784e33 in ip_output (m=<optimized out>, m@entry=0xfffff800a9815500, opt=opt@entry=0x0, ro=<optimized out>, ro@entry=0xfffff800a96ffd30, flags=0, imo=imo@entry=0x0, inp=0xfffff800a96ffba0) at ../../../sys/netinet/ip_output.c:799 #16 0xffffffff811af339 in rack_output (tp=0xfffffe015c6fd0e0) at ../../../../../../../../../../sys/netinet/tcp_stacks/rack.c:18270 #17 0xffffffff807aeb3f in tcp_usr_send (so=0xfffff80004ef2760, flags=0, m=<optimized out>, nam=0x0, control=<optimized out>, td=0xfffffe00dec7bac0) at ../../../sys/netinet/tcp_usrreq.c:1178 #18 0xffffffff80619902 in vn_sendfile (fp=<optimized out>, sockfd=11, hdr_uio=0x0, trl_uio=0x0, offset=<optimized out>, nbytes=4194304, sent=0xfffffe015b3e1dd0, flags=1, td=0xfffffe00dec7bac0) at ../../../sys/kern/kern_sendfile.c:1188 #19 0xffffffff8061a766 in fo_sendfile (fp=0x14, sockfd=80, hdr_uio=0xfffffe38, trl_uio=0x0, offset=0, nbytes=18446741880512058400, sent=0xfffffe015b3e1dd0, flags=733, td=0xfffffe00dec7bac0) at ../../../sys/sys/file.h:416 #20 sendfile (uap=0xfffffe00dec7bea8, td=<optimized out>, compat=<optimized out>) at ../../../sys/kern/kern_sendfile.c:1326 #21 sys_sendfile (td=0xfffffe00dec7bac0, uap=0xfffffe00dec7bea8) at ../../../sys/kern/kern_sendfile.c:1354 #22 0xffffffff8093bb50 in syscallenter (td=0xfffffe00dec7bac0) at ../../../sys/amd64/amd64/../../kern/subr_syscall.c:190 #23 amd64_syscall (td=0xfffffe00dec7bac0, traced=0) at ../../../sys/amd64/amd64/trap.c:1183 #24 <signal handler called> #25 0x000000080081e2ba in ?? () Backtrace stopped: Cannot access memory at address 0x7fffffffdaf8 My patch did not fix issue (even in my netgraph case) :( But stack trace is different now: [71] Fatal trap 12: page fault while in kernel mode [71] cpuid = 1; apic id = 01 [71] fault virtual address = 0x2dd [71] fault code = supervisor read data, page not present [71] instruction pointer = 0x20:0xffffffff818f7e93 [71] stack pointer = 0x28:0xfffffe00c4f82490 [71] frame pointer = 0x28:0xfffffe00c4f82500 [71] code segment = base 0x0, limit 0xfffff, type 0x1b [71] = DPL 0, pres 1, long 1, def32 0, gran 1 [71] processor eflags = interrupt enabled, resume, IOPL = 0 [71] current process = 11 (irq40: ahci0:ch0) [71] trap number = 12 [71] panic: page fault [71] cpuid = 1 [71] time = 1687148175 [71] KDB: stack backtrace: [71] #0 0xffffffff80665e1b at kdb_backtrace+0x6b [71] #1 0xffffffff8061bff2 at vpanic+0x152 [71] #2 0xffffffff8061be93 at panic+0x43 [71] #3 0xffffffff8093b377 at trap_fatal+0x387 [71] #4 0xffffffff8093b3cf at trap_pfault+0x4f [71] #5 0xffffffff8091229e at calltrap+0x8 [71] #6 0xffffffff818f756a at ng_bpf_rcvdata+0xda [71] #7 0xffffffff818ee777 at ng_apply_item+0x207 [71] #8 0xffffffff818ee25c at ng_snd_item+0x1cc [71] #9 0xffffffff818e8bdd at ng_ether_output+0x5d [71] #10 0xffffffff80737a27 at ether_output+0x6c7 [71] #11 0xffffffff807851d6 at ip_output_send+0xe6 [71] #12 0xffffffff80784f03 at ip_output+0xff3 [71] #13 0xffffffff811af339 at rack_output+0x3ee9 [71] #14 0xffffffff807af255 at tcp_usr_ready+0xf5 [71] #15 0xffffffff8061a3f7 at sendfile_iodone+0x107 [71] #16 0xffffffff808c0be3 at vnode_pager_generic_getpages_done_async+0x43 [71] #17 0xffffffff806cd448 at bufdone+0x48 [71] Uptime: 1m11s [71] Dumping 642 out of 8171 MB:..3%..13%..23%..33%..43%..53%..63%..73%..83%..93% __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 55 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu, (kgdb) #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55 #1 doadump (textdump=<optimized out>) at ../../../sys/kern/kern_shutdown.c:396 #2 0xffffffff8061bbe8 in kern_reboot (howto=260) at ../../../sys/kern/kern_shutdown.c:484 #3 0xffffffff8061c05f in vpanic (fmt=<optimized out>, ap=ap@entry=0xfffffe00c4f822e0) at ../../../sys/kern/kern_shutdown.c:923 #4 0xffffffff8061be93 in panic (fmt=<unavailable>) at ../../../sys/kern/kern_shutdown.c:847 #5 0xffffffff8093b377 in trap_fatal (frame=0xfffffe00c4f823d0, eva=733) at ../../../sys/amd64/amd64/trap.c:942 #6 0xffffffff8093b3cf in trap_pfault (frame=0xfffffe00c4f823d0, usermode=false, signo=<optimized out>, ucode=<optimized out>) at ../../../sys/amd64/amd64/trap.c:761 #7 <signal handler called> #8 0xffffffff818f7e93 in bpf_filter (pc=0xfffff80001a399bc, p=0xfffff800147cc700 "", wirelen=1514, buflen=0) at ../../../../../../../../../../sys/net/bpf_filter.c:261 #9 0xffffffff818f756a in ng_bpf_rcvdata (hook=<optimized out>, item=0xfffff80014de8c80) at ../../../../../../../../../../sys/netgraph/ng_bpf.c:475 #10 0xffffffff818ee777 in ng_apply_item (node=node@entry=0xfffff800019e9800, item=item@entry=0xfffff80014de8c80, rw=-456) at ../../../../../../../../../../sys/netgraph/ng_base.c:2406 #11 0xffffffff818ee25c in ng_snd_item (item=item@entry=0xfffff80014de8c80, flags=flags@entry=0) at ../../../../../../../../../../sys/netgraph/ng_base.c:2323 #12 0xffffffff818e8bdd in ng_ether_output (ifp=<optimized out>, mp=0xfffffe00c4f82678) at ../../../../../../../../../../sys/netgraph/ng_ether.c:294 #13 0xffffffff80737a27 in ether_output (ifp=<optimized out>, m=0xfffff800147cc700, dst=<optimized out>, ro=<optimized out>) at ../../../sys/net/if_ethersubr.c:431 #14 0xffffffff807851d6 in ip_output_send (inp=inp@entry=0xfffff80084ad7ba0, ifp=0x14, m=0x50, gw=0xfffffe38, gw@entry=0xfffff80084ad7d50, ro=0x0, ro@entry=0xfffff80084ad7d30, stamp_tag=<optimized out>) at ../../../sys/netinet/ip_output.c:277 #15 0xffffffff80784f03 in ip_output (m=<optimized out>, m@entry=0xfffff800147cc700, opt=opt@entry=0x0, ro=<optimized out>, ro@entry=0xfffff80084ad7d30, flags=0, imo=imo@entry=0x0, inp=0xfffff80084ad7ba0) at ../../../sys/netinet/ip_output.c:799 #16 0xffffffff811af339 in rack_output (tp=0xfffffe015c738950) at ../../../../../../../../../../sys/netinet/tcp_stacks/rack.c:18270 #17 0xffffffff807af255 in tcp_usr_ready (so=<optimized out>, m=0xfffff800147d2700, count=512) at ../../../sys/netinet/tcp_usrreq.c:1302 #18 0xffffffff8061a3f7 in sendfile_iodone (arg=0xfffffe015c741000, pa=0xfffffe013057e638, count=<optimized out>, error=<optimized out>) at ../../../sys/kern/kern_sendfile.c:399 #19 0xffffffff808c0be3 in vnode_pager_generic_getpages_done_async ( bp=0xfffffe013057e500) at ../../../sys/vm/vnode_pager.c:1121 #20 0xffffffff806cd448 in bufdone (bp=0xfffffe013057e500) at ../../../sys/kern/vfs_bio.c:4547 #21 0xffffffff8057b18e in g_io_deliver (bp=0xfffff800019a9178, error=0) at ../../../sys/geom/geom_io.c:687 #22 0xffffffff8057b18e in g_io_deliver (bp=0xfffff80004dd82f0, error=0) at ../../../sys/geom/geom_io.c:687 #23 0xffffffff8057b18e in g_io_deliver (bp=bp@entry=0xfffff80004dda000, error=0) at ../../../sys/geom/geom_io.c:687 #24 0xffffffff80578419 in g_disk_done (bp=0xfffff80004e41000) at ../../../sys/geom/geom_disk.c:259 #25 0xffffffff802d4c00 in xpt_done_process ( ccb_h=ccb_h@entry=0xfffff80084166000) at ../../../sys/cam/cam_xpt.c:5435 #26 0xffffffff802d4827 in xpt_done_direct (done_ccb=0xfffff80084166000) at ../../../sys/cam/cam_xpt.c:4616 #27 0xffffffff80416545 in ahci_ch_intr_direct (arg=0xfffffe00c51c1000) at ../../../sys/dev/ahci/ahci.c:1355 #28 0xffffffff80414db3 in ahci_intr_one (data=<optimized out>) at ../../../sys/dev/ahci/ahci.c:556 #29 0xffffffff805e6e31 in intr_event_execute_handlers (ie=0xfffff800018d3c00, p=<optimized out>) at ../../../sys/kern/kern_intr.c:1169 #30 ithread_execute_handlers (ie=0xfffff800018d3c00, p=<optimized out>) at ../../../sys/kern/kern_intr.c:1182 #31 ithread_loop (arg=0xfffff80001c69000) at ../../../sys/kern/kern_intr.c:1270 #32 0xffffffff805e3b96 in fork_exit ( callout=0xffffffff805e6bf0 <ithread_loop>, arg=0xfffff80001c69000, frame=0xfffffe00c4f82f40) at ../../../sys/kern/kern_fork.c:1094 #33 <signal handler called> #34 0xfffffe00c4d82fd0 in ?? () Backtrace stopped: Cannot access memory at address 0x0 -- You are receiving this mail because: You are the assignee for the bug.