kernel panics involving NFS+RPCSEC_GSS

[Clinton Adams]

On Thu, Aug 18, 2011 at 3:25 PM, Clinton Adams <clinton.adams at gmail.com> wrote:
> Hello,
>
> Kernel panics if clients hit the nfs server sufficiently hard -
> happens repeatedly with 13 clients logging in at the same approximate
> time, using nfsv4 mounted homes.
>
> server is running freebsd 8.2-RELEASE-p2. clients are linux 2.6.38-10
>
> Running a memtest on the server now to rule out bad mem. The server
> has been used for samba, and it's only with the attempted switch to
> nfs that this problem has appeared.
>

Err, wrong paste from another forum. Here's the trace from my server:

    Fatal trap 12: page fault while in kernel mode
    Fatal trap 12: page fault while in kernel mode
    cpuid = 0;
    cpuid = 2; apic id = 00apic id = 06
    fault virtual address   = 0x0
    fault virtual address   = 0x8
    fault code              = supervisor write data, page not present
    fault code              = supervisor read data, page not present
    instruction pointer     = 0x20:0xffffffff807db856
    instruction pointer     = 0x20:0xffffffff807dc0d7
    stack pointer           = 0x28:0xffffff8096c0d840
    stack pointer           = 0x28:0xffffff8096c17860
    frame pointer           = 0x28:0xffffff8096c0d860
    frame pointer           = 0x28:0xffffff8096c17a80
    code segment            = base 0x0, limit 0xfffff, type 0x1b
    code segment            = base 0x0, limit 0xfffff, type 0x1b
                            = DPL 0, pres 1, long 1, def32 0, gran 1
                            = DPL 0, pres 1, long 1, def32 0, gran 1
    processor eflags        =
    processor eflags        = interrupt enabled, interrupt enabled,
resume, resume, IOPL = 0IOPL = 0
    current process         =
    current process         = 765 (nfsd: service)765 (nfsd: service)
    trap number             = 12
    trap number             = 12
    panic: page fault

    cpuid = 1
    Uptime: 3h22m48s
    Physical memory: 2032 MB
    Dumping 406 MB: 391 375 359 343 327 311 295 279 263 247 231 215
199 183 167 151 135 119 103 87 71 55 39 23 7

    Reading symbols from /boot/kernel/linux.ko...done.
    Loaded symbols for /boot/kernel/linux.ko
    Reading symbols from /boot/kernel/nfscommon.ko...done.
    Loaded symbols for /boot/kernel/nfscommon.ko
    Reading symbols from /boot/kernel/nfsd.ko...done.
    Loaded symbols for /boot/kernel/nfsd.ko
    Reading symbols from /boot/kernel/snp.ko...done.
    Loaded symbols for /boot/kernel/snp.ko
    #0  doadump () at pcpu.h:224
    224             __asm("movq %%gs:0,%0" : "=r" (td));
    (kgdb) list *0xffffffff807db856
    0xffffffff807db856 is in svc_rpc_gss_forget_client
(/usr/src/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c:622).
    617             struct svc_rpc_gss_client_list *list;
    618
    619             list =
&svc_rpc_gss_client_hash[client->cl_id.ci_id % CLIENT_HASH_SIZE];
    620             sx_xlock(&svc_rpc_gss_lock);
    621             TAILQ_REMOVE(list, client, cl_link);
    622             TAILQ_REMOVE(&svc_rpc_gss_clients, client, cl_alllink);
    623             svc_rpc_gss_client_count--;
    624             sx_xunlock(&svc_rpc_gss_lock);
    625             svc_rpc_gss_release_client(client);
    626     }
    (kgdb) backtrace
    #0  doadump () at pcpu.h:224
    #1  0xffffffff805cbabe in boot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:419
    #2  0xffffffff805cbed3 in panic (fmt=0x0) at
/usr/src/sys/kern/kern_shutdown.c:592
    #3  0xffffffff808d239d in trap_fatal (frame=0xffffff0004c89460,
eva=Variable "eva" is not available.
    ) at /usr/src/sys/amd64/amd64/trap.c:783
    #4  0xffffffff808d275f in trap_pfault (frame=0xffffff8096c0d790,
usermode=0) at /usr/src/sys/amd64/amd64/trap.c:699
    #5  0xffffffff808d2b5f in trap (frame=0xffffff8096c0d790) at
/usr/src/sys/amd64/amd64/trap.c:449
    #6  0xffffffff808bada4 in calltrap () at
/usr/src/sys/amd64/amd64/exception.S:224
    #7  0xffffffff807db856 in svc_rpc_gss_forget_client
(client=0xffffff001c015200) at atomic.h:158
    #8  0xffffffff807dc0e3 in svc_rpc_gss (rqst=0xffffff0004a24000,
msg=0xffffff8096c0db20) at
/usr/src/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c:642
    #9  0xffffffff807d48f3 in svc_run_internal
(pool=0xffffff0004ca6200, ismaster=0) at /usr/src/sys/rpc/svc.c:837
    #10 0xffffffff807d50ab in svc_thread_start (arg=Variable "arg" is
not available.
    ) at /usr/src/sys/rpc/svc.c:1200
    #11 0xffffffff805a26f8 in fork_exit (callout=0xffffffff807d50a0
<svc_thread_start>, arg=0xffffff0004ca6200, frame=0xffffff8096c0dc40)
at /usr/src/sys/kern/kern_fork.c:845
    #12 0xffffffff808bb26e in fork_trampoline () at
/usr/src/sys/amd64/amd64/exception.S:565
    #13 0x0000000000000080 in ?? ()
    #14 0x00007fffffffe6e0 in ?? ()
    #15 0x000000000000002e in ?? ()
    #16 0x0000000000000000 in ?? ()
    #17 0xfffffffffffffef4 in ?? ()
    #18 0x0000000000000000 in ?? ()
    #19 0x000000000000009b in ?? ()
    #20 0x00007fffffffe6e0 in ?? ()
    #21 0x0000000000000008 in ?? ()
    #22 0x0000000000000003 in ?? ()
    #23 0x00000000ffffffff in ?? ()
    #24 0x00007fffffffe700 in ?? ()
    #25 0x0000000000000000 in ?? ()
    #26 0x0000000000000001 in ?? ()
    #27 0x0000000000000000 in ?? ()
    #28 0x001b00130000000c in ?? ()
    #29 0x00007fffffffffe8 in ?? ()
    #30 0x003b003b00000001 in ?? ()
    #31 0x0000000000000002 in ?? ()
    #32 0x00000008006a045c in ?? ()
    #33 0x0000000000000043 in ?? ()
    #34 0x0000000000000202 in ?? ()
    #35 0x00007fffffffe6a8 in ?? ()
    #36 0x000000000000003b in ?? ()
    #37 0xffffffff80c96840 in affinity ()
    #38 0x0000000000000000 in ?? ()
    #39 0x0000000000000000 in ?? ()
    #40 0xffffff0004b2a000 in ?? ()
    #41 0xffffff8096c0cd90 in ?? ()
    #42 0xffffff8096c0cd38 in ?? ()
    #43 0xffffff000488f000 in ?? ()
    #44 0xffffffff805f1379 in sched_switch (td=0xffffffff807d50a0,
newtd=0xffffff0004ca6200, flags=Variable "flags" is not available.
    ) at /usr/src/sys/kern/sched_ule.c:1852
    Previous frame inner to this frame (corrupt stack?)