Recent -STABLE panics during boot (100% reproducable)

[Dmitry Sivachenko]

Hello!

Using very recent -STABLE we encounter the following panic during boot:

<snip>
ipfw2 initialized, divert enabled, rule-based forwarding enabled, default to deny, logging limited to 100 packets/entry by default
Waiting 5 seconds for SCSI devices to settle
SMP: AP CPU #1 Launched!
panic: malloc(M_WAITOK) in interrupt context
mp_lock = 00000001; cpuid = 0; lapic.id = 00000000
boot() called on cpu#0

syncing disks... 


This is 100% reproducable on two different machines.
I am providing our kernel config file below.  If we remove two
INVARIANTS-related options, the panic goes away.


machine		i386
cpu		I586_CPU
cpu		I686_CPU
ident           GENERIC
maxusers        0

makeoptions	DEBUG=-g		#Build kernel with gdb(1) debug symbols
options         INVARIANTS
options         INVARIANT_SUPPORT

options 	INET			#InterNETworking
options 	FFS			#Berkeley Fast Filesystem
options 	FFS_ROOT		#FFS usable as root device [keep this!]
options 	SOFTUPDATES		#Enable FFS soft updates support

options         UFS_DIRHASH

options 	PROCFS			#Process filesystem
options 	COMPAT_43		#Compatible with BSD 4.3 [KEEP THIS!]
options         SCSI_DELAY=5000         #Delay (in ms) before probing SCSI
options 	UCONSOLE		#Allow users to grab the console
options 	USERCONFIG		#boot -c editor
options 	VISUAL_USERCONFIG	#visual boot -c editor

options         NMBCLUSTERS=12000

options 	SMP			# Symmetric MultiProcessor Kernel
options 	APIC_IO			# Symmetric (APIC) I/O

options 	P1003_1B
options 	_KPOSIX_PRIORITY_SCHEDULING
options 	_KPOSIX_VERSION=199309L

options         HZ=100

device		isa
device		pci

device		fdc0	at isa? port IO_FD1 irq 6 drq 2
device		fd0	at fdc0 drive 0

device          ahc0             # AHA2940 and onboard AIC7xxx devices
device          ahc1             # AHA2940 and onboard AIC7xxx devices

device		scbus		# SCSI bus (required)
device          da              # Direct Access (disks)
device        scbus0 at ahc0 bus 0         # Single bus device
device		cd		# CD

device pass

device		atkbdc0	at isa? port IO_KBD
device		atkbd0	at atkbdc? irq 1 flags 0x1
device		psm0	at atkbdc? irq 12

device		vga0	at isa?

device		sc0	at isa? flags 0x100

device		npx0	at nexus? port IO_NPX irq 13

device		sio0	at isa? port IO_COM1 flags 0x10 irq 4
device		sio1	at isa? port IO_COM2 irq 3

device		ppc0	at isa? irq 7
device		ppbus		# Parallel port bus (required)

device		miibus		# MII bus support
device		fxp		# Intel EtherExpress PRO/100B (82557, 82558)

pseudo-device	loop		# Network loopback
pseudo-device	ether		# Ethernet support
pseudo-device	pty		# Pseudo-ttys (telnet etc)
pseudo-device	vn

pseudo-device	bpf	2	#Berkeley packet filter

device		smbus		# Bus support, required for smb below.
device		intpm
device		ichsmb
device		smb

options		ICMP_BANDLIM
options         IPFIREWALL              #firewall
options         IPFIREWALL_VERBOSE      #enable logging to syslogd(8)
options         IPFIREWALL_FORWARD      #enable transparent proxy support
options         IPFIREWALL_VERBOSE_LIMIT=100    #limit verbosity
options         IPDIVERT                #divert sockets
options         IPFW2

options         MAXDSIZ="(1024*1024*1024)"
options         DFLDSIZ="(1024*1024*1024)"

options		INCLUDE_CONFIG_FILE


Full boot output:

Copyright (c) 1992-2003 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
	The Regents of the University of California. All rights reserved.
FreeBSD 4.9-PRERELEASE #0: Fri Sep  5 16:55:35 MSD 2003
    andy at west.demos.su:/mnt/obj/usr/src/sys/MX1
Timecounter "i8254"  frequency 1193182 Hz
CPU: Intel(R) Pentium(R) III CPU family      1266MHz (1266.07-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0x6b1  Stepping = 1
  Features=0x383fbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,MMX,FXSR,SSE>
real memory  = 1073676288 (1048512K bytes)
avail memory = 1042145280 (1017720K bytes)
Programming 16 pins in IOAPIC #0
IOAPIC #0 intpin 2 -> irq 0
Programming 16 pins in IOAPIC #1
FreeBSD/SMP: Multiprocessor motherboard: 2 CPUs
 cpu0 (BSP): apic id:  0, version: 0x00040011, at 0xfee00000
 cpu1 (AP):  apic id:  1, version: 0x00040011, at 0xfee00000
 io0 (APIC): apic id:  4, version: 0x000f0011, at 0xfec00000
 io1 (APIC): apic id:  5, version: 0x000f0011, at 0xfec01000
Preloaded elf kernel "kernel" at 0xc02db000.
Pentium Pro MTRR support enabled
Using $PIR table, 9 entries at 0xc00f5250
npx0: <math processor> on motherboard
npx0: INT 16 interface
pcib0: <ServerWorks NB6635 3.0LE host to PCI bridge> on motherboard
IOAPIC #1 intpin 4 -> irq 2
IOAPIC #1 intpin 6 -> irq 5
IOAPIC #1 intpin 15 -> irq 9
pci0: <PCI bus> on pcib0
pci0: <ATI Mach64-GR graphics accelerator> at 3.0 irq 2
fxp0: <Intel 82559 Pro/100 Ethernet> port 0xc000-0xc03f mem 0xfe300000-0xfe3fffff,0xfe6fd000-0xfe6fdfff irq 5 at device 4.0 on pci0
fxp0: Ethernet address 00:30:48:23:9e:df
inphy0: <i82555 10/100 media interface> on miibus0
inphy0:  10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
fxp1: <Intel 82559 Pro/100 Ethernet> port 0xc400-0xc43f mem 0xfe500000-0xfe5fffff,0xfe6fe000-0xfe6fefff irq 9 at device 6.0 on pci0
fxp1: Ethernet address 00:30:48:23:9e:e0
inphy1: <i82555 10/100 media interface> on miibus1
inphy1:  10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
isab0: <ServerWorks IB6566 PCI to ISA bridge> at device 15.0 on pci0
isa0: <ISA bus> on isab0
pci0: <Unknown PCI ATA controller> at 15.1
pci0: <OHCI USB controller> at 15.2 irq 0
pcib1: <ServerWorks NB6635 3.0LE host to PCI bridge> on motherboard
IOAPIC #1 intpin 8 -> irq 10
IOAPIC #1 intpin 12 -> irq 11
pci1: <PCI bus> on pcib1
fxp2: <Intel 82558 Pro/100 Ethernet> port 0xe400-0xe41f mem 0xfea00000-0xfeafffff,0xfbdff000-0xfbdfffff irq 10 at device 1.0 on pci1
fxp2: Ethernet address 00:50:8b:02:2c:80
inphy2: <i82555 10/100 media interface> on miibus2
inphy2:  10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
ahc0: <Adaptec aic7892 Ultra160 SCSI adapter> port 0xe800-0xe8ff mem 0xfebff000-0xfebfffff irq 11 at device 3.0 on pci1
aic7892: Ultra160 Wide Channel A, SCSI Id=7, 32/253 SCBs
orm0: <Option ROM> at iomem 0xc0000-0xc7fff on isa0
pmtimer0 on isa0
fdc0: <NEC 72065B or clone> at port 0x3f0-0x3f5,0x3f7 irq 6 drq 2 on isa0
fdc0: FIFO enabled, 8 bytes threshold
fd0: <1440-KB 3.5" drive> on fdc0 drive 0
atkbdc0: <Keyboard controller (i8042)> at port 0x60,0x64 on isa0
atkbd0: <AT Keyboard> flags 0x1 irq 1 on atkbdc0
psm0: <PS/2 Mouse> irq 12 on atkbdc0
psm0: model NetMouse/NetScroll Optical, device ID 0
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on isa0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=0x300>
sio0 at port 0x3f8-0x3ff irq 4 flags 0x10 on isa0
sio0: type 16550A
sio1 at port 0x2f8-0x2ff irq 3 on isa0
sio1: type 16550A
ppc0: <Parallel port> at port 0x378-0x37f irq 7 on isa0
ppc0: Generic chipset (ECP/PS2/NIBBLE) in COMPATIBLE mode
ppc0: FIFO with 16/16/8 bytes threshold
APIC_IO: Testing 8254 interrupt delivery
APIC_IO: Broken MP table detected: 8254 is not connected to IOAPIC #0 intpin 2
APIC_IO: routing 8254 via 8259 and IOAPIC #0 intpin 0
ipfw2 initialized, divert enabled, rule-based forwarding enabled, default to deny, logging limited to 100 packets/entry by default
Waiting 5 seconds for SCSI devices to settle
SMP: AP CPU #1 Launched!
panic: malloc(M_WAITOK) in interrupt context
mp_lock = 00000001; cpuid = 0; lapic.id = 00000000
boot() called on cpu#0

syncing disks... 
done
Uptime: 8s
Automatic reboot in 15 seconds - press a key on the console to abort
--> Press a key on the console to reboot,
--> or switch off the system now.
Rebooting...
cpu_reset called on cpu#0
cpu_reset: Stopping other CPUs