svn commit: r221914 - in projects/bhyve/sys: amd64/amd64 amd64/conf amd64/include amd64/vmm amd64/vmm/io conf dev/blackhole dev/bvm kern modules modules/blackhole modules/vmm x86/include x86/x86

John Baldwin jhb at FreeBSD.org
Sat May 14 20:35:02 UTC 2011


Author: jhb
Date: Sat May 14 20:35:01 2011
New Revision: 221914
URL: http://svn.freebsd.org/changeset/base/221914

Log:
  First cut at porting the kernel portions of 221828 and 221905 from the
  BHyVe reference branch to HEAD.

Added:
  projects/bhyve/sys/amd64/include/vmm.h
     - copied unchanged from r221828, projects/bhyve_ref/sys/amd64/include/vmm.h
  projects/bhyve/sys/amd64/include/vmm_dev.h
     - copied unchanged from r221828, projects/bhyve_ref/sys/amd64/include/vmm_dev.h
  projects/bhyve/sys/amd64/vmm/
     - copied from r221828, projects/bhyve_ref/sys/amd64/vmm/
  projects/bhyve/sys/dev/blackhole/
     - copied from r221905, projects/bhyve_ref/sys/dev/blackhole/
  projects/bhyve/sys/dev/bvm/
     - copied from r221905, projects/bhyve_ref/sys/dev/bvm/
  projects/bhyve/sys/modules/blackhole/
     - copied from r221905, projects/bhyve_ref/sys/modules/blackhole/
  projects/bhyve/sys/modules/vmm/
     - copied from r221828, projects/bhyve_ref/sys/modules/vmm/
Modified:
  projects/bhyve/sys/amd64/amd64/apic_vector.S
  projects/bhyve/sys/amd64/amd64/intr_machdep.c
  projects/bhyve/sys/amd64/amd64/minidump_machdep.c
  projects/bhyve/sys/amd64/amd64/mp_machdep.c
  projects/bhyve/sys/amd64/amd64/vm_machdep.c
  projects/bhyve/sys/amd64/conf/GENERIC
  projects/bhyve/sys/amd64/include/specialreg.h
  projects/bhyve/sys/amd64/vmm/io/ppt.c
  projects/bhyve/sys/amd64/vmm/io/vlapic.c
  projects/bhyve/sys/amd64/vmm/vmm.c
  projects/bhyve/sys/amd64/vmm/vmm_dev.c
  projects/bhyve/sys/amd64/vmm/vmm_ipi.c
  projects/bhyve/sys/amd64/vmm/vmm_msr.c
  projects/bhyve/sys/conf/files.amd64
  projects/bhyve/sys/conf/options.amd64
  projects/bhyve/sys/kern/subr_bus.c
  projects/bhyve/sys/modules/Makefile
  projects/bhyve/sys/modules/vmm/Makefile
  projects/bhyve/sys/x86/include/bus.h
  projects/bhyve/sys/x86/x86/local_apic.c

Modified: projects/bhyve/sys/amd64/amd64/apic_vector.S
==============================================================================
--- projects/bhyve/sys/amd64/amd64/apic_vector.S	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/amd64/amd64/apic_vector.S	Sat May 14 20:35:01 2011	(r221914)
@@ -57,8 +57,15 @@ IDTVEC(vec_name) ;							\
 	PUSH_FRAME ;							\
 	FAKE_MCOUNT(TF_RIP(%rsp)) ;					\
 	movq	lapic, %rdx ;	/* pointer to local APIC */		\
+	testq	%rdx, %rdx;						\
+	jnz	3f;							\
+	movl	$MSR_APIC_ISR ## index, %ecx;				\
+	rdmsr;								\
+	jmp	4f;							\
+3:	;								\
 	movl	LA_ISR + 16 * (index)(%rdx), %eax ;	/* load ISR */	\
-	bsrl	%eax, %eax ;	/* index of highest set bit in ISR */	\
+4:	;								\
+	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
 	jz	1f ;							\
 	addl	$(32 * index),%eax ;					\
 	movq	%rsp, %rsi	;                                       \
@@ -129,6 +136,26 @@ IDTVEC(errorint)
 	jmp	doreti
 
 #ifdef SMP
+
+/*
+ * We assume that %rax is being saved/restored outside of this macro
+ */
+#define	DO_EOI								\
+	movq	lapic, %rax;						\
+	testq	%rax, %rax;						\
+	jz	8f;							\
+	movl	$0, LA_EOI(%rax);					\
+	jmp	9f;							\
+8:;									\
+	pushq	%rcx;							\
+	pushq	%rdx;							\
+	xorl	%edx, %edx;	/* eax is already zero */		\
+	movl	$MSR_APIC_EOI, %ecx;					\
+	wrmsr;								\
+	popq	%rdx;							\
+	popq	%rcx;							\
+9:
+	
 /*
  * Global address space TLB shootdown.
  */
@@ -153,8 +180,7 @@ IDTVEC(invltlb)
 	movq	%cr3, %rax		/* invalidate the TLB */
 	movq	%rax, %cr3
 
-	movq	lapic, %rax
-	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	DO_EOI
 
 	lock
 	incl	smp_tlb_wait
@@ -186,8 +212,7 @@ IDTVEC(invlpg)
 	movq	smp_tlb_addr1, %rax
 	invlpg	(%rax)			/* invalidate single page */
 
-	movq	lapic, %rax
-	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	DO_EOI
 
 	lock
 	incl	smp_tlb_wait
@@ -224,8 +249,7 @@ IDTVEC(invlrng)
 	cmpq	%rax, %rdx
 	jb	1b
 
-	movq	lapic, %rax
-	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	DO_EOI
 
 	lock
 	incl	smp_tlb_wait
@@ -252,8 +276,7 @@ IDTVEC(invlcache)
 
 	wbinvd
 
-	movq	lapic, %rax
-	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	DO_EOI
 
 	lock
 	incl	smp_tlb_wait
@@ -269,9 +292,8 @@ IDTVEC(invlcache)
 IDTVEC(ipi_intr_bitmap_handler)		
 	PUSH_FRAME
 
-	movq	lapic, %rdx
-	movl	$0, LA_EOI(%rdx)	/* End Of Interrupt to APIC */
-	
+	DO_EOI
+
 	FAKE_MCOUNT(TF_RIP(%rsp))
 
 	call	ipi_bitmap_handler
@@ -286,8 +308,7 @@ IDTVEC(ipi_intr_bitmap_handler)		
 IDTVEC(cpustop)
 	PUSH_FRAME
 
-	movq	lapic, %rax
-	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	DO_EOI
 
 	call	cpustop_handler
 	jmp	doreti
@@ -300,8 +321,7 @@ IDTVEC(cpustop)
 IDTVEC(cpususpend)
 	PUSH_FRAME
 
-	movq	lapic, %rax
-	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	DO_EOI
 
 	call	cpususpend_handler
 
@@ -323,7 +343,6 @@ IDTVEC(rendezvous)
 	incq	(%rax)
 #endif
 	call	smp_rendezvous_action
-	movq	lapic, %rax
-	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	DO_EOI
 	jmp	doreti
 #endif /* SMP */

Modified: projects/bhyve/sys/amd64/amd64/intr_machdep.c
==============================================================================
--- projects/bhyve/sys/amd64/amd64/intr_machdep.c	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/amd64/amd64/intr_machdep.c	Sat May 14 20:35:01 2011	(r221914)
@@ -78,6 +78,8 @@ static STAILQ_HEAD(, pic) pics;
 
 #ifdef SMP
 static int assign_cpu;
+static int round_robin_interrupts = 1;
+TUNABLE_INT("round_robin_interrupts", &round_robin_interrupts);
 #endif
 
 static int	intr_assign_cpu(void *arg, u_char cpu);
@@ -460,6 +462,10 @@ intr_next_cpu(void)
 	if (!assign_cpu)
 		return (PCPU_GET(apic_id));
 
+	/* All interrupts go to the BSP if not allowed to round robin */
+	if (!round_robin_interrupts)
+		return (cpu_apic_ids[0]);
+
 	mtx_lock_spin(&icu_lock);
 	apic_id = cpu_apic_ids[current_cpu];
 	do {

Modified: projects/bhyve/sys/amd64/amd64/minidump_machdep.c
==============================================================================
--- projects/bhyve/sys/amd64/amd64/minidump_machdep.c	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/amd64/amd64/minidump_machdep.c	Sat May 14 20:35:01 2011	(r221914)
@@ -27,6 +27,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_pmap.h"
 #include "opt_watchdog.h"
 
 #include <sys/param.h>

Modified: projects/bhyve/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- projects/bhyve/sys/amd64/amd64/mp_machdep.c	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/amd64/amd64/mp_machdep.c	Sat May 14 20:35:01 2011	(r221914)
@@ -146,6 +146,26 @@ struct cpu_info {
 int cpu_apic_ids[MAXCPU];
 int apic_cpuids[MAX_APIC_ID + 1];
 
+/*
+ * Trampoline for hypervisor direct 64-bit jump.
+ *
+ *   0  -	signature for guest->host verification
+ *   8  -	virtual address of this page
+ *  16  -	instruction virtual address
+ *  24  -	stack pointer virtual address
+ *  32  -	CR3, physical address of kernel page table
+ *  40  -	24-byte area for null/code/data GDT entries
+ */
+#define MP_V64T_SIG	0xcafebabecafebabeULL
+struct mp_v64tramp {
+	uint64_t	mt_sig;
+	uint64_t	mt_virt;
+	uint64_t	mt_eip;
+	uint64_t	mt_rsp;
+	uint64_t	mt_cr3;
+	uint64_t	mt_gdtr[3];
+};
+
 /* Holds pending bitmap based IPIs per CPU */
 static volatile u_int cpu_ipi_pending[MAXCPU];
 
@@ -948,6 +968,29 @@ start_all_aps(void)
 		bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
 		bootAP = cpu;
 
+		/*
+		 * If running in a VM that doesn't support the unrestricted
+		 * guest 16-bit mode, forget most of the above and create
+		 * the data block that allows the hypervisor to direct-jump
+		 * into 64-bit mode. Copy this over the top of the 16-bit
+		 * bootstrap. The startup-IPI informs the hypervisor which
+		 * physical page this data block lies in. The hypervisor
+		 * will then use the block to initialise register state of
+		 * the AP in an almost identical fashion to how it builds
+		 * the BSP initial register state.
+		 */
+		if (testenv("hw.use_bvm_mptramp")) {
+			struct mp_v64tramp mv;
+
+			bzero(&mv, sizeof(mv));
+			mv.mt_sig = MP_V64T_SIG;
+			mv.mt_virt = (uint64_t) va;
+			mv.mt_eip = (uint64_t) init_secondary;
+			mv.mt_rsp = (uint64_t) bootSTK;
+			mv.mt_cr3 = KPML4phys;
+			bcopy(&mv, (void *) va, sizeof(mv));
+		}
+
 		/* attempt to start the Application Processor */
 		if (!start_ap(apic_id)) {
 			/* restore the warmstart vector */

Modified: projects/bhyve/sys/amd64/amd64/vm_machdep.c
==============================================================================
--- projects/bhyve/sys/amd64/amd64/vm_machdep.c	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/amd64/amd64/vm_machdep.c	Sat May 14 20:35:01 2011	(r221914)
@@ -514,8 +514,10 @@ cpu_reset_proxy()
 {
 
 	cpu_reset_proxy_active = 1;
-	while (cpu_reset_proxy_active == 1)
+	while (cpu_reset_proxy_active == 1) {
+		ia32_pause();
 		;	/* Wait for other cpu to see that we've started */
+	}
 	stop_cpus((1<<cpu_reset_proxyid));
 	printf("cpu_reset_proxy: Stopped CPU %d\n", cpu_reset_proxyid);
 	DELAY(1000000);
@@ -547,14 +549,17 @@ cpu_reset()
 			atomic_store_rel_int(&started_cpus, 1 << 0);
 
 			cnt = 0;
-			while (cpu_reset_proxy_active == 0 && cnt < 10000000)
+			while (cpu_reset_proxy_active == 0 && cnt < 10000000) {
+				ia32_pause();
 				cnt++;	/* Wait for BSP to announce restart */
+			}
 			if (cpu_reset_proxy_active == 0)
 				printf("cpu_reset: Failed to restart BSP\n");
 			enable_intr();
 			cpu_reset_proxy_active = 2;
 
-			while (1);
+			while (1)
+				ia32_pause();
 			/* NOTREACHED */
 		}
 

Modified: projects/bhyve/sys/amd64/conf/GENERIC
==============================================================================
--- projects/bhyve/sys/amd64/conf/GENERIC	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/amd64/conf/GENERIC	Sat May 14 20:35:01 2011	(r221914)
@@ -67,6 +67,7 @@ options 	INCLUDE_CONFIG_FILE     # Inclu
 
 # Debugging for use in -current
 options 	KDB			# Enable kernel debugger support.
+options		KDB_TRACE
 options 	DDB			# Support DDB.
 options 	GDB			# Support remote GDB.
 options 	DEADLKRES		# Enable the deadlock resolver
@@ -337,3 +338,8 @@ device		fwe		# Ethernet over FireWire (n
 device		fwip		# IP over FireWire (RFC 2734,3146)
 device		dcons		# Dumb console driver
 device		dcons_crom	# Configuration ROM for dcons
+
+# bhyve options
+device		bvmconsole	# brain dead simple bvm console
+device		bvmdebug	# brain dead simple bvm gdb pipe
+device		mptable

Modified: projects/bhyve/sys/amd64/include/specialreg.h
==============================================================================
--- projects/bhyve/sys/amd64/include/specialreg.h	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/amd64/include/specialreg.h	Sat May 14 20:35:01 2011	(r221914)
@@ -315,11 +315,42 @@
 #define	MSR_MC4_ADDR		0x412
 #define	MSR_MC4_MISC		0x413
 
+/* X2APIC MSRs */
+#define	MSR_APIC_ID		0x802
+#define	MSR_APIC_VERSION	0x803
+#define	MSR_APIC_TPR		0x808
+#define	MSR_APIC_EOI		0x80b
+#define	MSR_APIC_LDR		0x80d
+#define	MSR_APIC_SVR		0x80f
+#define	MSR_APIC_ISR0		0x810
+#define	MSR_APIC_ISR1		0x811
+#define	MSR_APIC_ISR2		0x812
+#define	MSR_APIC_ISR3		0x813
+#define	MSR_APIC_ISR4		0x814
+#define	MSR_APIC_ISR5		0x815
+#define	MSR_APIC_ISR6		0x816
+#define	MSR_APIC_ISR7		0x817
+#define	MSR_APIC_TMR0		0x818
+#define	MSR_APIC_IRR0		0x820
+#define	MSR_APIC_ESR		0x828
+#define	MSR_APIC_LVT_CMCI	0x82F
+#define	MSR_APIC_ICR		0x830
+#define	MSR_APIC_LVT_TIMER	0x832
+#define	MSR_APIC_LVT_THERMAL	0x833
+#define	MSR_APIC_LVT_PCINT	0x834
+#define	MSR_APIC_LVT_LINT0	0x835
+#define	MSR_APIC_LVT_LINT1	0x836
+#define	MSR_APIC_LVT_ERROR	0x837
+#define	MSR_APIC_ICR_TIMER	0x838
+#define	MSR_APIC_CCR_TIMER	0x839
+#define	MSR_APIC_DCR_TIMER	0x83e
+
 /*
  * Constants related to MSR's.
  */
-#define	APICBASE_RESERVED	0x000006ff
+#define	APICBASE_RESERVED	0x000002ff
 #define	APICBASE_BSP		0x00000100
+#define	APICBASE_X2APIC		0x00000400
 #define	APICBASE_ENABLED	0x00000800
 #define	APICBASE_ADDRESS	0xfffff000
 

Copied: projects/bhyve/sys/amd64/include/vmm.h (from r221828, projects/bhyve_ref/sys/amd64/include/vmm.h)
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ projects/bhyve/sys/amd64/include/vmm.h	Sat May 14 20:35:01 2011	(r221914, copy of r221828, projects/bhyve_ref/sys/amd64/include/vmm.h)
@@ -0,0 +1,268 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: vmm.h 482 2011-05-09 21:22:43Z grehan $
+ */
+
+#ifndef _VMM_H_
+#define	_VMM_H_
+
+#ifdef _KERNEL
+
+#define	VM_MAX_NAMELEN	32
+
+struct vm;
+struct vm_memory_segment;
+struct seg_desc;
+struct vm_exit;
+struct vm_run;
+struct vlapic;
+
+typedef int	(*vmm_init_func_t)(void);
+typedef int	(*vmm_cleanup_func_t)(void);
+typedef void *	(*vmi_init_func_t)(struct vm *vm); /* instance specific apis */
+typedef int	(*vmi_run_func_t)(void *vmi, int vcpu, register_t rip,
+				  struct vm_exit *vmexit);
+typedef void	(*vmi_cleanup_func_t)(void *vmi);
+typedef int	(*vmi_mmap_func_t)(void *vmi, vm_paddr_t gpa, vm_paddr_t hpa,
+				   size_t length, vm_memattr_t attr,
+				   int prot, boolean_t superpages_ok);
+typedef int	(*vmi_get_register_t)(void *vmi, int vcpu, int num,
+				      uint64_t *retval);
+typedef int	(*vmi_set_register_t)(void *vmi, int vcpu, int num,
+				      uint64_t val);
+typedef int	(*vmi_get_desc_t)(void *vmi, int vcpu, int num,
+				  struct seg_desc *desc);
+typedef int	(*vmi_set_desc_t)(void *vmi, int vcpu, int num,
+				  struct seg_desc *desc);
+typedef int	(*vmi_inject_event_t)(void *vmi, int vcpu,
+				      int type, int vector,
+				      uint32_t code, int code_valid);
+typedef	int	(*vmi_inject_nmi_t)(void *vmi, int vcpu);
+typedef int	(*vmi_get_cap_t)(void *vmi, int vcpu, int num, int *retval);
+typedef int	(*vmi_set_cap_t)(void *vmi, int vcpu, int num, int val);
+
+struct vmm_ops {
+	vmm_init_func_t		init;		/* module wide initialization */
+	vmm_cleanup_func_t	cleanup;
+
+	vmi_init_func_t		vminit;		/* vm-specific initialization */
+	vmi_run_func_t		vmrun;
+	vmi_cleanup_func_t	vmcleanup;
+	vmi_mmap_func_t		vmmmap;
+	vmi_get_register_t	vmgetreg;
+	vmi_set_register_t	vmsetreg;
+	vmi_get_desc_t		vmgetdesc;
+	vmi_set_desc_t		vmsetdesc;
+	vmi_inject_event_t	vminject;
+	vmi_inject_nmi_t	vmnmi;
+	vmi_get_cap_t		vmgetcap;
+	vmi_set_cap_t		vmsetcap;
+};
+
+extern struct vmm_ops vmm_ops_intel;
+extern struct vmm_ops vmm_ops_amd;
+
+struct vm *vm_create(const char *name);
+void vm_destroy(struct vm *vm);
+const char *vm_name(struct vm *vm);
+int vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t *ret_hpa);
+int vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
+int vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len);
+vm_paddr_t vm_gpa2hpa(struct vm *vm, vm_paddr_t gpa, size_t size);
+int vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase,
+	      struct vm_memory_segment *seg);
+int vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval);
+int vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val);
+int vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
+		    struct seg_desc *ret_desc);
+int vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
+		    struct seg_desc *desc);
+int vm_get_pinning(struct vm *vm, int vcpu, int *cpuid);
+int vm_set_pinning(struct vm *vm, int vcpu, int cpuid);
+int vm_run(struct vm *vm, struct vm_run *vmrun);
+int vm_inject_event(struct vm *vm, int vcpu, int type,
+		    int vector, uint32_t error_code, int error_code_valid);
+int vm_inject_nmi(struct vm *vm, int vcpu);
+uint64_t *vm_guest_msrs(struct vm *vm, int cpu);
+struct vlapic *vm_lapic(struct vm *vm, int cpu);
+int vm_get_capability(struct vm *vm, int vcpu, int type, int *val);
+int vm_set_capability(struct vm *vm, int vcpu, int type, int val);
+void vm_activate_cpu(struct vm *vm, int vcpu);
+cpumask_t vm_active_cpus(struct vm *vm);
+
+/*
+ * Return 1 if device indicated by bus/slot/func is supposed to be a
+ * pci passthrough device.
+ *
+ * Return 0 otherwise.
+ */
+int vmm_is_pptdev(int bus, int slot, int func);
+
+void *vm_iommu_domain(struct vm *vm);
+
+#define	VCPU_STOPPED	0
+#define	VCPU_RUNNING	1
+void vm_set_run_state(struct vm *vm, int vcpu, int running);
+int vm_get_run_state(struct vm *vm, int vcpu, int *hostcpu);
+
+void *vcpu_stats(struct vm *vm, int vcpu);
+
+static int __inline
+vcpu_is_running(struct vm *vm, int vcpu, int *hostcpu)
+{
+	return (vm_get_run_state(vm, vcpu, hostcpu) == VCPU_RUNNING);
+}
+
+static cpumask_t __inline
+vcpu_mask(int vcpuid)
+{
+	return ((cpumask_t)1 << vcpuid);
+}
+
+#endif	/* KERNEL */
+
+#define	VM_MAXCPU	8			/* maximum virtual cpus */
+
+/*
+ * Identifiers for events that can be injected into the VM
+ */
+enum vm_event_type {
+	VM_EVENT_NONE,
+	VM_HW_INTR,
+	VM_NMI,
+	VM_HW_EXCEPTION,
+	VM_SW_INTR,
+	VM_PRIV_SW_EXCEPTION,
+	VM_SW_EXCEPTION,
+	VM_EVENT_MAX
+};
+
+/*
+ * Identifiers for architecturally defined registers.
+ */
+enum vm_reg_name {
+	VM_REG_GUEST_RAX,
+	VM_REG_GUEST_RBX,
+	VM_REG_GUEST_RCX,
+	VM_REG_GUEST_RDX,
+	VM_REG_GUEST_RSI,
+	VM_REG_GUEST_RDI,
+	VM_REG_GUEST_RBP,
+	VM_REG_GUEST_R8,
+	VM_REG_GUEST_R9,
+	VM_REG_GUEST_R10,
+	VM_REG_GUEST_R11,
+	VM_REG_GUEST_R12,
+	VM_REG_GUEST_R13,
+	VM_REG_GUEST_R14,
+	VM_REG_GUEST_R15,
+	VM_REG_GUEST_CR0,
+	VM_REG_GUEST_CR3,
+	VM_REG_GUEST_CR4,
+	VM_REG_GUEST_DR7,
+	VM_REG_GUEST_RSP,
+	VM_REG_GUEST_RIP,
+	VM_REG_GUEST_RFLAGS,
+	VM_REG_GUEST_ES,
+	VM_REG_GUEST_CS,
+	VM_REG_GUEST_SS,
+	VM_REG_GUEST_DS,
+	VM_REG_GUEST_FS,
+	VM_REG_GUEST_GS,
+	VM_REG_GUEST_LDTR,
+	VM_REG_GUEST_TR,
+	VM_REG_GUEST_IDTR,
+	VM_REG_GUEST_GDTR,
+	VM_REG_GUEST_EFER,
+	VM_REG_LAST
+};
+
+/*
+ * Identifiers for optional vmm capabilities
+ */
+enum vm_cap_type {
+	VM_CAP_HALT_EXIT,
+	VM_CAP_MTRAP_EXIT,
+	VM_CAP_PAUSE_EXIT,
+	VM_CAP_UNRESTRICTED_GUEST,
+	VM_CAP_MAX
+};
+
+/*
+ * The 'access' field has the format specified in Table 21-2 of the Intel
+ * Architecture Manual vol 3b.
+ *
+ * XXX The contents of the 'access' field are architecturally defined except
+ * bit 16 - Segment Unusable.
+ */
+struct seg_desc {
+	uint64_t	base;
+	uint32_t	limit;
+	uint32_t	access;
+};
+
+enum vm_exitcode {
+	VM_EXITCODE_INOUT,
+	VM_EXITCODE_VMX,
+	VM_EXITCODE_BOGUS,
+	VM_EXITCODE_RDMSR,
+	VM_EXITCODE_WRMSR,
+	VM_EXITCODE_HLT,
+	VM_EXITCODE_MTRAP,
+	VM_EXITCODE_PAUSE,
+	VM_EXITCODE_MAX,
+};
+
+struct vm_exit {
+	enum vm_exitcode	exitcode;
+	int			inst_length;	/* 0 means unknown */
+	uint64_t		rip;
+	union {
+		struct {
+			uint16_t	bytes:3;	/* 1 or 2 or 4 */
+			uint16_t	in:1;		/* out is 0, in is 1 */
+			uint16_t	string:1;
+			uint16_t	rep:1;
+			uint16_t	port;
+			uint32_t	eax;		/* valid for out */
+		} inout;
+		/*
+		 * VMX specific payload. Used when there is no "better"
+		 * exitcode to represent the VM-exit.
+		 */
+		struct {
+			int		error;		/* vmx inst error */
+			uint32_t	exit_reason;
+			uint64_t	exit_qualification;
+		} vmx;
+		struct {
+			uint32_t	code;		/* ecx value */
+			uint64_t	wval;
+		} msr;
+	} u;
+};
+
+#endif	/* _VMM_H_ */

Copied: projects/bhyve/sys/amd64/include/vmm_dev.h (from r221828, projects/bhyve_ref/sys/amd64/include/vmm_dev.h)
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ projects/bhyve/sys/amd64/include/vmm_dev.h	Sat May 14 20:35:01 2011	(r221914, copy of r221828, projects/bhyve_ref/sys/amd64/include/vmm_dev.h)
@@ -0,0 +1,191 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: vmm_dev.h 482 2011-05-09 21:22:43Z grehan $
+ */
+
+#ifndef	_VMM_DEV_H_
+#define	_VMM_DEV_H_
+
+#ifdef _KERNEL
+void	vmmdev_init(void);
+void	vmmdev_cleanup(void);
+#endif
+
+struct vm_memory_segment {
+	vm_paddr_t	hpa;	/* out */
+	vm_paddr_t	gpa;	/* in */
+	size_t		len;	/* in */
+};
+
+struct vm_register {
+	int		cpuid;
+	int		regnum;		/* enum vm_reg_name */
+	uint64_t	regval;
+};
+
+struct vm_seg_desc {			/* data or code segment */
+	int		cpuid;
+	int		regnum;		/* enum vm_reg_name */
+	struct seg_desc desc;
+};
+
+struct vm_pin {
+	int		vm_cpuid;
+	int		host_cpuid;	/* -1 to unpin */
+};
+
+struct vm_run {
+	int		cpuid;
+	uint64_t	rip;		/* start running here */
+	struct vm_exit	vm_exit;
+};
+
+struct vm_event {
+	int		cpuid;
+	enum vm_event_type type;
+	int		vector;
+	uint32_t	error_code;
+	int		error_code_valid;
+};
+
+struct vm_lapic_irq {
+	int		cpuid;
+	int		vector;
+};
+
+struct vm_capability {
+	int		cpuid;
+	enum vm_cap_type captype;
+	int		capval;
+	int		allcpus;
+};
+
+struct vm_pptdev {
+	int		bus;
+	int		slot;
+	int		func;
+};
+
+struct vm_pptdev_mmio {
+	int		bus;
+	int		slot;
+	int		func;
+	vm_paddr_t	gpa;
+	vm_paddr_t	hpa;
+	size_t		len;
+};
+
+struct vm_pptdev_msi {
+	int		vcpu;
+	int		bus;
+	int		slot;
+	int		func;
+	int		numvec;		/* 0 means disabled */
+	int		vector;
+	int		destcpu;
+};
+
+struct vm_nmi {
+	int		cpuid;
+};
+
+#define	MAX_VM_STATS	64
+struct vm_stats {
+	int		cpuid;				/* in */
+	int		num_entries;			/* out */
+	struct timeval	tv;
+	uint64_t	statbuf[MAX_VM_STATS];
+};
+
+struct vm_stat_desc {
+	int		index;				/* in */
+	char		desc[128];			/* out */
+};
+
+enum {
+	IOCNUM_RUN,
+	IOCNUM_SET_PINNING,
+	IOCNUM_GET_PINNING,
+	IOCNUM_MAP_MEMORY,
+	IOCNUM_GET_MEMORY_SEG,
+	IOCNUM_SET_REGISTER,
+	IOCNUM_GET_REGISTER,
+	IOCNUM_SET_SEGMENT_DESCRIPTOR,
+	IOCNUM_GET_SEGMENT_DESCRIPTOR,
+	IOCNUM_INJECT_EVENT,
+	IOCNUM_LAPIC_IRQ,
+	IOCNUM_SET_CAPABILITY,
+	IOCNUM_GET_CAPABILITY,
+	IOCNUM_BIND_PPTDEV,
+	IOCNUM_UNBIND_PPTDEV,
+	IOCNUM_MAP_PPTDEV_MMIO,
+	IOCNUM_PPTDEV_MSI,
+	IOCNUM_INJECT_NMI,
+	IOCNUM_VM_STATS,
+	IOCNUM_VM_STAT_DESC,
+};
+
+#define	VM_RUN		\
+	_IOWR('v', IOCNUM_RUN, struct vm_run)
+#define	VM_SET_PINNING	\
+	_IOW('v', IOCNUM_SET_PINNING, struct vm_pin)
+#define	VM_GET_PINNING	\
+	_IOWR('v', IOCNUM_GET_PINNING, struct vm_pin)
+#define	VM_MAP_MEMORY	\
+	_IOWR('v', IOCNUM_MAP_MEMORY, struct vm_memory_segment)
+#define	VM_GET_MEMORY_SEG \
+	_IOWR('v', IOCNUM_GET_MEMORY_SEG, struct vm_memory_segment)
+#define	VM_SET_REGISTER \
+	_IOW('v', IOCNUM_SET_REGISTER, struct vm_register)
+#define	VM_GET_REGISTER \
+	_IOWR('v', IOCNUM_GET_REGISTER, struct vm_register)
+#define	VM_SET_SEGMENT_DESCRIPTOR \
+	_IOW('v', IOCNUM_SET_SEGMENT_DESCRIPTOR, struct vm_seg_desc)
+#define	VM_GET_SEGMENT_DESCRIPTOR \
+	_IOWR('v', IOCNUM_GET_SEGMENT_DESCRIPTOR, struct vm_seg_desc)
+#define	VM_INJECT_EVENT	\
+	_IOW('v', IOCNUM_INJECT_EVENT, struct vm_event)
+#define	VM_LAPIC_IRQ 		\
+	_IOW('v', IOCNUM_LAPIC_IRQ, struct vm_lapic_irq)
+#define	VM_SET_CAPABILITY \
+	_IOW('v', IOCNUM_SET_CAPABILITY, struct vm_capability)
+#define	VM_GET_CAPABILITY \
+	_IOWR('v', IOCNUM_GET_CAPABILITY, struct vm_capability)
+#define	VM_BIND_PPTDEV \
+	_IOW('v', IOCNUM_BIND_PPTDEV, struct vm_pptdev)
+#define	VM_UNBIND_PPTDEV \
+	_IOW('v', IOCNUM_UNBIND_PPTDEV, struct vm_pptdev)
+#define	VM_MAP_PPTDEV_MMIO \
+	_IOW('v', IOCNUM_MAP_PPTDEV_MMIO, struct vm_pptdev_mmio)
+#define	VM_PPTDEV_MSI \
+	_IOW('v', IOCNUM_PPTDEV_MSI, struct vm_pptdev_msi)
+#define VM_INJECT_NMI \
+	_IOW('v', IOCNUM_INJECT_NMI, struct vm_nmi)
+#define	VM_STATS \
+	_IOWR('v', IOCNUM_VM_STATS, struct vm_stats)
+#define	VM_STAT_DESC \
+	_IOWR('v', IOCNUM_VM_STAT_DESC, struct vm_stat_desc)
+#endif

Modified: projects/bhyve/sys/amd64/vmm/io/ppt.c
==============================================================================
--- projects/bhyve_ref/sys/amd64/vmm/io/ppt.c	Fri May 13 04:54:01 2011	(r221828)
+++ projects/bhyve/sys/amd64/vmm/io/ppt.c	Sat May 14 20:35:01 2011	(r221914)
@@ -433,7 +433,7 @@ ppt_setup_msi(struct vm *vm, int vcpu, i
 		ppt->msi.arg[i].msg = i;
 
 		error = bus_setup_intr(ppt->dev, ppt->msi.res[i],
-				       INTR_TYPE_NET | INTR_MPSAFE | INTR_FAST,
+				       INTR_TYPE_NET | INTR_MPSAFE,
 				       pptintr, NULL, &ppt->msi.arg[i],
 				       &ppt->msi.cookie[i]);
 		if (error != 0)

Modified: projects/bhyve/sys/amd64/vmm/io/vlapic.c
==============================================================================
--- projects/bhyve_ref/sys/amd64/vmm/io/vlapic.c	Fri May 13 04:54:01 2011	(r221828)
+++ projects/bhyve/sys/amd64/vmm/io/vlapic.c	Sat May 14 20:35:01 2011	(r221914)
@@ -63,7 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 
 #include <machine/clock.h>
-#include <machine/apicreg.h>
+#include <x86/apicreg.h>
 
 #include <machine/vmm.h>
 

Modified: projects/bhyve/sys/amd64/vmm/vmm.c
==============================================================================
--- projects/bhyve_ref/sys/amd64/vmm/vmm.c	Fri May 13 04:54:01 2011	(r221828)
+++ projects/bhyve/sys/amd64/vmm/vmm.c	Sat May 14 20:35:01 2011	(r221914)
@@ -46,7 +46,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/vm.h>
 #include <machine/pcb.h>
-#include <machine/apicreg.h>
+#include <x86/apicreg.h>
 
 #include <machine/vmm.h>
 #include "vmm_mem.h"
@@ -160,7 +160,8 @@ vcpu_init(struct vm *vm, uint32_t vcpu_i
 	vcpu->hostcpu = -1;
 	vcpu->vcpuid = vcpu_id;
 	vcpu->vlapic = vlapic_init(vm, vcpu_id);
-	fpugetregs(curthread, &vcpu->savefpu);
+	fpugetregs(curthread);
+	vcpu->savefpu = curthread->td_pcb->pcb_user_save;
 	vcpu->stats = vmm_stat_alloc();
 }
 
@@ -545,7 +546,7 @@ vm_run(struct vm *vm, struct vm_run *vmr
 	tscval = rdtsc();
 
 	pcb = PCPU_GET(curpcb);
-	pcb->pcb_full_iret = 1;
+	set_pcb_flags(pcb, PCB_FULL_IRET);
 
 	vcpu->hostcpu = curcpu;
 

Modified: projects/bhyve/sys/amd64/vmm/vmm_dev.c
==============================================================================
--- projects/bhyve_ref/sys/amd64/vmm/vmm_dev.c	Fri May 13 04:54:01 2011	(r221828)
+++ projects/bhyve/sys/amd64/vmm/vmm_dev.c	Sat May 14 20:35:01 2011	(r221914)
@@ -336,7 +336,8 @@ done:
 }
 
 static int
-vmmdev_mmap(struct cdev *cdev, vm_offset_t offset, vm_paddr_t *paddr, int nprot)
+vmmdev_mmap(struct cdev *cdev, vm_ooffset_t offset, vm_paddr_t *paddr,
+    int nprot, vm_memattr_t *memattr)
 {
 	int error;
 	struct vmmdev_softc *sc;

Modified: projects/bhyve/sys/amd64/vmm/vmm_ipi.c
==============================================================================
--- projects/bhyve_ref/sys/amd64/vmm/vmm_ipi.c	Fri May 13 04:54:01 2011	(r221828)
+++ projects/bhyve/sys/amd64/vmm/vmm_ipi.c	Sat May 14 20:35:01 2011	(r221914)
@@ -99,5 +99,5 @@ vm_interrupt_hostcpu(struct vm *vm, int 
 	int hostcpu;
 
 	if (vcpu_is_running(vm, vcpu, &hostcpu) && hostcpu != curcpu)
-		ipi_selected((cpumask_t)1 << hostcpu, ipinum);
+		ipi_cpu(hostcpu, ipinum);
 }

Modified: projects/bhyve/sys/amd64/vmm/vmm_msr.c
==============================================================================
--- projects/bhyve_ref/sys/amd64/vmm/vmm_msr.c	Fri May 13 04:54:01 2011	(r221828)
+++ projects/bhyve/sys/amd64/vmm/vmm_msr.c	Sat May 14 20:35:01 2011	(r221914)
@@ -33,7 +33,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 
 #include <machine/specialreg.h>
-#include <machine/apicreg.h>
+#include <x86/apicreg.h>
 
 #include <machine/vmm.h>
 #include "vmm_lapic.h"

Modified: projects/bhyve/sys/conf/files.amd64
==============================================================================
--- projects/bhyve/sys/conf/files.amd64	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/conf/files.amd64	Sat May 14 20:35:01 2011	(r221914)
@@ -295,6 +295,11 @@ libkern/memset.c		standard
 compat/x86bios/x86bios.c	optional x86bios | atkbd | dpms | vesa
 contrib/x86emu/x86emu.c		optional x86bios | atkbd | dpms | vesa
 #
+# bvm console
+#
+dev/bvm/bvm_console.c		optional	bvmconsole
+dev/bvm/bvm_dbg.c		optional	bvmdebug
+#
 # x86 shared code between IA32, AMD64 and PC98 architectures
 #
 x86/acpica/OsdEnvironment.c	optional	acpi

Modified: projects/bhyve/sys/conf/options.amd64
==============================================================================
--- projects/bhyve/sys/conf/options.amd64	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/conf/options.amd64	Sat May 14 20:35:01 2011	(r221914)
@@ -10,6 +10,7 @@ PERFMON
 PMAP_SHPGPERPROC	opt_pmap.h
 MPTABLE_FORCE_HTT
 MP_WATCHDOG
+NKPT			opt_pmap.h
 
 # Options for emulators.  These should only be used at config time, so
 # they are handled like options for static filesystems

Modified: projects/bhyve/sys/kern/subr_bus.c
==============================================================================
--- projects/bhyve/sys/kern/subr_bus.c	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/kern/subr_bus.c	Sat May 14 20:35:01 2011	(r221914)
@@ -2683,7 +2683,7 @@ device_attach(device_t dev)
 		printf("device_attach: %s%d attach returned %d\n",
 		    dev->driver->name, dev->unit, error);
 		/* Unset the class; set in device_probe_child */
-		if (dev->devclass == NULL)
+		if ((dev->flags & DF_FIXEDCLASS) == 0)
 			device_set_devclass(dev, NULL);
 		device_set_driver(dev, NULL);
 		device_sysctl_fini(dev);

Modified: projects/bhyve/sys/modules/Makefile
==============================================================================
--- projects/bhyve/sys/modules/Makefile	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/modules/Makefile	Sat May 14 20:35:01 2011	(r221914)
@@ -43,6 +43,7 @@ SUBDIR=	${_3dfx} \
 	${_bxe} \
 	${_bios} \
 	${_bktr} \
+	${_blackhole} \
 	${_bm} \
 	bridgestp \
 	bwi \
@@ -311,6 +312,7 @@ SUBDIR=	${_3dfx} \
 	${_vesa} \
 	vge \
 	vkbd \
+	${_vmm} \
 	${_vpo} \
 	vr \
 	vte \
@@ -541,6 +543,7 @@ _amdsbwd=	amdsbwd
 _amdtemp=	amdtemp
 _arcmsr=	arcmsr
 _asmc=		asmc
+_blackhole=	blackhole
 _bxe=		bxe
 _cardbus=	cardbus
 _cbb=		cbb
@@ -617,6 +620,7 @@ _sppp=		sppp
 _tpm=		tpm
 _twa=		twa
 _vesa=		vesa
+_vmm=		vmm
 _x86bios=	x86bios
 _wi=		wi
 _wpi=		wpi

Modified: projects/bhyve/sys/modules/vmm/Makefile
==============================================================================
--- projects/bhyve_ref/sys/modules/vmm/Makefile	Fri May 13 04:54:01 2011	(r221828)
+++ projects/bhyve/sys/modules/vmm/Makefile	Sat May 14 20:35:01 2011	(r221914)
@@ -9,7 +9,7 @@ KMOD=	vmm
 
 SRCS=	device_if.h bus_if.h pci_if.h
 
-CFLAGS+= -DVMM_KEEP_STATS
+CFLAGS+= -DVMM_KEEP_STATS -DSMP
 CFLAGS+= -DOLD_BINUTILS
 CFLAGS+= -I${.CURDIR}/../../amd64/vmm	
 CFLAGS+= -I${.CURDIR}/../../amd64/vmm/io	
@@ -60,7 +60,7 @@ vmx_support.o:	vmx_support.S vmx_assym.s
 	${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \
 	    ${.IMPSRC} -o ${.TARGET}
 
-vmx_genassym.o: vmx_genassym.c @ machine
+vmx_genassym.o: vmx_genassym.c @ machine x86
 	${CC} -c ${CFLAGS:N-fno-common} ${.IMPSRC}
 
 .include <bsd.kmod.mk>

Modified: projects/bhyve/sys/x86/include/bus.h
==============================================================================
--- projects/bhyve/sys/x86/include/bus.h	Sat May 14 20:31:04 2011	(r221913)
+++ projects/bhyve/sys/x86/include/bus.h	Sat May 14 20:35:01 2011	(r221914)
@@ -279,9 +279,13 @@ bus_space_read_multi_1(bus_space_tag_t t
 		       bus_size_t offset, u_int8_t *addr, size_t count)
 {
 
-	if (tag == X86_BUS_SPACE_IO)
-		insb(bsh + offset, addr, count);
-	else {
+	if (tag == X86_BUS_SPACE_IO) {
+		while (count > 0) {
+			*addr = inb(bsh + offset);
+			count--;
+			addr++;
+		}
+	} else {
 #ifdef __GNUCLIKE_ASM
 		__asm __volatile("				\n\
 			cld					\n\
@@ -300,9 +304,13 @@ bus_space_read_multi_2(bus_space_tag_t t
 		       bus_size_t offset, u_int16_t *addr, size_t count)
 {
 
-	if (tag == X86_BUS_SPACE_IO)
-		insw(bsh + offset, addr, count);
-	else {
+	if (tag == X86_BUS_SPACE_IO) {
+		while (count > 0) {
+			*addr = inw(bsh + offset);
+			count--;
+			addr++;
+		}
+	} else {
 #ifdef __GNUCLIKE_ASM
 		__asm __volatile("				\n\
 			cld					\n\
@@ -321,9 +329,13 @@ bus_space_read_multi_4(bus_space_tag_t t
 		       bus_size_t offset, u_int32_t *addr, size_t count)
 {
 
-	if (tag == X86_BUS_SPACE_IO)
-		insl(bsh + offset, addr, count);
-	else {
+	if (tag == X86_BUS_SPACE_IO) {
+		while (count > 0) {

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***


More information about the svn-src-projects mailing list