hack for getting suspend/resume to half work on an IBM Thinkpad x60s [SMP]

[Andrea Bittau]

This is a half working hack for getting suspend/resume to "work" on an IBM
Thinkpad x60s [Intel Duo].  It has two limitations:
1) All interrupts are routed to the BSP.
2) Upon resume, the AP sits in the idle loop.  You can try to make it escape,
   but it doesn't quite work yet.
I think these issues are both related and it has to do with the way how i
restore the second CPU.  I think the page tables are messed up, and perhaps even
other state.  Hopefully someone can give me some insight on how to fix this.
[Question: from init_secondary() how do I get back to where I was [the IPI_STOP
handler]? =D  Do I need to do something like acpi_wakeup.c?  Isn't there an
easier way?] Also, if people seem interested, I can try and write a proper
patch.

In order to get suspend/resume to work, there are three main hurdles to jump:
1) SATA suspend/resume seems broken.  Change the SATA option in the BIOS to
   "compatible".

2) apic.  FreeBSD reconfigures the io apic upon resume, but not the local apic.
   The patch attached to this mail fixes this.  Indeed, it almost does so in the
   "proper" way and not so much of a hack =D.

3) SMP.  The second core needs to be killed and woken up as appropriate.  The
   way I do this is quite lame.
   - Force the second core in the idle loop by setting machdep.hlt_cpus=2.
   - make system look like UP instead of SMP [i.e. deactivate SMP] & suspend.
   - resume, wake up other core [which will run idle process] and activate SMP.

To get this to work in practice:
0) unload all drivers.
1) Apply the patch against today's freebsd-current.
2) My /etc/rc.suspend looks like this:
#!/bin/sh
sync
sync
sysctl machdep.hlt_cpus=2
sleep 2

3) zzz

4) if you want to crash: sysctl machdep.hlt_cpus=0

have fun

---

Index: dev/acpica/acpi.c
===================================================================
RCS file: /home/ncvs/src/sys/dev/acpica/acpi.c,v
retrieving revision 1.228
diff -u -p -r1.228 acpi.c
--- dev/acpica/acpi.c	11 Sep 2006 19:32:54 -0000	1.228
+++ dev/acpica/acpi.c	20 Sep 2006 23:19:35 -0000
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD: src/sys/dev/acpica/a
 #include <sys/power.h>
 #include <sys/sbuf.h>
 #include <sys/smp.h>
+#include <sys/sched.h>
 
 #include <machine/resource.h>
 #include <machine/bus.h>
@@ -2111,6 +2112,30 @@ enum acpi_sleep_state {
     ACPI_SS_SLEPT,
 };
 
+static void sorbo_stop_cpus(void);
+static void sorbo_stop_cpus(void)
+{
+	printf("stopping cpus\n");
+
+	/* make sure BSP is running */
+	mtx_lock_spin(&sched_lock);
+	sched_bind(curthread, 0);
+	mtx_unlock_spin(&sched_lock);
+	KASSERT(PCPU_GET(cpuid) == 0, ("life sux"));
+
+	/* kill em all */
+	stop_cpus(PCPU_GET(other_cpus));
+
+	/* tell fbsd we aint smp anymore */
+	mp_ncpus = 1;
+	all_cpus = 1;
+	smp_cpus = 1;
+	smp_started = 0;
+
+	printf("cpus stopped\n");
+}
+
+extern void sorbo_start_cpus(void);
 /*
  * Set the system sleep state
  *
@@ -2123,9 +2148,16 @@ acpi_SetSleepState(struct acpi_softc *sc
     UINT8	TypeA;
     UINT8	TypeB;
     enum acpi_sleep_state slp_state;
+    static int smp = 0;
 
     ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, state);
 
+	if (smp_started)
+		smp = 1;
+
+	if (smp)
+		sorbo_stop_cpus();
+
     status = AE_OK;
     ACPI_LOCK(acpi);
     if (sc->acpi_sleep_disabled) {
@@ -2241,6 +2273,10 @@ acpi_SetSleepState(struct acpi_softc *sc
 	timeout(acpi_sleep_enable, (caddr_t)sc, hz * ACPI_MINIMUM_AWAKETIME);
 
     mtx_unlock(&Giant);
+
+	if (smp)
+		sorbo_start_cpus();
+
     return_ACPI_STATUS (status);
 }
 
Index: i386/i386/intr_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/intr_machdep.c,v
retrieving revision 1.19
diff -u -p -r1.19 intr_machdep.c
--- i386/i386/intr_machdep.c	12 Jul 2006 21:22:43 -0000	1.19
+++ i386/i386/intr_machdep.c	20 Sep 2006 23:19:38 -0000
@@ -252,6 +252,7 @@ intr_execute_handlers(struct intsrc *isr
 	td->td_intr_nesting_level--;
 }
 
+extern void lapic_resume(void);
 void
 intr_resume(void)
 {
@@ -259,6 +260,7 @@ intr_resume(void)
 	int i;
 
 	mtx_lock_spin(&intr_table_lock);
+	lapic_resume();
 	for (i = 0, isrc = interrupt_sources; i < NUM_IO_INTS; i++, isrc++)
 		if (*isrc != NULL && (*isrc)->is_pic->pic_resume != NULL)
 			(*isrc)->is_pic->pic_resume(*isrc);
Index: i386/i386/local_apic.c
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/local_apic.c,v
retrieving revision 1.31
diff -u -p -r1.31 local_apic.c
--- i386/i386/local_apic.c	11 Sep 2006 20:12:42 -0000	1.31
+++ i386/i386/local_apic.c	20 Sep 2006 23:19:38 -0000
@@ -1117,3 +1117,14 @@ lapic_ipi_vectored(u_int vector, int des
 #endif /* DETECT_DEADLOCK */
 }
 #endif /* SMP */
+
+void lapic_resume(void);
+void lapic_resume(void)
+{
+	lapic_setup();
+
+	/* setup BSP timer */
+	lapic_timer_set_divisor(lapic_timer_divisor);
+	lapic_timer_periodic(lapic_timer_period);
+	lapic_timer_enable_intr();
+}
Index: i386/i386/mp_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/mp_machdep.c,v
retrieving revision 1.270
diff -u -p -r1.270 mp_machdep.c
--- i386/i386/mp_machdep.c	11 Sep 2006 20:10:42 -0000	1.270
+++ i386/i386/mp_machdep.c	20 Sep 2006 23:19:39 -0000
@@ -503,6 +503,7 @@ init_secondary(void)
 	int	gsel_tss;
 	int	x, myid;
 	u_int	cr0;
+	static int resuming = 0;
 
 	/* bootAP is set in start_ap() to our ID. */
 	myid = bootAP;
@@ -643,7 +644,10 @@ init_secondary(void)
 	 * spinlock_exit() will simply adjust the counts without allowing
 	 * spin lock using code to interrupt us.
 	 */
-	spinlock_exit();
+	if (!resuming) {
+		spinlock_exit();
+		resuming = 1;
+	}
 	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
 
 	PCPU_SET(switchtime, cpu_ticks());
@@ -682,7 +686,7 @@ set_interrupt_apic_ids(void)
 		    apic_id % hyperthreading_cpus != 0)
 			continue;
 
-		intr_add_cpu(apic_id);
+		/* intr_add_cpu(apic_id); */
 	}
 }
 
@@ -1528,3 +1532,44 @@ mp_ipi_intrcnt(void *dummy)
 }
 SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL)
 #endif
+
+void sorbo_start_cpus(void);
+void sorbo_start_cpus(void)
+{
+	uintptr_t kptbase = (uintptr_t)(void *)KPTphys;
+	int i;
+	unsigned int *x = (unsigned int*) (KERNBASE + (u_int) IdlePTD);
+	register_t eflags;
+
+	eflags = intr_disable();
+	printf("starting cpus\n");
+
+	/* identity map the boot code of AP */
+	for (i = 0; i < NKPT; i++)
+		x[i] = (pd_entry_t)(PG_V | PG_RW |
+				    ((kptbase + i * PAGE_SIZE) & PG_FRAME));
+
+	mp_ncpus = 2;
+	aps_ready = 0;
+	invltlb();
+
+	/* start the AP */
+	if (!start_ap(1))
+		panic("Can't start AP\n");
+	all_cpus |= (1 << 1);
+
+	/* clear the identity map */
+	for (i = 0; i < NKPT; i++)
+		x[i] = 0;
+	pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
+
+	/* we're SMP again */
+	atomic_store_rel_int(&stopped_cpus, 0);
+	atomic_store_rel_int(&started_cpus, 3);
+
+	/* let the APs own it */
+	release_aps(NULL);
+
+	printf("cpus started\n");
+	intr_restore(eflags);
+}