Some evidence about the PowerMac G5 multiprocessor boot hang ups with the modern VM_MAX_KERNEL_ADDRESS value

[Mark Millard]

I added some bootverbose messages to:

static int
powermac_smp_start_cpu(platform_t plat, struct pcpu *pc)
(in /usr/src/sys/powerpc/powermac/platform_powermac.c )

and:

void
machdep_ap_bootstrap(void)
( /usr/src/sys/powerpc/powerpc/mp_machdep.c )

and:

static void
cpu_mp_unleash(void *dummy)
( /usr/src/sys/powerpc/powerpc/mp_machdep.c )



The result was (typed from a image and
showing normal messages too):

Adding CPU 0, hwref=cd38, awkae=1
Waling up CPU 3 (dev=c480)
Before reset 4&0 for CPU 3, hwref=c480, awake=0
After reset 4&0 for CPU3, hwref=c480, awake=0
After attempted wait for awake CPU 3, hwref=c480, awake=0
cpu_mp_unleash after platform_smp_start_cpu and waiting: CPU3, hwref=x480, awake=0
cpu_mp_unleash adding pc_cpuid to stopped_cpus: CPU 3 (dev=c480)
Waking up CPU 2 (dev=c768)
Before reset 4&0 for CPU 2, hwref=c768, awake=0

There is no more text.

Part of the information is what messages were not displayed.

Some notes:

Only CPU 0 gots a: smp_cpus++ (resulting in 1).

There is no solid evidence for if machdep_ap_bootstrap was
ever used for CPU 3 or CPU 2.

The wait loop after the platform_smp_start_cpu call in
cpu_mp_unleash for CPU 3 is not used. The code in
machdep_ap_bootstrap:

        PCPU_SET(awake, 1);
        __asm __volatile("msync; isync");

did not have a visible effect on the pc->pc_awake value in
platform_smp_start_cpu and cpu_mp_unleash. I have no
unique evidence that it was executed at all.

My messages added to machdep_ap_bootstrap were not displayed.

platform_smp_start_cpu did not return for CPU 2.
cpu_mp_unleash makes no more progress. In fact the
2 resets do not complete overall for CPU 2 (but did
for CPU 3, even if it is unclear what code was
executed):

        *rstvec = 4;
        powerpc_sync();
        (void)(*rstvec);
        powerpc_sync();
        DELAY(1);
        *rstvec = 0;
        powerpc_sync();
        (void)(*rstvec);
        powerpc_sync();

I'll see about getting more information about the resets used.



For reference for the added messages:

# svnlite diff /usr/src/sys/powerpc/powermac/platform_powermac.c /usr/src/sys/powerpc/powerpc/mp_machdep.c
Index: /usr/src/sys/powerpc/powermac/platform_powermac.c
===================================================================
--- /usr/src/sys/powerpc/powermac/platform_powermac.c	(revision 344018)
+++ /usr/src/sys/powerpc/powermac/platform_powermac.c	(working copy)
@@ -371,6 +371,11 @@
 		rstvec_virtbase = pmap_mapdev(0x80000000, PAGE_SIZE);
 
 	rstvec = rstvec_virtbase + reset;
+ 
+        if (bootverbose) // HACK!!!
+                printf("Before reset 4&0 for CPU %d, hwref=%jx, awake=%x\n",
+                    pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+                    pc->pc_awake);
 
 	*rstvec = 4;
 	powerpc_sync();
@@ -382,10 +387,20 @@
 	(void)(*rstvec);
 	powerpc_sync();
 
+        if (bootverbose) // HACK!!!
+                printf("After reset 4&0 for CPU %d, hwref=%jx, awake=%x\n",
+                    pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+                    pc->pc_awake);
+
 	timeout = 10000;
 	while (!pc->pc_awake && timeout--)
 		DELAY(100);
 
+        if (bootverbose) // HACK!!!
+                printf("After attempted wait for awake CPU %d, hwref=%jx, awake=%x\n",
+                    pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+                    pc->pc_awake);
+
 	return ((pc->pc_awake) ? 0 : EBUSY);
 #else
 	/* No SMP support */
Index: /usr/src/sys/powerpc/powerpc/mp_machdep.c
===================================================================
--- /usr/src/sys/powerpc/powerpc/mp_machdep.c	(revision 344018)
+++ /usr/src/sys/powerpc/powerpc/mp_machdep.c	(working copy)
@@ -97,6 +97,9 @@
 	/* Initialize decrementer */
 	decr_ap_init();
 
+        if (bootverbose) // HACK!!!
+                printf("machdep_ap_bootstrap before ap_boot_mtx lock: AP CPU #%d launched\n", PCPU_GET(cpuid));
+
 	/* Serialize console output and AP count increment */
 	mtx_lock_spin(&ap_boot_mtx);
 	ap_awake++;
@@ -109,6 +112,8 @@
 
 	while(smp_started == 0)
 		;
+        if (bootverbose) // HACK!!!
+                printf("machdep_ap_bootstrap after smp_started!=0: AP CPU #%d launched\n", PCPU_GET(cpuid));
 
 	/* Start per-CPU event timers. */
 	cpu_initclocks_ap();
@@ -238,10 +243,19 @@
 
 			ret = platform_smp_start_cpu(pc);
 			if (ret == 0) {
+                                if (bootverbose) // HACK!!!
+                                        printf("cpu_mp_unleash attempting to wait for pc_awake: CPU %d, hwref=%jx, awake=%x\n",
+                                            pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+                                            pc->pc_awake);
+
 				timeout = 2000;	/* wait 2sec for the AP */
 				while (!pc->pc_awake && --timeout > 0)
 					DELAY(1000);
 			}
+                        if (bootverbose) // HACK!!!
+                                printf("cpu_mp_unleash after platform_smp_start_cpu and waiting: CPU %d, hwref=%jx, awake=%x\n",
+                                    pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+                                    pc->pc_awake);
 		} else {
 			pc->pc_awake = 1;
 		}
@@ -252,7 +266,13 @@
 				    pc->pc_awake);
 			smp_cpus++;
 		} else
+                { // HACK!!!
+                        if (bootverbose) // HACK!!!
+                                printf("cpu_mp_unleash adding pc_cpuid to stopped_cpus: CPU %d (dev=%x)\n",
+                                    pc->pc_cpuid, (int)pc->pc_hwref);
 			CPU_SET(pc->pc_cpuid, &stopped_cpus);
+                } // HACK!!!
+
 	}
 
 	ap_awake = 1;


===
Mark Millard
marklmi at yahoo.com
( dsl-only.net went
away in early 2018-Mar)