svn commit: r338300 - in stable/11/sys/amd64: amd64 include

[Konstantin Belousov]

Author: kib
Date: Fri Aug 24 18:13:45 2018
New Revision: 338300
URL: https://svnweb.freebsd.org/changeset/base/338300

Log:
  MFC r337773, r337838, r338112, r338202:
  Fixes for early EFIRT usage on amd64.
  
  Activate vmspace0 pmap, not kernel_pmap.  Use special version of the
  pmap activation code, pmap_activate_boot().  Ensure that PCID state for
  the vmspace0 pmap is valid early.

Modified:
  stable/11/sys/amd64/amd64/mp_machdep.c
  stable/11/sys/amd64/amd64/pmap.c
  stable/11/sys/amd64/include/pmap.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- stable/11/sys/amd64/amd64/mp_machdep.c	Fri Aug 24 18:09:23 2018	(r338299)
+++ stable/11/sys/amd64/amd64/mp_machdep.c	Fri Aug 24 18:13:45 2018	(r338300)
@@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_param.h>
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
+#include <vm/vm_map.h>
 #include <vm/vm_extern.h>
 
 #include <x86/apicreg.h>
@@ -263,9 +264,9 @@ init_secondary(void)
 	pc->pc_gs32p = &gdt[NGDT * cpu + GUGS32_SEL];
 	pc->pc_ldt = (struct system_segment_descriptor *)&gdt[NGDT * cpu +
 	    GUSERLDT_SEL];
-	pc->pc_curpmap = kernel_pmap;
+	/* See comment in pmap_bootstrap(). */
+	pc->pc_pcid_next = PMAP_PCID_KERN + 2;
 	pc->pc_pcid_gen = 1;
-	pc->pc_pcid_next = PMAP_PCID_KERN + 1;
 	common_tss[cpu].tss_rsp0 = 0;
 
 	/* Save the per-cpu pointer for use by the NMI handler. */
@@ -308,6 +309,7 @@ init_secondary(void)
 	while (atomic_load_acq_int(&aps_ready) == 0)
 		ia32_pause();
 
+	pmap_activate_boot(vmspace_pmap(proc0.p_vmspace));
 	init_secondary_tail();
 }
 

Modified: stable/11/sys/amd64/amd64/pmap.c
==============================================================================
--- stable/11/sys/amd64/amd64/pmap.c	Fri Aug 24 18:09:23 2018	(r338299)
+++ stable/11/sys/amd64/amd64/pmap.c	Fri Aug 24 18:13:45 2018	(r338300)
@@ -1101,9 +1101,17 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
 			kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN;
 			kernel_pmap->pm_pcids[i].pm_gen = 1;
 		}
-		__pcpu[0].pc_pcid_next = PMAP_PCID_KERN + 1;
-		__pcpu[0].pc_pcid_gen = 1;
+
 		/*
+		 * PMAP_PCID_KERN + 1 is used for initialization of
+		 * proc0 pmap.  The pmap' pcid state might be used by
+		 * EFIRT entry before first context switch, so it
+		 * needs to be valid.
+		 */
+		PCPU_SET(pcid_next, PMAP_PCID_KERN + 2);
+		PCPU_SET(pcid_gen, 1);
+
+		/*
 		 * pcpu area for APs is zeroed during AP startup.
 		 * pc_pcid_next and pc_pcid_gen are initialized by AP
 		 * during pcpu setup.
@@ -2570,16 +2578,10 @@ pmap_pinit0(pmap_t pmap)
 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 	pmap->pm_flags = pmap_flags;
 	CPU_FOREACH(i) {
-		pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;
-		pmap->pm_pcids[i].pm_gen = 0;
-		if (!pti) {
-			__pcpu[i].pc_kcr3 = PMAP_NO_CR3;
-			__pcpu[i].pc_ucr3 = PMAP_NO_CR3;
-		}
+		pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN + 1;
+		pmap->pm_pcids[i].pm_gen = 1;
 	}
-	PCPU_SET(curpmap, kernel_pmap);
-	pmap_activate(curthread);
-	CPU_FILL(&kernel_pmap->pm_active);
+	pmap_activate_boot(pmap);
 }
 
 void
@@ -7475,7 +7477,7 @@ pmap_activate_sw(struct thread *td)
 			intr_restore(rflags);
 		if (cached)
 			PCPU_INC(pm_save_cnt);
-	} else if (cr3 != pmap->pm_cr3) {
+	} else {
 		load_cr3(pmap->pm_cr3);
 		PCPU_SET(curpmap, pmap);
 		if (pti) {
@@ -7503,6 +7505,32 @@ pmap_activate(struct thread *td)
 	critical_enter();
 	pmap_activate_sw(td);
 	critical_exit();
+}
+
+void
+pmap_activate_boot(pmap_t pmap)
+{
+	uint64_t kcr3;
+	u_int cpuid;
+
+	/*
+	 * kernel_pmap must be never deactivated, and we ensure that
+	 * by never activating it at all.
+	 */
+	MPASS(pmap != kernel_pmap);
+
+	cpuid = PCPU_GET(cpuid);
+#ifdef SMP
+	CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
+#else
+	CPU_SET(cpuid, &pmap->pm_active);
+#endif
+	PCPU_SET(curpmap, pmap);
+	kcr3 = pmap->pm_cr3;
+	if (pmap_pcid_enabled)
+		kcr3 |= pmap->pm_pcids[cpuid].pm_pcid | CR3_PCID_SAVE;
+	PCPU_SET(kcr3, kcr3);
+	PCPU_SET(ucr3, PMAP_NO_CR3);
 }
 
 void

Modified: stable/11/sys/amd64/include/pmap.h
==============================================================================
--- stable/11/sys/amd64/include/pmap.h	Fri Aug 24 18:09:23 2018	(r338299)
+++ stable/11/sys/amd64/include/pmap.h	Fri Aug 24 18:13:45 2018	(r338300)
@@ -405,6 +405,7 @@ extern int invpcid_works;
 
 struct thread;
 
+void	pmap_activate_boot(pmap_t pmap);
 void	pmap_activate_sw(struct thread *);
 void	pmap_bootstrap(vm_paddr_t *);
 int	pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde);