svn commit: r334282 - stable/11/sys/amd64/amd64

[Konstantin Belousov]

Author: kib
Date: Mon May 28 10:55:09 2018
New Revision: 334282
URL: https://svnweb.freebsd.org/changeset/base/334282

Log:
  MFC r333990, r333992:
  Add missed barrier for pm_gen/pm_active interaction.
  
  Approved by:	re (marius)

Modified:
  stable/11/sys/amd64/amd64/pmap.c
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/amd64/amd64/pmap.c
==============================================================================
--- stable/11/sys/amd64/amd64/pmap.c	Mon May 28 10:54:24 2018	(r334281)
+++ stable/11/sys/amd64/amd64/pmap.c	Mon May 28 10:55:09 2018	(r334282)
@@ -1645,6 +1645,18 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 				if (cpuid != i)
 					pmap->pm_pcids[i].pm_gen = 0;
 			}
+
+			/*
+			 * The fence is between stores to pm_gen and the read of
+			 * the pm_active mask.  We need to ensure that it is
+			 * impossible for us to miss the bit update in pm_active
+			 * and simultaneously observe a non-zero pm_gen in
+			 * pmap_activate_sw(), otherwise TLB update is missed.
+			 * Without the fence, IA32 allows such an outcome.
+			 * Note that pm_active is updated by a locked operation,
+			 * which provides the reciprocal fence.
+			 */
+			atomic_thread_fence_seq_cst();
 		}
 		mask = &pmap->pm_active;
 	}
@@ -1716,6 +1728,8 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm
 				if (cpuid != i)
 					pmap->pm_pcids[i].pm_gen = 0;
 			}
+			/* See the comment in pmap_invalidate_page(). */
+			atomic_thread_fence_seq_cst();
 		}
 		mask = &pmap->pm_active;
 	}
@@ -1787,6 +1801,8 @@ pmap_invalidate_all(pmap_t pmap)
 				if (cpuid != i)
 					pmap->pm_pcids[i].pm_gen = 0;
 			}
+			/* See the comment in pmap_invalidate_page(). */
+			atomic_thread_fence_seq_cst();
 		}
 		mask = &pmap->pm_active;
 	}