svn commit: r329381 - in stable/11/sys: sys vm

[Mateusz Guzik]

Author: mjg
Date: Fri Feb 16 16:16:33 2018
New Revision: 329381
URL: https://svnweb.freebsd.org/changeset/base/329381

Log:
  MFC r324610:
  
      Reduce traffic on vm_cnt.v_free_count
  
      The variable is modified with the highly contended page free queue lock.
      It unnecessarily shares a cacheline with purely read-only fields and is
      re-read after the lock is dropped in the page allocation code making the
      hold time longer.
  
      Pad the variable just like the others and store the value as found with
      the lock held instead of re-reading.
  
      Provides a modest 1%-ish speed up in concurrent page faults.
  
  Due to KBI constraints the field is not moved in this commit, only re-read is
  avoided.

Modified:
  stable/11/sys/sys/vmmeter.h
  stable/11/sys/vm/vm_page.c
  stable/11/sys/vm/vm_phys.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/sys/vmmeter.h
==============================================================================
--- stable/11/sys/sys/vmmeter.h	Fri Feb 16 16:07:58 2018	(r329380)
+++ stable/11/sys/sys/vmmeter.h	Fri Feb 16 16:16:33 2018	(r329381)
@@ -175,10 +175,10 @@ vm_paging_target(void)
  * Returns TRUE if the pagedaemon needs to be woken up.
  */
 static inline int
-vm_paging_needed(void)
+vm_paging_needed(u_int free_count)
 {
 
-	return (vm_cnt.v_free_count < vm_pageout_wakeup_thresh);
+	return (free_count < vm_pageout_wakeup_thresh);
 }
 
 /*

Modified: stable/11/sys/vm/vm_page.c
==============================================================================
--- stable/11/sys/vm/vm_page.c	Fri Feb 16 16:07:58 2018	(r329380)
+++ stable/11/sys/vm/vm_page.c	Fri Feb 16 16:16:33 2018	(r329381)
@@ -1601,6 +1601,7 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pi
 {
 	vm_page_t m;
 	int flags, req_class;
+	u_int free_count;
 
 	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
 	    (object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&
@@ -1672,7 +1673,7 @@ again:
 	 *  At this point we had better have found a good page.
 	 */
 	KASSERT(m != NULL, ("missing page"));
-	vm_phys_freecnt_adj(m, -1);
+	free_count = vm_phys_freecnt_adj(m, -1);
 	if ((m->flags & PG_ZERO) != 0)
 		vm_page_zero_count--;
 	mtx_unlock(&vm_page_queue_free_mtx);
@@ -1737,7 +1738,7 @@ again:
 	 * Don't wakeup too often - wakeup the pageout daemon when
 	 * we would be nearly out of memory.
 	 */
-	if (vm_paging_needed())
+	if (vm_paging_needed(free_count))
 		pagedaemon_wakeup();
 
 	return (m);
@@ -1933,7 +1934,7 @@ retry:
 			pmap_page_set_memattr(m, memattr);
 		pindex++;
 	}
-	if (vm_paging_needed())
+	if (vm_paging_needed(vm_cnt.v_free_count))
 		pagedaemon_wakeup();
 	return (m_ret);
 }
@@ -1982,7 +1983,7 @@ vm_page_t
 vm_page_alloc_freelist(int flind, int req)
 {
 	vm_page_t m;
-	u_int flags;
+	u_int flags, free_count;
 	int req_class;
 
 	req_class = req & VM_ALLOC_CLASS_MASK;
@@ -2013,7 +2014,7 @@ again:
 		mtx_unlock(&vm_page_queue_free_mtx);
 		return (NULL);
 	}
-	vm_phys_freecnt_adj(m, -1);
+	free_count = vm_phys_freecnt_adj(m, -1);
 	if ((m->flags & PG_ZERO) != 0)
 		vm_page_zero_count--;
 	mtx_unlock(&vm_page_queue_free_mtx);
@@ -2037,7 +2038,7 @@ again:
 	}
 	/* Unmanaged pages don't use "act_count". */
 	m->oflags = VPO_UNMANAGED;
-	if (vm_paging_needed())
+	if (vm_paging_needed(free_count))
 		pagedaemon_wakeup();
 	return (m);
 }

Modified: stable/11/sys/vm/vm_phys.h
==============================================================================
--- stable/11/sys/vm/vm_phys.h	Fri Feb 16 16:07:58 2018	(r329380)
+++ stable/11/sys/vm/vm_phys.h	Fri Feb 16 16:16:33 2018	(r329381)
@@ -112,13 +112,13 @@ vm_phys_domain(vm_page_t m)
 #endif
 }
 
-static inline void
+static inline u_int
 vm_phys_freecnt_adj(vm_page_t m, int adj)
 {
 
 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
-	vm_cnt.v_free_count += adj;
 	vm_phys_domain(m)->vmd_free_count += adj;
+	return (vm_cnt.v_free_count += adj);
 }
 
 #endif	/* _KERNEL */