svn commit: r232852 - head/sys/sys

[Alexander Motin]

Author: mav
Date: Mon Mar 12 07:02:16 2012
New Revision: 232852
URL: http://svn.freebsd.org/changeset/base/232852

Log:
  Tune cpuset macros to optimize cases when CPU_SETSIZE fits into single
  machine word. For example, it turns CPU_SET() into expected shift and OR,
  removing two extra shifts and additional index on memory access.
  
  Generated code checked for kernel (optimized) and user-level (unoptimized)
  cases with GCC and CLANG.
  
  Reviewed by:	attilio
  MFC after:	2 weeks

Modified:
  head/sys/sys/cpuset.h

Modified: head/sys/sys/cpuset.h
==============================================================================
--- head/sys/sys/cpuset.h	Mon Mar 12 05:28:02 2012	(r232851)
+++ head/sys/sys/cpuset.h	Mon Mar 12 07:02:16 2012	(r232852)
@@ -36,11 +36,18 @@
 
 #define	CPUSETBUFSIZ	((2 + sizeof(long) * 2) * _NCPUWORDS)
 
-#define	__cpuset_mask(n)	((long)1 << ((n) % _NCPUBITS))
-#define	CPU_CLR(n, p)	((p)->__bits[(n)/_NCPUBITS] &= ~__cpuset_mask(n))
+/*
+ * Macros addressing word and bit within it, tuned to make compiler
+ * optimize cases when CPU_SETSIZE fits into single machine word.
+ */
+#define	__cpuset_mask(n)				\
+	((long)1 << ((_NCPUWORDS == 1) ? (__size_t)(n) : ((n) % _NCPUBITS)))
+#define	__cpuset_word(n)	((_NCPUWORDS == 1) ? 0 : ((n) / _NCPUBITS))
+
+#define	CPU_CLR(n, p)	((p)->__bits[__cpuset_word(n)] &= ~__cpuset_mask(n))
 #define	CPU_COPY(f, t)	(void)(*(t) = *(f))
-#define	CPU_ISSET(n, p)	(((p)->__bits[(n)/_NCPUBITS] & __cpuset_mask(n)) != 0)
-#define	CPU_SET(n, p)	((p)->__bits[(n)/_NCPUBITS] |= __cpuset_mask(n))
+#define	CPU_ISSET(n, p)	(((p)->__bits[__cpuset_word(n)] & __cpuset_mask(n)) != 0)
+#define	CPU_SET(n, p)	((p)->__bits[__cpuset_word(n)] |= __cpuset_mask(n))
 #define	CPU_ZERO(p) do {				\
 	__size_t __i;					\
 	for (__i = 0; __i < _NCPUWORDS; __i++)		\
@@ -55,7 +62,7 @@
 
 #define	CPU_SETOF(n, p) do {					\
 	CPU_ZERO(p);						\
-	((p)->__bits[(n)/_NCPUBITS] = __cpuset_mask(n));	\
+	((p)->__bits[__cpuset_word(n)] = __cpuset_mask(n));	\
 } while (0)
 
 /* Is p empty. */
@@ -126,10 +133,10 @@
 } while (0)
 
 #define	CPU_CLR_ATOMIC(n, p)						\
-	atomic_clear_long(&(p)->__bits[(n)/_NCPUBITS], __cpuset_mask(n))
+	atomic_clear_long(&(p)->__bits[__cpuset_word(n)], __cpuset_mask(n))
 
 #define	CPU_SET_ATOMIC(n, p)						\
-	atomic_set_long(&(p)->__bits[(n)/_NCPUBITS], __cpuset_mask(n))
+	atomic_set_long(&(p)->__bits[__cpuset_word(n)], __cpuset_mask(n))
 
 /* Convenience functions catering special cases. */ 
 #define	CPU_OR_ATOMIC(d, s) do {			\