PERFORCE change 78683 for review
John Baldwin
jhb at FreeBSD.org
Fri Jun 17 19:55:49 GMT 2005
http://perforce.freebsd.org/chv.cgi?CH=78683
Change 78683 by jhb at jhb_slimer on 2005/06/17 19:55:36
Backout the atomic patches for now so I can break it up into
more palatable chunks.
Affected files ...
.. //depot/projects/smpng/sys/alpha/include/atomic.h#21 edit
.. //depot/projects/smpng/sys/amd64/include/atomic.h#15 edit
.. //depot/projects/smpng/sys/amd64/include/bus.h#7 edit
.. //depot/projects/smpng/sys/arm/include/atomic.h#11 edit
.. //depot/projects/smpng/sys/arm/include/endian.h#11 edit
.. //depot/projects/smpng/sys/dev/hatm/if_hatm_intr.c#12 edit
.. //depot/projects/smpng/sys/i386/include/atomic.h#32 edit
.. //depot/projects/smpng/sys/i386/include/bus.h#8 edit
.. //depot/projects/smpng/sys/ia64/include/atomic.h#9 edit
.. //depot/projects/smpng/sys/kern/kern_mutex.c#99 edit
.. //depot/projects/smpng/sys/pc98/include/bus.h#4 edit
.. //depot/projects/smpng/sys/powerpc/include/atomic.h#12 edit
.. //depot/projects/smpng/sys/sparc64/include/atomic.h#12 edit
.. //depot/projects/smpng/sys/sparc64/include/cpufunc.h#24 edit
.. //depot/projects/smpng/sys/sys/mutex.h#56 edit
Differences ...
==== //depot/projects/smpng/sys/alpha/include/atomic.h#21 (text+ko) ====
@@ -56,12 +56,12 @@
#ifdef __GNUCLIKE_ASM
__asm __volatile (
- "1:\tldl_l %0, %1\n\t" /* load old value */
- "bis %0, %2, %0\n\t" /* calculate new value */
+ "1:\tldl_l %0, %2\n\t" /* load old value */
+ "bis %0, %3, %0\n\t" /* calculate new value */
"stl_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n" /* spin if failed */
: "=&r" (temp), "=m" (*p)
- : "r" (v), "m" (*p)
+ : "m" (*p), "r" (v)
: "memory");
#endif
}
@@ -76,8 +76,8 @@
"bic %0, %2, %0\n\t" /* calculate new value */
"stl_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n" /* spin if failed */
- : "=&r" (temp), "=m" (*p)
- : "r" (v), "m" (*p)
+ : "=&r" (temp), "+m" (*p)
+ : "r" (v)
: "memory");
#endif
}
@@ -92,8 +92,8 @@
"addl %0, %2, %0\n\t" /* calculate new value */
"stl_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n" /* spin if failed */
- : "=&r" (temp), "=m" (*p)
- : "r" (v), "m" (*p)
+ : "=&r" (temp), "+m" (*p)
+ : "r" (v)
: "memory");
#endif
}
@@ -108,8 +108,8 @@
"subl %0, %2, %0\n\t" /* calculate new value */
"stl_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n" /* spin if failed */
- : "=&r" (temp), "=m" (*p)
- : "r" (v), "m" (*p)
+ : "=&r" (temp), "+m" (*p)
+ : "r" (v)
: "memory");
#endif
}
@@ -125,8 +125,8 @@
"ldiq %1,0\n\t" /* value to store */
"stl_c %1,%2\n\t" /* attempt to store */
"beq %1,1b\n" /* if the store failed, spin */
- : "=&r"(result), "=&r"(temp), "=m" (*addr)
- : "m" (*addr)
+ : "=&r"(result), "=&r"(temp), "+m" (*addr)
+ :
: "memory");
#endif
@@ -143,8 +143,8 @@
"bis %0, %2, %0\n\t" /* calculate new value */
"stq_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n" /* spin if failed */
- : "=&r" (temp), "=m" (*p)
- : "r" (v), "m" (*p)
+ : "=&r" (temp), "+m" (*p)
+ : "r" (v)
: "memory");
#endif
}
@@ -159,8 +159,8 @@
"bic %0, %2, %0\n\t" /* calculate new value */
"stq_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n" /* spin if failed */
- : "=&r" (temp), "=m" (*p)
- : "r" (v), "m" (*p)
+ : "=&r" (temp), "+m" (*p)
+ : "r" (v)
: "memory");
#endif
}
@@ -175,8 +175,8 @@
"addq %0, %2, %0\n\t" /* calculate new value */
"stq_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n" /* spin if failed */
- : "=&r" (temp), "=m" (*p)
- : "r" (v), "m" (*p)
+ : "=&r" (temp), "+m" (*p)
+ : "r" (v)
: "memory");
#endif
}
@@ -191,8 +191,8 @@
"subq %0, %2, %0\n\t" /* calculate new value */
"stq_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n" /* spin if failed */
- : "=&r" (temp), "=m" (*p)
- : "r" (v), "m" (*p)
+ : "=&r" (temp), "+m" (*p)
+ : "r" (v)
: "memory");
#endif
}
@@ -208,14 +208,36 @@
"ldiq %1,0\n\t" /* value to store */
"stq_c %1,%2\n\t" /* attempt to store */
"beq %1,1b\n" /* if the store failed, spin */
- : "=&r"(result), "=&r"(temp), "=m" (*addr)
- : "m" (*addr)
+ : "=&r"(result), "=&r"(temp), "+m" (*addr)
+ :
: "memory");
#endif
return result;
}
+#define atomic_set_char atomic_set_8
+#define atomic_clear_char atomic_clear_8
+#define atomic_add_char atomic_add_8
+#define atomic_subtract_char atomic_subtract_8
+
+#define atomic_set_short atomic_set_16
+#define atomic_clear_short atomic_clear_16
+#define atomic_add_short atomic_add_16
+#define atomic_subtract_short atomic_subtract_16
+
+#define atomic_set_int atomic_set_32
+#define atomic_clear_int atomic_clear_32
+#define atomic_add_int atomic_add_32
+#define atomic_subtract_int atomic_subtract_32
+#define atomic_readandclear_int atomic_readandclear_32
+
+#define atomic_set_long atomic_set_64
+#define atomic_clear_long atomic_clear_64
+#define atomic_add_long atomic_add_64
+#define atomic_subtract_long atomic_subtract_64
+#define atomic_readandclear_long atomic_readandclear_64
+
#define ATOMIC_ACQ_REL(NAME, WIDTH, TYPE) \
static __inline void \
atomic_##NAME##_acq_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\
@@ -227,11 +249,24 @@
static __inline void \
atomic_##NAME##_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\
{ \
- alpha_wmb(); \
+ alpha_mb(); \
+ atomic_##NAME##_##WIDTH(p, v); \
+} \
+ \
+static __inline void \
+atomic_##NAME##_acq_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\
+{ \
+ atomic_##NAME##_##WIDTH(p, v); \
+ alpha_mb(); \
+} \
+ \
+static __inline void \
+atomic_##NAME##_rel_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\
+{ \
+ alpha_mb(); \
atomic_##NAME##_##WIDTH(p, v); \
}
-/* Variants of simple arithmetic with memory barriers. */
ATOMIC_ACQ_REL(set, 8, char)
ATOMIC_ACQ_REL(clear, 8, char)
ATOMIC_ACQ_REL(add, 8, char)
@@ -254,11 +289,27 @@
/*
* We assume that a = b will do atomic loads and stores.
*/
-#define ATOMIC_STORE_LOAD(WIDTH) \
-static __inline u_int##WIDTH##_t \
-atomic_load_acq_##WIDTH(volatile u_int##WIDTH##_t *p) \
+#define ATOMIC_STORE_LOAD(TYPE, WIDTH) \
+static __inline u_##TYPE \
+atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \
+{ \
+ u_##TYPE v; \
+ \
+ v = *p; \
+ alpha_mb(); \
+ return (v); \
+} \
+ \
+static __inline void \
+atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v)\
+{ \
+ alpha_mb(); \
+ *p = v; \
+} \
+static __inline u_##TYPE \
+atomic_load_acq_##TYPE(volatile u_##TYPE *p) \
{ \
- u_int##WIDTH##_t v; \
+ u_##TYPE v; \
\
v = *p; \
alpha_mb(); \
@@ -266,14 +317,16 @@
} \
\
static __inline void \
-atomic_store_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\
+atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
{ \
- alpha_wmb(); \
+ alpha_mb(); \
*p = v; \
}
-ATOMIC_STORE_LOAD(32)
-ATOMIC_STORE_LOAD(64)
+ATOMIC_STORE_LOAD(char, 8)
+ATOMIC_STORE_LOAD(short, 16)
+ATOMIC_STORE_LOAD(int, 32)
+ATOMIC_STORE_LOAD(long, 64)
#undef ATOMIC_STORE_LOAD
@@ -296,8 +349,8 @@
"stl_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n\t" /* if it failed, spin */
"2:\n"
- : "=&r" (ret), "=m" (*p)
- : "r" ((long)(int)cmpval), "r" (newval), "m" (*p)
+ : "=&r" (ret), "+m" (*p)
+ : "r" ((long)(int)cmpval), "r" (newval)
: "memory");
#endif
@@ -323,14 +376,25 @@
"stq_c %0, %1\n\t" /* attempt to store */
"beq %0, 1b\n\t" /* if it failed, spin */
"2:\n"
- : "=&r" (ret), "=m" (*p)
- : "r" (cmpval), "r" (newval), "m" (*p)
+ : "=&r" (ret), "+m" (*p)
+ : "r" (cmpval), "r" (newval)
: "memory");
#endif
return ret;
}
+#define atomic_cmpset_int atomic_cmpset_32
+#define atomic_cmpset_long atomic_cmpset_64
+
+static __inline int
+atomic_cmpset_ptr(volatile void *dst, void *exp, void *src)
+{
+
+ return (atomic_cmpset_long((volatile u_long *)dst, (u_long)exp,
+ (u_long)src));
+}
+
static __inline u_int32_t
atomic_cmpset_acq_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval)
{
@@ -344,7 +408,7 @@
static __inline u_int32_t
atomic_cmpset_rel_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval)
{
- alpha_wmb();
+ alpha_mb();
return (atomic_cmpset_32(p, cmpval, newval));
}
@@ -361,96 +425,67 @@
static __inline u_int64_t
atomic_cmpset_rel_64(volatile u_int64_t *p, u_int64_t cmpval, u_int64_t newval)
{
- alpha_wmb();
+ alpha_mb();
return (atomic_cmpset_64(p, cmpval, newval));
}
-/* Operations on chars. */
-#define atomic_set_char atomic_set_8
-#define atomic_set_acq_char atomic_set_acq_8
-#define atomic_set_rel_char atomic_set_rel_8
-#define atomic_clear_char atomic_clear_8
-#define atomic_clear_acq_char atomic_clear_acq_8
-#define atomic_clear_rel_char atomic_clear_rel_8
-#define atomic_add_char atomic_add_8
-#define atomic_add_acq_char atomic_add_acq_8
-#define atomic_add_rel_char atomic_add_rel_8
-#define atomic_subtract_char atomic_subtract_8
-#define atomic_subtract_acq_char atomic_subtract_acq_8
-#define atomic_subtract_rel_char atomic_subtract_rel_8
-
-/* Operations on shorts. */
-#define atomic_set_short atomic_set_16
-#define atomic_set_acq_short atomic_set_acq_16
-#define atomic_set_rel_short atomic_set_rel_16
-#define atomic_clear_short atomic_clear_16
-#define atomic_clear_acq_short atomic_clear_acq_16
-#define atomic_clear_rel_short atomic_clear_rel_16
-#define atomic_add_short atomic_add_16
-#define atomic_add_acq_short atomic_add_acq_16
-#define atomic_add_rel_short atomic_add_rel_16
-#define atomic_subtract_short atomic_subtract_16
-#define atomic_subtract_acq_short atomic_subtract_acq_16
-#define atomic_subtract_rel_short atomic_subtract_rel_16
-
-/* Operations on ints. */
-#define atomic_set_int atomic_set_32
-#define atomic_set_acq_int atomic_set_acq_32
-#define atomic_set_rel_int atomic_set_rel_32
-#define atomic_clear_int atomic_clear_32
-#define atomic_clear_acq_int atomic_clear_acq_32
-#define atomic_clear_rel_int atomic_clear_rel_32
-#define atomic_add_int atomic_add_32
-#define atomic_add_acq_int atomic_add_acq_32
-#define atomic_add_rel_int atomic_add_rel_32
-#define atomic_subtract_int atomic_subtract_32
-#define atomic_subtract_acq_int atomic_subtract_acq_32
-#define atomic_subtract_rel_int atomic_subtract_rel_32
-#define atomic_cmpset_int atomic_cmpset_32
#define atomic_cmpset_acq_int atomic_cmpset_acq_32
#define atomic_cmpset_rel_int atomic_cmpset_rel_32
-#define atomic_load_acq_int atomic_load_acq_32
-#define atomic_store_rel_int atomic_store_rel_32
-#define atomic_readandclear_int atomic_readandclear_32
-
-/* Operations on longs. */
-#define atomic_set_long atomic_set_64
-#define atomic_set_acq_long atomic_set_acq_64
-#define atomic_set_rel_long atomic_set_rel_64
-#define atomic_clear_long atomic_clear_64
-#define atomic_clear_acq_long atomic_clear_acq_64
-#define atomic_clear_rel_long atomic_clear_rel_64
-#define atomic_add_long atomic_add_64
-#define atomic_add_acq_long atomic_add_acq_64
-#define atomic_add_rel_long atomic_add_rel_64
-#define atomic_subtract_long atomic_subtract_64
-#define atomic_subtract_acq_long atomic_subtract_acq_64
-#define atomic_subtract_rel_long atomic_subtract_rel_64
-#define atomic_cmpset_long atomic_cmpset_64
#define atomic_cmpset_acq_long atomic_cmpset_acq_64
#define atomic_cmpset_rel_long atomic_cmpset_rel_64
-#define atomic_load_acq_long atomic_load_acq_64
-#define atomic_store_rel_long atomic_store_rel_64
-#define atomic_readandclear_long atomic_readandclear_64
+
+static __inline int
+atomic_cmpset_acq_ptr(volatile void *dst, void *exp, void *src)
+{
+
+ return (atomic_cmpset_acq_long((volatile u_long *)dst, (u_long)exp,
+ (u_long)src));
+}
+
+static __inline int
+atomic_cmpset_rel_ptr(volatile void *dst, void *exp, void *src)
+{
+
+ return (atomic_cmpset_rel_long((volatile u_long *)dst, (u_long)exp,
+ (u_long)src));
+}
+
+static __inline void *
+atomic_load_acq_ptr(volatile void *p)
+{
+ return (void *)atomic_load_acq_long((volatile u_long *)p);
+}
+
+static __inline void
+atomic_store_rel_ptr(volatile void *p, void *v)
+{
+ atomic_store_rel_long((volatile u_long *)p, (u_long)v);
+}
+
+#define ATOMIC_PTR(NAME) \
+static __inline void \
+atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \
+{ \
+ atomic_##NAME##_long((volatile u_long *)p, v); \
+} \
+ \
+static __inline void \
+atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \
+{ \
+ atomic_##NAME##_acq_long((volatile u_long *)p, v);\
+} \
+ \
+static __inline void \
+atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \
+{ \
+ atomic_##NAME##_rel_long((volatile u_long *)p, v);\
+}
+
+ATOMIC_PTR(set)
+ATOMIC_PTR(clear)
+ATOMIC_PTR(add)
+ATOMIC_PTR(subtract)
-/* Operations on pointers. */
-#define atomic_set_ptr atomic_set_64
-#define atomic_set_acq_ptr atomic_set_acq_64
-#define atomic_set_rel_ptr atomic_set_rel_64
-#define atomic_clear_ptr atomic_clear_64
-#define atomic_clear_acq_ptr atomic_clear_acq_64
-#define atomic_clear_rel_ptr atomic_clear_rel_64
-#define atomic_add_ptr atomic_add_64
-#define atomic_add_acq_ptr atomic_add_acq_64
-#define atomic_add_rel_ptr atomic_add_rel_64
-#define atomic_subtract_ptr atomic_subtract_64
-#define atomic_subtract_acq_ptr atomic_subtract_acq_64
-#define atomic_subtract_rel_ptr atomic_subtract_rel_64
-#define atomic_cmpset_ptr atomic_cmpset_64
-#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
-#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
-#define atomic_load_acq_ptr atomic_load_acq_64
-#define atomic_store_rel_ptr atomic_store_rel_64
-#define atomic_readandclear_ptr atomic_readandclear_64
+#undef ATOMIC_PTR
#endif /* ! _MACHINE_ATOMIC_H_ */
==== //depot/projects/smpng/sys/amd64/include/atomic.h#15 (text+ko) ====
@@ -68,15 +68,13 @@
* This allows kernel modules to be portable between UP and SMP systems.
*/
#if defined(KLD_MODULE)
-#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
-void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \
-void atomic_##NAME##_##TYPE_acq(volatile u_##TYPE *p, u_##TYPE v); \
-void atomic_##NAME##_##TYPE_rel(volatile u_##TYPE *p, u_##TYPE v)
+#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
+void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src);
int atomic_cmpset_long(volatile u_long *dst, u_long exp, u_long src);
-#define ATOMIC_STORE_LOAD(TYPE) \
+#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \
u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p); \
void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
@@ -89,13 +87,9 @@
* the binaries will run on both types of systems.
*/
#if defined(SMP) || !defined(_KERNEL)
-#define MPLOCKED "lock ; "
-#define MPLFENCE "lfence ; "
-#define MPSFENCE "sfence ; "
+#define MPLOCKED lock ;
#else
-#define MPLOCKED ""
-#define MPLFENCE ""
-#define MPSFENCE ""
+#define MPLOCKED
#endif
/*
@@ -106,32 +100,16 @@
static __inline void \
atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
{ \
- __asm __volatile(MPLOCKED OP \
- : "=m" (*p) \
- : CONS (V), "m" (*p)); \
-} \
- \
-static __inline void \
-atomic_##NAME##_##TYPE##_acq(volatile u_##TYPE *p, u_##TYPE v)\
-{ \
- atomic_##NAME_##TYPE(p, v); \
- __asm __volatile(MPLFENCE ::: "memory"); \
+ __asm __volatile(__XSTRING(MPLOCKED) OP \
+ : "+m" (*p) \
+ : CONS (V)); \
} \
- \
-static __inline void \
-atomic_##NAME##_##TYPE##_rel(volatile u_##TYPE *p, u_##TYPE v)\
-{ \
- __asm __volatile(MPSFENCE); \
- atomic_##NAME_##TYPE(p, v); \
-} \
struct __hack
#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \
-extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \
-extern void atomic_##NAME##_##TYPE##_acq(volatile u_##TYPE *p, u_##TYPE v); \
-extern void atomic_##NAME##_##TYPE##_rel(volatile u_##TYPE *p, u_##TYPE v)
+extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
@@ -151,17 +129,16 @@
int res = exp;
__asm __volatile (
- " " MPLOCKED " "
- " cmpxchgl %2,%1 ; "
+ " " __XSTRING(MPLOCKED) " "
+ " cmpxchgl %1,%2 ; "
" setz %%al ; "
" movzbl %%al,%0 ; "
"1: "
"# atomic_cmpset_int"
- : "+a" (res), /* 0 (result) */
- "=m" (*dst) /* 1 */
- : "r" (src), /* 2 */
- "m" (*dst) /* 3 */
- : "memory");
+ : "+a" (res) /* 0 (result) */
+ : "r" (src), /* 1 */
+ "m" (*(dst)) /* 2 */
+ : "memory");
return (res);
}
@@ -172,56 +149,47 @@
long res = exp;
__asm __volatile (
- " " MPLOCKED " "
- " cmpxchgq %2,%1 ; "
+ " " __XSTRING(MPLOCKED) " "
+ " cmpxchgq %1,%2 ; "
" setz %%al ; "
" movzbq %%al,%0 ; "
"1: "
"# atomic_cmpset_long"
- : "+a" (res), /* 0 (result) */
- "=m" (*dst) /* 1 */
- : "r" (src), /* 2 */
- "m" (*dst) /* 3 */
- : "memory");
+ : "+a" (res) /* 0 (result) */
+ : "r" (src), /* 1 */
+ "m" (*(dst)) /* 2 */
+ : "memory");
return (res);
}
+#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
-#define ATOMIC_CMPSET(TYPE) \
-static __inline int \
-atomic_cmpset_acq_##TYPE(volatile u_##TYPE *dst, u_##TYPE exp, u_##TYPE src)\
-{ \
- int retval; \
- \
- retval = atomic_cmpset_##TYPE(dst, exp, src); \
- __asm __volatile(MPLFENCE ::: "memory"); \
- return (retval); \
-} \
- \
-static __inline int \
-atomic_cmpset_rel_##TYPE(volatile u_##TYPE *dst, u_##TYPE exp, u_##TYPE src)\
-{ \
- __asm __volatile(MPSFENCE); \
- return (atomic_cmpset_##TYPE(dst, exp, src)); \
-} \
-struct __hack
+#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
-#define ATOMIC_STORE_LOAD(TYPE) \
+#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \
static __inline u_##TYPE \
atomic_load_acq_##TYPE(volatile u_##TYPE *p) \
{ \
- u_##TYPE v; \
+ u_##TYPE res; \
+ \
+ __asm __volatile(__XSTRING(MPLOCKED) LOP \
+ : "=a" (res), /* 0 (result) */\
+ "+m" (*p) /* 1 */ \
+ : : "memory"); \
\
- v = *p; \
- __asm __volatile(MPLFENCE ::: "memory"); \
- return (v); \
+ return (res); \
} \
\
+/* \
+ * The XCHG instruction asserts LOCK automagically. \
+ */ \
static __inline void \
atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
{ \
- __asm __volatile(MPSFENCE); \
- *p = v; \
+ __asm __volatile(SOP \
+ : "+m" (*p), /* 0 */ \
+ "+r" (v) /* 1 */ \
+ : : "memory"); \
} \
struct __hack
@@ -230,13 +198,7 @@
extern int atomic_cmpset_int(volatile u_int *, u_int, u_int);
extern int atomic_cmpset_long(volatile u_long *, u_long, u_long);
-#define ATOMIC_CMPSET(TYPE) \
-extern int atomic_cmpset_acq_##TYPE(volatile u_##TYPE *dst, u_##TYPE exp,\
- u_##TYPE src); \
-extern int atomic_cmpset_rel_##TYPE(volatile u_##TYPE *dst, u_##TYPE exp,\
- u_##TYPE src)
-
-#define ATOMIC_STORE_LOAD(TYPE) \
+#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \
extern u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p); \
extern void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
@@ -264,17 +226,55 @@
ATOMIC_ASM(add, long, "addq %1,%0", "ir", v);
ATOMIC_ASM(subtract, long, "subq %1,%0", "ir", v);
-ATOMIC_CMPSET(int);
-ATOMIC_CMPSET(long);
-
-ATOMIC_STORE_LOAD(char);
-ATOMIC_STORE_LOAD(short);
-ATOMIC_STORE_LOAD(int);
-ATOMIC_STORE_LOAD(long);
+ATOMIC_STORE_LOAD(char, "cmpxchgb %b0,%1", "xchgb %b1,%0");
+ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0");
+ATOMIC_STORE_LOAD(int, "cmpxchgl %0,%1", "xchgl %1,%0");
+ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0");
#undef ATOMIC_ASM
#undef ATOMIC_STORE_LOAD
+#define atomic_set_acq_char atomic_set_char
+#define atomic_set_rel_char atomic_set_char
+#define atomic_clear_acq_char atomic_clear_char
+#define atomic_clear_rel_char atomic_clear_char
+#define atomic_add_acq_char atomic_add_char
+#define atomic_add_rel_char atomic_add_char
+#define atomic_subtract_acq_char atomic_subtract_char
+#define atomic_subtract_rel_char atomic_subtract_char
+
+#define atomic_set_acq_short atomic_set_short
+#define atomic_set_rel_short atomic_set_short
+#define atomic_clear_acq_short atomic_clear_short
+#define atomic_clear_rel_short atomic_clear_short
+#define atomic_add_acq_short atomic_add_short
+#define atomic_add_rel_short atomic_add_short
+#define atomic_subtract_acq_short atomic_subtract_short
+#define atomic_subtract_rel_short atomic_subtract_short
+
+#define atomic_set_acq_int atomic_set_int
+#define atomic_set_rel_int atomic_set_int
+#define atomic_clear_acq_int atomic_clear_int
+#define atomic_clear_rel_int atomic_clear_int
+#define atomic_add_acq_int atomic_add_int
+#define atomic_add_rel_int atomic_add_int
+#define atomic_subtract_acq_int atomic_subtract_int
+#define atomic_subtract_rel_int atomic_subtract_int
+#define atomic_cmpset_acq_int atomic_cmpset_int
+#define atomic_cmpset_rel_int atomic_cmpset_int
+
+#define atomic_set_acq_long atomic_set_long
+#define atomic_set_rel_long atomic_set_long
+#define atomic_clear_acq_long atomic_clear_long
+#define atomic_clear_rel_long atomic_clear_long
+#define atomic_add_acq_long atomic_add_long
+#define atomic_add_rel_long atomic_add_long
+#define atomic_subtract_acq_long atomic_subtract_long
+#define atomic_subtract_rel_long atomic_subtract_long
+
+#define atomic_cmpset_acq_ptr atomic_cmpset_ptr
+#define atomic_cmpset_rel_ptr atomic_cmpset_ptr
+
#define atomic_set_8 atomic_set_char
#define atomic_set_acq_8 atomic_set_acq_char
#define atomic_set_rel_8 atomic_set_rel_char
@@ -317,33 +317,63 @@
#define atomic_subtract_32 atomic_subtract_int
#define atomic_subtract_acq_32 atomic_subtract_acq_int
#define atomic_subtract_rel_32 atomic_subtract_rel_int
+#define atomic_load_acq_32 atomic_load_acq_int
+#define atomic_store_rel_32 atomic_store_rel_int
#define atomic_cmpset_32 atomic_cmpset_int
#define atomic_cmpset_acq_32 atomic_cmpset_acq_int
#define atomic_cmpset_rel_32 atomic_cmpset_rel_int
-#define atomic_load_acq_32 atomic_load_acq_int
-#define atomic_store_rel_32 atomic_store_rel_int
#define atomic_readandclear_32 atomic_readandclear_int
-#define atomic_set_ptr atomic_set_int
-#define atomic_set_acq_ptr atomic_set_acq_int
-#define atomic_set_rel_ptr atomic_set_rel_int
-#define atomic_clear_ptr atomic_clear_int
-#define atomic_clear_acq_ptr atomic_clear_acq_int
-#define atomic_clear_rel_ptr atomic_clear_rel_int
-#define atomic_add_ptr atomic_add_int
-#define atomic_add_acq_ptr atomic_add_acq_int
-#define atomic_add_rel_ptr atomic_add_rel_int
-#define atomic_subtract_ptr atomic_subtract_int
-#define atomic_subtract_acq_ptr atomic_subtract_acq_int
-#define atomic_subtract_rel_ptr atomic_subtract_rel_int
-#define atomic_cmpset_ptr atomic_cmpset_int
-#define atomic_cmpset_acq_ptr atomic_cmpset_acq_int
-#define atomic_cmpset_rel_ptr atomic_cmpset_rel_int
-#define atomic_load_acq_ptr atomic_load_acq_int
-#define atomic_store_rel_ptr atomic_store_rel_int
-#define atomic_readandclear_ptr atomic_readandclear_int
+#if !defined(WANT_FUNCTIONS)
+static __inline int
+atomic_cmpset_ptr(volatile void *dst, void *exp, void *src)
+{
+
+ return (atomic_cmpset_long((volatile u_long *)dst,
+ (u_long)exp, (u_long)src));
+}
+
+static __inline void *
+atomic_load_acq_ptr(volatile void *p)
+{
+ /*
+ * The apparently-bogus cast to intptr_t in the following is to
+ * avoid a warning from "gcc -Wbad-function-cast".
+ */
+ return ((void *)(intptr_t)atomic_load_acq_long((volatile u_long *)p));
+}
+
+static __inline void
+atomic_store_rel_ptr(volatile void *p, void *v)
+{
+ atomic_store_rel_long((volatile u_long *)p, (u_long)v);
+}
+
+#define ATOMIC_PTR(NAME) \
+static __inline void \
+atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \
+{ \
+ atomic_##NAME##_long((volatile u_long *)p, v); \
+} \
+ \
+static __inline void \
+atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \
+{ \
+ atomic_##NAME##_acq_long((volatile u_long *)p, v);\
+} \
+ \
+static __inline void \
+atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \
+{ \
+ atomic_##NAME##_rel_long((volatile u_long *)p, v);\
+}
+
+ATOMIC_PTR(set)
+ATOMIC_PTR(clear)
+ATOMIC_PTR(add)
+ATOMIC_PTR(subtract)
-#if !defined(WANT_FUNCTIONS)
+#undef ATOMIC_PTR
#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
@@ -356,9 +386,8 @@
" xorl %0,%0 ; "
" xchgl %1,%0 ; "
"# atomic_readandclear_int"
- : "=&r" (result), /* 0 (result) */
- "=m" (*addr) /* 1 (addr) */
- : "m" (*addr)); /* 2 (addr) */
+ : "=&r" (result) /* 0 (result) */
+ : "m" (*addr)); /* 1 (addr) */
return (result);
}
@@ -372,9 +401,8 @@
" xorq %0,%0 ; "
" xchgq %1,%0 ; "
"# atomic_readandclear_int"
- : "=&r" (result), /* 0 (result) */
- "=m" (*addr) /* 1 (addr) */
- : "m" (*addr)); /* 2 (addr) */
+ : "=&r" (result) /* 0 (result) */
+ : "m" (*addr)); /* 1 (addr) */
return (result);
}
==== //depot/projects/smpng/sys/amd64/include/bus.h#7 (text+ko) ====
@@ -278,8 +278,8 @@
1: movb (%2),%%al \n\
stosb \n\
loop 1b" :
- "+D" (addr), "+c" (count) :
- "r" (bsh + offset) :
+ "=D" (addr), "=c" (count) :
+ "r" (bsh + offset), "0" (addr), "1" (count) :
"%eax", "memory");
#endif
}
@@ -299,8 +299,8 @@
1: movw (%2),%%ax \n\
stosw \n\
loop 1b" :
- "+D" (addr), "+c" (count) :
- "r" (bsh + offset) :
+ "=D" (addr), "=c" (count) :
+ "r" (bsh + offset), "0" (addr), "1" (count) :
"%eax", "memory");
#endif
}
@@ -320,8 +320,8 @@
1: movl (%2),%%eax \n\
stosl \n\
loop 1b" :
- "+D" (addr), "+c" (count) :
- "r" (bsh + offset) :
+ "=D" (addr), "=c" (count) :
+ "r" (bsh + offset), "0" (addr), "1" (count) :
"%eax", "memory");
#endif
}
@@ -366,7 +366,8 @@
stosb \n\
incl %2 \n\
loop 1b" :
- "+D" (addr), "+c" (count), "+d" (_port_) ::
+ "=D" (addr), "=c" (count), "=d" (_port_) :
+ "0" (addr), "1" (count), "2" (_port_) :
"%eax", "memory", "cc");
#endif
} else {
@@ -376,7 +377,8 @@
cld \n\
repne \n\
movsb" :
- "+D" (addr), "+c" (count), "+S" (_port_) ::
+ "=D" (addr), "=c" (count), "=S" (_port_) :
+ "0" (addr), "1" (count), "2" (_port_) :
"memory", "cc");
#endif
}
@@ -396,7 +398,8 @@
stosw \n\
addl $2,%2 \n\
loop 1b" :
- "+D" (addr), "+c" (count), "+d" (_port_) ::
+ "=D" (addr), "=c" (count), "=d" (_port_) :
+ "0" (addr), "1" (count), "2" (_port_) :
"%eax", "memory", "cc");
#endif
} else {
@@ -406,7 +409,8 @@
cld \n\
repne \n\
movsw" :
- "+D" (addr), "+c" (count), "+S" (_port_) ::
+ "=D" (addr), "=c" (count), "=S" (_port_) :
+ "0" (addr), "1" (count), "2" (_port_) :
"memory", "cc");
#endif
}
@@ -426,7 +430,8 @@
stosl \n\
addl $4,%2 \n\
loop 1b" :
- "+D" (addr), "+c" (count), "+d" (_port_) ::
+ "=D" (addr), "=c" (count), "=d" (_port_) :
+ "0" (addr), "1" (count), "2" (_port_) :
"%eax", "memory", "cc");
#endif
} else {
@@ -436,7 +441,8 @@
cld \n\
repne \n\
movsl" :
- "+D" (addr), "+c" (count), "+S" (_port_) ::
+ "=D" (addr), "=c" (count), "=S" (_port_) :
+ "0" (addr), "1" (count), "2" (_port_) :
"memory", "cc");
#endif
}
@@ -536,8 +542,8 @@
1: lodsb \n\
movb %%al,(%2) \n\
loop 1b" :
- "+S" (addr), "+c" (count) :
- "r" (bsh + offset) :
+ "=S" (addr), "=c" (count) :
+ "r" (bsh + offset), "0" (addr), "1" (count) :
"%eax", "memory", "cc");
#endif
}
@@ -557,8 +563,8 @@
1: lodsw \n\
movw %%ax,(%2) \n\
loop 1b" :
- "+S" (addr), "+c" (count) :
- "r" (bsh + offset) :
+ "=S" (addr), "=c" (count) :
+ "r" (bsh + offset), "0" (addr), "1" (count) :
"%eax", "memory", "cc");
#endif
}
@@ -578,8 +584,8 @@
1: lodsl \n\
movl %%eax,(%2) \n\
loop 1b" :
- "+S" (addr), "+c" (count) :
- "r" (bsh + offset) :
+ "=S" (addr), "=c" (count) :
+ "r" (bsh + offset), "0" (addr), "1" (count) :
"%eax", "memory", "cc");
#endif
}
@@ -625,7 +631,8 @@
outb %%al,%w0 \n\
incl %0 \n\
loop 1b" :
- "+d" (_port_), "+S" (addr), "+c" (count) ::
+ "=d" (_port_), "=S" (addr), "=c" (count) :
+ "0" (_port_), "1" (addr), "2" (count) :
"%eax", "memory", "cc");
#endif
} else {
@@ -635,7 +642,8 @@
cld \n\
repne \n\
movsb" :
- "+D" (_port_), "+S" (addr), "+c" (count) ::
+ "=D" (_port_), "=S" (addr), "=c" (count) :
+ "0" (_port_), "1" (addr), "2" (count) :
"memory", "cc");
#endif
}
@@ -655,7 +663,8 @@
outw %%ax,%w0 \n\
addl $2,%0 \n\
loop 1b" :
- "+d" (_port_), "+S" (addr), "+c" (count) ::
+ "=d" (_port_), "=S" (addr), "=c" (count) :
+ "0" (_port_), "1" (addr), "2" (count) :
"%eax", "memory", "cc");
#endif
} else {
@@ -665,7 +674,8 @@
cld \n\
repne \n\
movsw" :
- "+D" (_port_), "+S" (addr), "+c" (count) ::
+ "=D" (_port_), "=S" (addr), "=c" (count) :
+ "0" (_port_), "1" (addr), "2" (count) :
"memory", "cc");
#endif
}
@@ -685,7 +695,8 @@
outl %%eax,%w0 \n\
addl $4,%0 \n\
loop 1b" :
- "+d" (_port_), "+S" (addr), "+c" (count) ::
+ "=d" (_port_), "=S" (addr), "=c" (count) :
+ "0" (_port_), "1" (addr), "2" (count) :
"%eax", "memory", "cc");
#endif
} else {
@@ -695,7 +706,8 @@
cld \n\
repne \n\
movsl" :
- "+D" (_port_), "+S" (addr), "+c" (count) ::
+ "=D" (_port_), "=S" (addr), "=c" (count) :
>>> TRUNCATED FOR MAIL (1000 lines) <<<
More information about the p4-projects
mailing list