svn commit: r195365 - projects/mips/sys/mips/include
Warner Losh
imp at FreeBSD.org
Sun Jul 5 15:13:25 UTC 2009
Author: imp
Date: Sun Jul 5 15:13:24 2009
New Revision: 195365
URL: http://svn.freebsd.org/changeset/base/195365
Log:
First cut at atomics for 64-bit machines and SMP machines.
# Note: Cavium provided a port that has atomics similar to these, but
# that does a syncw; sync; atomic; sync; syncw where we just do the classic
# mips 'atomic' operation (eg ll; frob; sc). It is unclear to me why
# the extra is needed. Since my initial target is one core, I'll defer
# investigation until I bring up multiple cores. syncw is an octeon specific
# instruction.
Modified:
projects/mips/sys/mips/include/atomic.h
Modified: projects/mips/sys/mips/include/atomic.h
==============================================================================
--- projects/mips/sys/mips/include/atomic.h Sun Jul 5 15:10:07 2009 (r195364)
+++ projects/mips/sys/mips/include/atomic.h Sun Jul 5 15:13:24 2009 (r195365)
@@ -34,6 +34,17 @@
#error this file needs sys/cdefs.h as a prerequisite
#endif
+/*
+ * Note: All the 64-bit atomic operations are only atomic when running
+ * in 64-bit mode. It is assumed that code compiled for n32 and n64
+ * fits into this definition and no further safeties are needed.
+ *
+ * It is also assumed that the add, subtract and other arithmetic is
+ * done on numbers not pointers. The special rules for n32 pointers
+ * do not have atomic operations defined for them, but generally shouldn't
+ * need atomic operations.
+ */
+
static __inline void
mips_sync(void)
{
@@ -166,6 +177,110 @@ atomic_readandset_32(__volatile uint32_t
return result;
}
+#if defined(__mips_n64) || defined(__mips_n32)
+static __inline void
+atomic_set_64(__volatile uint64_t *p, uint64_t v)
+{
+ uint64_t temp;
+
+ __asm __volatile (
+ "1:\n\t"
+ "lld %0, %3\n\t" /* load old value */
+ "or %0, %2, %0\n\t" /* calculate new value */
+ "scd %0, %1\n\t" /* attempt to store */
+ "beqz %0, 1b\n\t" /* spin if failed */
+ : "=&r" (temp), "=m" (*p)
+ : "r" (v), "m" (*p)
+ : "memory");
+
+}
+
+static __inline void
+atomic_clear_64(__volatile uint64_t *p, uint64_t v)
+{
+ uint64_t temp;
+ v = ~v;
+
+ __asm __volatile (
+ "1:\n\t"
+ "lld %0, %3\n\t" /* load old value */
+ "and %0, %2, %0\n\t" /* calculate new value */
+ "scd %0, %1\n\t" /* attempt to store */
+ "beqz %0, 1b\n\t" /* spin if failed */
+ : "=&r" (temp), "=m" (*p)
+ : "r" (v), "m" (*p)
+ : "memory");
+}
+
+static __inline void
+atomic_add_64(__volatile uint64_t *p, uint64_t v)
+{
+ uint64_t temp;
+
+ __asm __volatile (
+ "1:\n\t"
+ "lld %0, %3\n\t" /* load old value */
+ "addu %0, %2, %0\n\t" /* calculate new value */
+ "scd %0, %1\n\t" /* attempt to store */
+ "beqz %0, 1b\n\t" /* spin if failed */
+ : "=&r" (temp), "=m" (*p)
+ : "r" (v), "m" (*p)
+ : "memory");
+}
+
+static __inline void
+atomic_subtract_64(__volatile uint64_t *p, uint64_t v)
+{
+ uint64_t temp;
+
+ __asm __volatile (
+ "1:\n\t"
+ "lld %0, %3\n\t" /* load old value */
+ "subu %0, %2\n\t" /* calculate new value */
+ "scd %0, %1\n\t" /* attempt to store */
+ "beqz %0, 1b\n\t" /* spin if failed */
+ : "=&r" (temp), "=m" (*p)
+ : "r" (v), "m" (*p)
+ : "memory");
+}
+
+static __inline uint64_t
+atomic_readandclear_64(__volatile uint64_t *addr)
+{
+ uint64_t result,temp;
+
+ __asm __volatile (
+ "1:\n\t"
+ "lld %0, %3\n\t" /* load old value */
+ "li %1, 0\n\t" /* value to store */
+ "scd %1, %2\n\t" /* attempt to store */
+ "beqz %1, 1b\n\t" /* if the store failed, spin */
+ : "=&r"(result), "=&r"(temp), "=m" (*addr)
+ : "m" (*addr)
+ : "memory");
+
+ return result;
+}
+
+static __inline uint64_t
+atomic_readandset_64(__volatile uint64_t *addr, uint64_t value)
+{
+ uint64_t result,temp;
+
+ __asm __volatile (
+ "1:\n\t"
+ "lld %0,%3\n\t" /* Load old value*/
+ "or %1,$0,%4\n\t"
+ "scd %1,%2\n\t" /* attempt to store */
+ "beqz %1, 1b\n\t" /* if the store failed, spin */
+ : "=&r"(result), "=&r"(temp), "=m" (*addr)
+ : "m" (*addr), "r" (value)
+ : "memory");
+
+ return result;
+}
+#endif
+
#define ATOMIC_ACQ_REL(NAME, WIDTH) \
static __inline void \
atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\
@@ -194,7 +309,7 @@ ATOMIC_ACQ_REL(set, 32)
ATOMIC_ACQ_REL(clear, 32)
ATOMIC_ACQ_REL(add, 32)
ATOMIC_ACQ_REL(subtract, 32)
-#if 0
+#if defined(__mips_n64) || defined(__mips_n32)
ATOMIC_ACQ_REL(set, 64)
ATOMIC_ACQ_REL(clear, 64)
ATOMIC_ACQ_REL(add, 64)
@@ -226,8 +341,23 @@ atomic_store_rel_##WIDTH(__volatile uint
ATOMIC_STORE_LOAD(32)
ATOMIC_STORE_LOAD(64)
-void atomic_store_64 (__volatile uint64_t *, uint64_t *);
-void atomic_load_64 (__volatile uint64_t *, uint64_t *);
+#if !defined(__mips_n64) && !defined(__mips_n32)
+void atomic_store_64(__volatile uint64_t *, uint64_t *);
+void atomic_load_64(__volatile uint64_t *, uint64_t *);
+#else
+static __inline void
+atomic_store_64(__volatile uint64_t *p, uint64_t v)
+{
+
+ *p = v;
+}
+
+static __inline void
+atomic_load_64(__volatile uint64_t *p, uint64_t *v)
+{
+ *v = *p;
+}
+#endif
#undef ATOMIC_STORE_LOAD
@@ -299,6 +429,78 @@ atomic_fetchadd_32(__volatile uint32_t *
return (value);
}
+#if defined(__mips_n64) || defined(__mips_n32)
+/*
+ * Atomically compare the value stored at *p with cmpval and if the
+ * two values are equal, update the value of *p with newval. Returns
+ * zero if the compare failed, nonzero otherwise.
+ */
+static __inline uint64_t
+atomic_cmpset_64(__volatile uint64_t* p, uint64_t cmpval, uint64_t newval)
+{
+ uint64_t ret;
+
+ __asm __volatile (
+ "1:\n\t"
+ "lld %0, %4\n\t" /* load old value */
+ "bne %0, %2, 2f\n\t" /* compare */
+ "move %0, %3\n\t" /* value to store */
+ "scd %0, %1\n\t" /* attempt to store */
+ "beqz %0, 1b\n\t" /* if it failed, spin */
+ "j 3f\n\t"
+ "2:\n\t"
+ "li %0, 0\n\t"
+ "3:\n"
+ : "=&r" (ret), "=m" (*p)
+ : "r" (cmpval), "r" (newval), "m" (*p)
+ : "memory");
+
+ return ret;
+}
+
+/*
+ * Atomically compare the value stored at *p with cmpval and if the
+ * two values are equal, update the value of *p with newval. Returns
+ * zero if the compare failed, nonzero otherwise.
+ */
+static __inline uint64_t
+atomic_cmpset_acq_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
+{
+ int retval;
+
+ retval = atomic_cmpset_64(p, cmpval, newval);
+ mips_sync();
+ return (retval);
+}
+
+static __inline uint64_t
+atomic_cmpset_rel_64(__volatile uint64_t *p, uint64_t cmpval, uint64_t newval)
+{
+ mips_sync();
+ return (atomic_cmpset_64(p, cmpval, newval));
+}
+
+/*
+ * Atomically add the value of v to the integer pointed to by p and return
+ * the previous value of *p.
+ */
+static __inline uint64_t
+atomic_fetchadd_64(__volatile uint64_t *p, uint64_t v)
+{
+ uint64_t value, temp;
+
+ __asm __volatile (
+ "1:\n\t"
+ "lld %0, %1\n\t" /* load old value */
+ "addu %2, %3, %0\n\t" /* calculate new value */
+ "scd %2, %1\n\t" /* attempt to store */
+ "beqz %2, 1b\n\t" /* spin if failed */
+ : "=&r" (value), "=m" (*p), "=&r" (temp)
+ : "r" (v), "m" (*p));
+ return (value);
+}
+#endif
+
/* Operations on chars. */
#define atomic_set_char atomic_set_8
#define atomic_set_acq_char atomic_set_acq_8
@@ -349,7 +551,13 @@ atomic_fetchadd_32(__volatile uint32_t *
#define atomic_readandset_int atomic_readandset_32
#define atomic_fetchadd_int atomic_fetchadd_32
-#ifdef __mips64
+/*
+ * I think the following is right, even for n32. For n32 the pointers
+ * are still 32-bits, so we need to operate on them as 32-bit quantities,
+ * even though they are sign extended in operation. For longs, there's
+ * no question because they are always 32-bits.
+ */
+#ifdef __mips_n64
/* Operations on longs. */
#define atomic_set_long atomic_set_64
#define atomic_set_acq_long atomic_set_acq_64
@@ -371,27 +579,7 @@ atomic_fetchadd_32(__volatile uint32_t *
#define atomic_fetchadd_long atomic_fetchadd_64
#define atomic_readandclear_long atomic_readandclear_64
-/* Operations on pointers. */
-#define atomic_set_ptr atomic_set_64
-#define atomic_set_acq_ptr atomic_set_acq_64
-#define atomic_set_rel_ptr atomic_set_rel_64
-#define atomic_clear_ptr atomic_clear_64
-#define atomic_clear_acq_ptr atomic_clear_acq_64
-#define atomic_clear_rel_ptr atomic_clear_rel_64
-#define atomic_add_ptr atomic_add_64
-#define atomic_add_acq_ptr atomic_add_acq_64
-#define atomic_add_rel_ptr atomic_add_rel_64
-#define atomic_subtract_ptr atomic_subtract_64
-#define atomic_subtract_acq_ptr atomic_subtract_acq_64
-#define atomic_subtract_rel_ptr atomic_subtract_rel_64
-#define atomic_cmpset_ptr atomic_cmpset_64
-#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64
-#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64
-#define atomic_load_acq_ptr atomic_load_acq_64
-#define atomic_store_rel_ptr atomic_store_rel_64
-#define atomic_readandclear_ptr atomic_readandclear_64
-
-#else /* __mips64 */
+#else /* !__mips_n64 */
/* Operations on longs. */
#define atomic_set_long atomic_set_32
@@ -421,27 +609,26 @@ atomic_fetchadd_32(__volatile uint32_t *
atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v))
#define atomic_readandclear_long atomic_readandclear_32
+#endif /* __mips_n64 */
+
/* Operations on pointers. */
-#define atomic_set_ptr atomic_set_32
-#define atomic_set_acq_ptr atomic_set_acq_32
-#define atomic_set_rel_ptr atomic_set_rel_32
-#define atomic_clear_ptr atomic_clear_32
-#define atomic_clear_acq_ptr atomic_clear_acq_32
-#define atomic_clear_rel_ptr atomic_clear_rel_32
-#define atomic_add_ptr atomic_add_32
-#define atomic_add_acq_ptr atomic_add_acq_32
-#define atomic_add_rel_ptr atomic_add_rel_32
-#define atomic_subtract_ptr atomic_subtract_32
-#define atomic_subtract_acq_ptr atomic_subtract_acq_32
-#define atomic_subtract_rel_ptr atomic_subtract_rel_32
-#define atomic_cmpset_ptr atomic_cmpset_32
-#define atomic_cmpset_acq_ptr(dst, old, new) \
- atomic_cmpset_acq_32((volatile u_int *)(dst), \
- (u_int)(old), (u_int)(new))
-#define atomic_cmpset_rel_ptr atomic_cmpset_rel_32
-#define atomic_load_acq_ptr atomic_load_acq_32
-#define atomic_store_rel_ptr atomic_store_rel_32
-#define atomic_readandclear_ptr atomic_readandclear_32
-#endif /* __mips64 */
+#define atomic_set_ptr atomic_set_long
+#define atomic_set_acq_ptr atomic_set_acq_long
+#define atomic_set_rel_ptr atomic_set_rel_long
+#define atomic_clear_ptr atomic_clear_long
+#define atomic_clear_acq_ptr atomic_clear_acq_long
+#define atomic_clear_rel_ptr atomic_clear_rel_long
+#define atomic_add_ptr atomic_add_long
+#define atomic_add_acq_ptr atomic_add_acq_long
+#define atomic_add_rel_ptr atomic_add_rel_long
+#define atomic_subtract_ptr atomic_subtract_long
+#define atomic_subtract_acq_ptr atomic_subtract_acq_long
+#define atomic_subtract_rel_ptr atomic_subtract_rel_long
+#define atomic_cmpset_ptr atomic_cmpset_long
+#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
+#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
+#define atomic_load_acq_ptr atomic_load_acq_long
+#define atomic_store_rel_ptr atomic_store_rel_long
+#define atomic_readandclear_ptr atomic_readandclear_long
#endif /* ! _MACHINE_ATOMIC_H_ */
More information about the svn-src-projects
mailing list