svn commit: r334762 - in stable/11/sys: cddl/compat/opensolaris/kern cddl/compat/opensolaris/sys i386/include

[Hans Petter Selasky]

Author: hselasky
Date: Thu Jun  7 07:42:48 2018
New Revision: 334762
URL: https://svnweb.freebsd.org/changeset/base/334762

Log:
  MFC r334320 and r334328:
  
  Implement atomic_add_64() and atomic_subtract_64() for the i386 target.
  
  While at it add missing _acq_ and _rel_ variants for 64-bit atomic
  operations under i386.
  
  Reviewed by:	kib @
  Sponsored by:	Mellanox Technologies

Modified:
  stable/11/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c
  stable/11/sys/cddl/compat/opensolaris/sys/atomic.h
  stable/11/sys/i386/include/atomic.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c
==============================================================================
--- stable/11/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c	Thu Jun  7 07:38:37 2018	(r334761)
+++ stable/11/sys/cddl/compat/opensolaris/kern/opensolaris_atomic.c	Thu Jun  7 07:42:48 2018	(r334762)
@@ -52,7 +52,8 @@ atomic_init(void)
 }
 #endif
 
-#if !defined(__LP64__) && !defined(__mips_n32) && !defined(ARM_HAVE_ATOMIC64)
+#if !defined(__LP64__) && !defined(__mips_n32) && \
+    !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64)
 void
 atomic_add_64(volatile uint64_t *target, int64_t delta)
 {

Modified: stable/11/sys/cddl/compat/opensolaris/sys/atomic.h
==============================================================================
--- stable/11/sys/cddl/compat/opensolaris/sys/atomic.h	Thu Jun  7 07:38:37 2018	(r334761)
+++ stable/11/sys/cddl/compat/opensolaris/sys/atomic.h	Thu Jun  7 07:42:48 2018	(r334762)
@@ -36,7 +36,12 @@
 	atomic_cmpset_ptr((volatile uintptr_t *)(_a), (uintptr_t)(_b), (uintptr_t) (_c))
 #define cas32	atomic_cmpset_32
 
-#if !defined(__LP64__) && !defined(__mips_n32) && !defined(ARM_HAVE_ATOMIC64)
+#if defined(__i386__) && (defined(_KERNEL) || defined(KLD_MODULE))
+#define	I386_HAVE_ATOMIC64
+#endif
+
+#if !defined(__LP64__) && !defined(__mips_n32) && \
+    !defined(ARM_HAVE_ATOMIC64) && !defined(I386_HAVE_ATOMIC64)
 extern void atomic_add_64(volatile uint64_t *target, int64_t delta);
 extern void atomic_dec_64(volatile uint64_t *target);
 #endif
@@ -85,7 +90,8 @@ atomic_dec_32_nv(volatile uint32_t *target)
 	return (atomic_fetchadd_32(target, -1) - 1);
 }
 
-#if defined(__LP64__) || defined(__mips_n32) || defined(ARM_HAVE_ATOMIC64)
+#if defined(__LP64__) || defined(__mips_n32) || \
+    defined(ARM_HAVE_ATOMIC64) || defined(I386_HAVE_ATOMIC64)
 static __inline void
 atomic_dec_64(volatile uint64_t *target)
 {

Modified: stable/11/sys/i386/include/atomic.h
==============================================================================
--- stable/11/sys/i386/include/atomic.h	Thu Jun  7 07:38:37 2018	(r334761)
+++ stable/11/sys/i386/include/atomic.h	Thu Jun  7 07:42:48 2018	(r334762)
@@ -132,6 +132,8 @@ uint64_t	atomic_load_acq_64(volatile uint64_t *);
 void		atomic_store_rel_64(volatile uint64_t *, uint64_t);
 uint64_t	atomic_swap_64(volatile uint64_t *, uint64_t);
 uint64_t	atomic_fetchadd_64(volatile uint64_t *, uint64_t);
+void		atomic_add_64(volatile uint64_t *, uint64_t);
+void		atomic_subtract_64(volatile uint64_t *, uint64_t);
 
 #else /* !KLD_MODULE && __GNUCLIKE_ASM */
 
@@ -579,6 +581,30 @@ atomic_fetchadd_64(volatile uint64_t *p, uint64_t v)
 	}
 }
 
+static __inline void
+atomic_add_64(volatile uint64_t *p, uint64_t v)
+{
+	uint64_t t;
+
+	for (;;) {
+		t = *p;
+		if (atomic_cmpset_64(p, t, t + v))
+			break;
+	}
+}
+
+static __inline void
+atomic_subtract_64(volatile uint64_t *p, uint64_t v)
+{
+	uint64_t t;
+
+	for (;;) {
+		t = *p;
+		if (atomic_cmpset_64(p, t, t - v))
+			break;
+	}
+}
+
 #endif /* _KERNEL */
 
 #endif /* KLD_MODULE || !__GNUCLIKE_ASM */
@@ -802,6 +828,16 @@ u_long	atomic_swap_long(volatile u_long *p, u_long v);
 #define	atomic_fetchadd_32	atomic_fetchadd_int
 #define	atomic_testandset_32	atomic_testandset_int
 #define	atomic_testandclear_32	atomic_testandclear_int
+
+/* Operations on 64-bit quad words. */
+#define	atomic_cmpset_acq_64 atomic_cmpset_64
+#define	atomic_cmpset_rel_64 atomic_cmpset_64
+#define	atomic_fetchadd_acq_64	atomic_fetchadd_64
+#define	atomic_fetchadd_rel_64	atomic_fetchadd_64
+#define	atomic_add_acq_64 atomic_add_64
+#define	atomic_add_rel_64 atomic_add_64
+#define	atomic_subtract_acq_64 atomic_subtract_64
+#define	atomic_subtract_rel_64 atomic_subtract_64
 
 /* Operations on pointers. */
 #define	atomic_set_ptr(p, v) \