svn commit: r327195 - in stable/11/sys: amd64/include arm/include arm64/include i386/include mips/include mips/mips powerpc/include riscv/include sparc64/include sys
Konstantin Belousov
kib at FreeBSD.org
Tue Dec 26 10:07:20 UTC 2017
Author: kib
Date: Tue Dec 26 10:07:17 2017
New Revision: 327195
URL: https://svnweb.freebsd.org/changeset/base/327195
Log:
MFC r326971, r327047 (by ian), r327053 (by marius), r327074, r327097:
Add atomic_load(9) and atomic_store(9) operations.
Added:
stable/11/sys/sys/atomic_common.h
- copied unchanged from r326971, head/sys/sys/atomic_common.h
Modified:
stable/11/sys/amd64/include/atomic.h
stable/11/sys/arm/include/atomic.h
stable/11/sys/arm64/include/atomic.h
stable/11/sys/i386/include/atomic.h
stable/11/sys/mips/include/atomic.h
stable/11/sys/mips/mips/db_interface.c
stable/11/sys/mips/mips/support.S
stable/11/sys/powerpc/include/atomic.h
stable/11/sys/riscv/include/atomic.h
stable/11/sys/sparc64/include/atomic.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/amd64/include/atomic.h
==============================================================================
--- stable/11/sys/amd64/include/atomic.h Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/amd64/include/atomic.h Tue Dec 26 10:07:17 2017 (r327195)
@@ -55,6 +55,8 @@
#define wmb() __asm __volatile("sfence;" : : : "memory")
#define rmb() __asm __volatile("lfence;" : : : "memory")
+#include <sys/atomic_common.h>
+
/*
* Various simple operations on memory, each of which is atomic in the
* presence of interrupts and multiple processors.
Modified: stable/11/sys/arm/include/atomic.h
==============================================================================
--- stable/11/sys/arm/include/atomic.h Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/arm/include/atomic.h Tue Dec 26 10:07:17 2017 (r327195)
@@ -39,6 +39,8 @@
#ifndef _MACHINE_ATOMIC_H_
#define _MACHINE_ATOMIC_H_
+#include <sys/atomic_common.h>
+
#include <machine/armreg.h>
#ifndef _KERNEL
@@ -51,32 +53,6 @@
#include <machine/atomic-v4.h>
#endif /* Arch >= v6 */
-static __inline int
-atomic_load_32(volatile uint32_t *v)
-{
-
- return (*v);
-}
-
-static __inline void
-atomic_store_32(volatile uint32_t *dst, uint32_t src)
-{
- *dst = src;
-}
-
-static __inline int
-atomic_load_long(volatile u_long *v)
-{
-
- return (*v);
-}
-
-static __inline void
-atomic_store_long(volatile u_long *dst, u_long src)
-{
- *dst = src;
-}
-
#define atomic_clear_ptr atomic_clear_32
#define atomic_clear_acq_ptr atomic_clear_acq_32
#define atomic_clear_rel_ptr atomic_clear_rel_32
@@ -90,7 +66,6 @@ atomic_store_long(volatile u_long *dst, u_long src)
#define atomic_cmpset_acq_ptr atomic_cmpset_acq_32
#define atomic_cmpset_rel_ptr atomic_cmpset_rel_32
#define atomic_load_acq_ptr atomic_load_acq_32
-#define atomic_store_ptr atomic_store_32
#define atomic_store_rel_ptr atomic_store_rel_32
#define atomic_swap_ptr atomic_swap_32
#define atomic_readandclear_ptr atomic_readandclear_32
Modified: stable/11/sys/arm64/include/atomic.h
==============================================================================
--- stable/11/sys/arm64/include/atomic.h Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/arm64/include/atomic.h Tue Dec 26 10:07:17 2017 (r327195)
@@ -29,6 +29,8 @@
#ifndef _MACHINE_ATOMIC_H_
#define _MACHINE_ATOMIC_H_
+#include <sys/atomic_common.h>
+
#define isb() __asm __volatile("isb" : : : "memory")
/*
Modified: stable/11/sys/i386/include/atomic.h
==============================================================================
--- stable/11/sys/i386/include/atomic.h Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/i386/include/atomic.h Tue Dec 26 10:07:17 2017 (r327195)
@@ -32,6 +32,8 @@
#error this file needs sys/cdefs.h as a prerequisite
#endif
+#include <sys/atomic_common.h>
+
#ifdef _KERNEL
#include <machine/md_var.h>
#include <machine/specialreg.h>
Modified: stable/11/sys/mips/include/atomic.h
==============================================================================
--- stable/11/sys/mips/include/atomic.h Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/mips/include/atomic.h Tue Dec 26 10:07:17 2017 (r327195)
@@ -34,6 +34,8 @@
#error this file needs sys/cdefs.h as a prerequisite
#endif
+#include <sys/atomic_common.h>
+
/*
* Note: All the 64-bit atomic operations are only atomic when running
* in 64-bit mode. It is assumed that code compiled for n32 and n64
@@ -337,23 +339,6 @@ atomic_store_rel_##WIDTH(__volatile uint##WIDTH##_t *p
ATOMIC_STORE_LOAD(32)
ATOMIC_STORE_LOAD(64)
-#if !defined(__mips_n64) && !defined(__mips_n32)
-void atomic_store_64(__volatile uint64_t *, uint64_t *);
-void atomic_load_64(__volatile uint64_t *, uint64_t *);
-#else
-static __inline void
-atomic_store_64(__volatile uint64_t *p, uint64_t *v)
-{
- *p = *v;
-}
-
-static __inline void
-atomic_load_64(__volatile uint64_t *p, uint64_t *v)
-{
- *v = *p;
-}
-#endif
-
#undef ATOMIC_STORE_LOAD
/*
Modified: stable/11/sys/mips/mips/db_interface.c
==============================================================================
--- stable/11/sys/mips/mips/db_interface.c Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/mips/mips/db_interface.c Tue Dec 26 10:07:17 2017 (r327195)
@@ -150,6 +150,7 @@ db_read_bytes(vm_offset_t addr, size_t size, char *dat
/*
* 'addr' could be a memory-mapped I/O address. Try to
* do atomic load/store in unit of size requested.
+ * size == 8 is only atomic on 64bit or n32 kernel.
*/
if ((size == 2 || size == 4 || size == 8) &&
((addr & (size -1)) == 0) &&
@@ -162,9 +163,8 @@ db_read_bytes(vm_offset_t addr, size_t size, char *dat
*(uint32_t *)data = *(uint32_t *)addr;
break;
case 8:
- atomic_load_64((volatile u_int64_t *)addr,
- (u_int64_t *)data);
- break;
+ *(uint64_t *)data = *(uint64_t *)addr;
+ break;
}
} else {
char *src;
@@ -193,6 +193,7 @@ db_write_bytes(vm_offset_t addr, size_t size, char *da
/*
* 'addr' could be a memory-mapped I/O address. Try to
* do atomic load/store in unit of size requested.
+ * size == 8 is only atomic on 64bit or n32 kernel.
*/
if ((size == 2 || size == 4 || size == 8) &&
((addr & (size -1)) == 0) &&
@@ -205,9 +206,8 @@ db_write_bytes(vm_offset_t addr, size_t size, char *da
*(uint32_t *)addr = *(uint32_t *)data;
break;
case 8:
- atomic_store_64((volatile u_int64_t *)addr,
- (u_int64_t *)data);
- break;
+ *(uint64_t *)addr = *(uint64_t *)data;
+ break;
}
} else {
char *dst;
Modified: stable/11/sys/mips/mips/support.S
==============================================================================
--- stable/11/sys/mips/mips/support.S Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/mips/mips/support.S Tue Dec 26 10:07:17 2017 (r327195)
@@ -839,74 +839,7 @@ LEAF(atomic_subtract_8)
nop
END(atomic_subtract_8)
-/*
- * atomic 64-bit register read/write assembly language support routines.
- */
-
.set noreorder # Noreorder is default style!
-
-#if !defined(__mips_n64) && !defined(__mips_n32)
- /*
- * I don't know if these routines have the right number of
- * NOPs in it for all processors. XXX
- *
- * Maybe it would be better to just leave this undefined in that case.
- *
- * XXX These routines are not safe in the case of a TLB miss on a1 or
- * a0 unless the trapframe is 64-bit, which it just isn't with O32.
- * If we take any exception, not just an interrupt, the upper
- * 32-bits will be clobbered. Use only N32 and N64 kernels if you
- * want to use 64-bit registers while interrupts are enabled or
- * with memory operations. Since this isn't even using load-linked
- * and store-conditional, perhaps it should just use two registers
- * instead, as is right and good with the O32 ABI.
- */
-LEAF(atomic_store_64)
- mfc0 t1, MIPS_COP_0_STATUS
- and t2, t1, ~MIPS_SR_INT_IE
- mtc0 t2, MIPS_COP_0_STATUS
- nop
- nop
- nop
- nop
- ld t0, (a1)
- nop
- nop
- sd t0, (a0)
- nop
- nop
- mtc0 t1,MIPS_COP_0_STATUS
- nop
- nop
- nop
- nop
- j ra
- nop
-END(atomic_store_64)
-
-LEAF(atomic_load_64)
- mfc0 t1, MIPS_COP_0_STATUS
- and t2, t1, ~MIPS_SR_INT_IE
- mtc0 t2, MIPS_COP_0_STATUS
- nop
- nop
- nop
- nop
- ld t0, (a0)
- nop
- nop
- sd t0, (a1)
- nop
- nop
- mtc0 t1,MIPS_COP_0_STATUS
- nop
- nop
- nop
- nop
- j ra
- nop
-END(atomic_load_64)
-#endif
#if defined(DDB) || defined(DEBUG)
Modified: stable/11/sys/powerpc/include/atomic.h
==============================================================================
--- stable/11/sys/powerpc/include/atomic.h Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/powerpc/include/atomic.h Tue Dec 26 10:07:17 2017 (r327195)
@@ -36,6 +36,8 @@
#error this file needs sys/cdefs.h as a prerequisite
#endif
+#include <sys/atomic_common.h>
+
/*
* The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
* with the atomic lXarx/stXcx. sequences below. They are not exposed outside
Modified: stable/11/sys/riscv/include/atomic.h
==============================================================================
--- stable/11/sys/riscv/include/atomic.h Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/riscv/include/atomic.h Tue Dec 26 10:07:17 2017 (r327195)
@@ -37,6 +37,8 @@
#ifndef _MACHINE_ATOMIC_H_
#define _MACHINE_ATOMIC_H_
+#include <sys/atomic_common.h>
+
#define fence() __asm __volatile("fence" ::: "memory");
#define mb() fence()
#define rmb() fence()
Modified: stable/11/sys/sparc64/include/atomic.h
==============================================================================
--- stable/11/sys/sparc64/include/atomic.h Tue Dec 26 09:57:15 2017 (r327194)
+++ stable/11/sys/sparc64/include/atomic.h Tue Dec 26 10:07:17 2017 (r327195)
@@ -37,6 +37,8 @@
#define wmb() mb()
#define rmb() mb()
+#include <sys/atomic_common.h>
+
/* Userland needs different ASI's. */
#ifdef _KERNEL
#define __ASI_ATOMIC ASI_N
@@ -253,11 +255,6 @@ atomic_fcmpset_rel_ ## name(volatile ptype p, vtype *e
return (0); \
} \
\
-static __inline vtype \
-atomic_load_ ## name(volatile ptype p) \
-{ \
- return ((vtype)atomic_cas((p), 0, 0, sz)); \
-} \
static __inline vtype \
atomic_load_acq_ ## name(volatile ptype p) \
{ \
Copied: stable/11/sys/sys/atomic_common.h (from r326971, head/sys/sys/atomic_common.h)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ stable/11/sys/sys/atomic_common.h Tue Dec 26 10:07:17 2017 (r327195, copy of r326971, head/sys/sys/atomic_common.h)
@@ -0,0 +1,73 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2017 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Konstantin Belousov <kib at FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+#ifndef _SYS_ATOMIC_COMMON_H_
+#define _SYS_ATOMIC_COMMON_H_
+
+#ifndef _MACHINE_ATOMIC_H_
+#error do not include this header, use machine/atomic.h
+#endif
+
+#define atomic_load_char(p) (*(volatile u_char *)(p))
+#define atomic_load_short(p) (*(volatile u_short *)(p))
+#define atomic_load_int(p) (*(volatile u_int *)(p))
+#define atomic_load_long(p) (*(volatile u_long *)(p))
+#define atomic_load_ptr(p) (*(volatile uintptr_t*)(p))
+#define atomic_load_8(p) (*(volatile uint8_t *)(p))
+#define atomic_load_16(p) (*(volatile uint16_t *)(p))
+#define atomic_load_32(p) (*(volatile uint32_t *)(p))
+#ifdef _LP64
+#define atomic_load_64(p) (*(volatile uint64_t *)(p))
+#endif
+
+#define atomic_store_char(p, v) \
+ (*(volatile u_char *)(p) = (u_char)(v))
+#define atomic_store_short(p, v) \
+ (*(volatile u_short *)(p) = (u_short)(v))
+#define atomic_store_int(p, v) \
+ (*(volatile u_int *)(p) = (u_int)(v))
+#define atomic_store_long(p, v) \
+ (*(volatile u_long *)(p) = (u_long)(v))
+#define atomic_store_ptr(p, v) \
+ (*(uintptr_t *)(p) = (uintptr_t)(v))
+#define atomic_store_8(p, v) \
+ (*(volatile uint8_t *)(p) = (uint8_t)(v))
+#define atomic_store_16(p, v) \
+ (*(volatile uint16_t *)(p) = (uint16_t)(v))
+#define atomic_store_32(p, v) \
+ (*(volatile uint32_t *)(p) = (uint32_t)(v))
+#ifdef _LP64
+#define atomic_store_64(p, v) \
+ (*(volatile uint64_t *)(p) = (uint64_t)(v))
+#endif
+
+#endif
More information about the svn-src-stable
mailing list