git: 81e2d24bc6ea - main - riscv: support for Cache-Block Operations (CBO).
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 22 Apr 2025 09:07:08 UTC
The branch main has been updated by br:
URL: https://cgit.FreeBSD.org/src/commit/?id=81e2d24bc6ea3edaa0338ea6020c2eb9f93de0ed
commit 81e2d24bc6ea3edaa0338ea6020c2eb9f93de0ed
Author: Ruslan Bukin <br@FreeBSD.org>
AuthorDate: 2025-04-22 08:14:02 +0000
Commit: Ruslan Bukin <br@FreeBSD.org>
CommitDate: 2025-04-22 09:03:45 +0000
riscv: support for Cache-Block Operations (CBO).
CBO represents a subset of Cache-Management Operations (CMO) spec.
While the CMO spec encompasses all operations on caches, the CBO subset
operates on cache blocks only.
Detect Zicbom, Zicboz and Zicbop extensions and provide cache invalidation
handlers based on Zicbom instructions.
Sponsored by: UKRI
Differential Revision: https://reviews.freebsd.org/D49852
---
sys/conf/files.riscv | 1 +
sys/riscv/include/cbo.h | 33 ++++++++++++++
sys/riscv/riscv/cbo.c | 104 +++++++++++++++++++++++++++++++++++++++++++++
sys/riscv/riscv/identcpu.c | 56 +++++++++++++++++++++++-
4 files changed, 192 insertions(+), 2 deletions(-)
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv
index 0d782239c89c..e77a15ce8dae 100644
--- a/sys/conf/files.riscv
+++ b/sys/conf/files.riscv
@@ -46,6 +46,7 @@ riscv/riscv/bus_space_asm.S standard
riscv/riscv/busdma_bounce.c standard
riscv/riscv/busdma_machdep.c standard
riscv/riscv/cache.c standard
+riscv/riscv/cbo.c standard
riscv/riscv/clock.c standard
riscv/riscv/copyinout.S standard
riscv/riscv/cpufunc_asm.S standard
diff --git a/sys/riscv/include/cbo.h b/sys/riscv/include/cbo.h
new file mode 100644
index 000000000000..cdf10069ba66
--- /dev/null
+++ b/sys/riscv/include/cbo.h
@@ -0,0 +1,33 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _RISCV_CBO_H_
+#define _RISCV_CBO_H_
+
+void cbo_zicbom_setup_cache(int cbom_block_size);
+
+#endif /* _RISCV_CBO_H_ */
diff --git a/sys/riscv/riscv/cbo.c b/sys/riscv/riscv/cbo.c
new file mode 100644
index 000000000000..9b8891c514af
--- /dev/null
+++ b/sys/riscv/riscv/cbo.c
@@ -0,0 +1,104 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* Cache Block Operations. */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <machine/cbo.h>
+
+static void
+cbo_zicbom_cpu_dcache_wbinv_range(vm_offset_t va, vm_size_t len)
+{
+ vm_offset_t addr;
+
+ /*
+ * A flush operation atomically performs a clean operation followed by
+ * an invalidate operation.
+ */
+
+ va &= ~(dcache_line_size - 1);
+ for (addr = va; addr < va + len; addr += dcache_line_size)
+ __asm __volatile(".option push; .option arch, +zicbom\n"
+ "cbo.flush (%0); .option pop\n" :: "r"(addr));
+}
+
+static void
+cbo_zicbom_cpu_dcache_inv_range(vm_offset_t va, vm_size_t len)
+{
+ vm_offset_t addr;
+
+ /*
+ * An invalidate operation makes data from store operations performed by
+ * a set of non-coherent agents visible to the set of coherent agents at
+ * a point common to both sets by deallocating all copies of a cache
+ * block from the set of coherent caches up to that point.
+ */
+
+ va &= ~(dcache_line_size - 1);
+ for (addr = va; addr < va + len; addr += dcache_line_size)
+ __asm __volatile(".option push; .option arch, +zicbom\n"
+ "cbo.inval (%0); .option pop\n" :: "r"(addr));
+}
+
+static void
+cbo_zicbom_cpu_dcache_wb_range(vm_offset_t va, vm_size_t len)
+{
+ vm_offset_t addr;
+
+ /*
+ * A clean operation makes data from store operations performed by the
+ * set of coherent agents visible to a set of non-coherent agents at a
+ * point common to both sets by performing a write transfer of a copy of
+ * a cache block to that point provided a coherent agent performed a
+ * store operation that modified the data in the cache block since the
+ * previous invalidate, clean, or flush operation on the cache block.
+ */
+
+ va &= ~(dcache_line_size - 1);
+ for (addr = va; addr < va + len; addr += dcache_line_size)
+ __asm __volatile(".option push; .option arch, +zicbom\n"
+ "cbo.clean (%0); .option pop\n" :: "r"(addr));
+}
+
+void
+cbo_zicbom_setup_cache(int cbom_block_size)
+{
+ struct riscv_cache_ops zicbom_ops;
+
+ if (cbom_block_size <= 0 || !powerof2(cbom_block_size)) {
+ printf("Zicbom: could not initialise (invalid cache line %d)\n",
+ cbom_block_size);
+ return;
+ }
+
+ zicbom_ops.dcache_wbinv_range = cbo_zicbom_cpu_dcache_wbinv_range;
+ zicbom_ops.dcache_inv_range = cbo_zicbom_cpu_dcache_inv_range;
+ zicbom_ops.dcache_wb_range = cbo_zicbom_cpu_dcache_wb_range;
+ riscv_cache_install_hooks(&zicbom_ops, cbom_block_size);
+}
diff --git a/sys/riscv/riscv/identcpu.c b/sys/riscv/riscv/identcpu.c
index 54e008122eab..c76732b2ef46 100644
--- a/sys/riscv/riscv/identcpu.c
+++ b/sys/riscv/riscv/identcpu.c
@@ -53,6 +53,7 @@
#include <machine/elf.h>
#include <machine/md_var.h>
#include <machine/thead.h>
+#include <machine/cbo.h>
#ifdef FDT
#include <dev/fdt/fdt_common.h>
@@ -78,6 +79,11 @@ bool __read_frequently has_sstc;
bool __read_frequently has_sscofpmf;
bool has_svpbmt;
+/* Z-extensions support. */
+bool has_zicbom;
+bool has_zicboz;
+bool has_zicbop;
+
struct cpu_desc {
const char *cpu_mvendor_name;
const char *cpu_march_name;
@@ -89,6 +95,12 @@ struct cpu_desc {
#define SV_SVPBMT (1 << 2)
#define SV_SVINVAL (1 << 3)
#define SV_SSCOFPMF (1 << 4)
+ u_int z_extensions; /* Multi-letter extensions. */
+#define Z_ZICBOM (1 << 0)
+#define Z_ZICBOZ (1 << 1)
+#define Z_ZICBOP (1 << 2)
+ int cbom_block_size;
+ int cboz_block_size;
};
struct cpu_desc cpu_desc[MAXCPU];
@@ -196,11 +208,24 @@ parse_ext_x(struct cpu_desc *desc __unused, char *isa, int idx, int len)
static __inline int
parse_ext_z(struct cpu_desc *desc __unused, char *isa, int idx, int len)
{
+#define CHECK_Z_EXT(str, flag) \
+ do { \
+ if (strncmp(&isa[idx], (str), \
+ MIN(strlen(str), len - idx)) == 0) { \
+ desc->z_extensions |= flag; \
+ return (idx + strlen(str)); \
+ } \
+ } while (0)
+
+ /* Check for known/supported extensions. */
+ CHECK_Z_EXT("zicbom", Z_ZICBOM);
+ CHECK_Z_EXT("zicboz", Z_ZICBOZ);
+ CHECK_Z_EXT("zicbop", Z_ZICBOP);
+
+#undef CHECK_Z_EXT
/*
* Proceed to the next multi-letter extension or the end of the
* string.
- *
- * TODO: parse some of these.
*/
while (isa[idx] != '_' && idx < len) {
idx++;
@@ -321,6 +346,22 @@ parse_mmu_fdt(struct cpu_desc *desc, phandle_t node)
}
}
+static void
+parse_cbo_fdt(struct cpu_desc *desc, phandle_t node)
+{
+ int error;
+
+ error = OF_getencprop(node, "riscv,cbom-block-size",
+ &desc->cbom_block_size, sizeof(desc->cbom_block_size));
+ if (error == -1)
+ desc->cbom_block_size = 0;
+
+ error = OF_getencprop(node, "riscv,cboz-block-size",
+ &desc->cboz_block_size, sizeof(desc->cboz_block_size));
+ if (error == -1)
+ desc->cboz_block_size = 0;
+}
+
static void
identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc)
{
@@ -372,6 +413,9 @@ identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc)
/* Check MMU features. */
parse_mmu_fdt(desc, node);
+ /* Cache-block operations (CBO). */
+ parse_cbo_fdt(desc, node);
+
/* We are done. */
break;
}
@@ -422,6 +466,11 @@ update_global_capabilities(u_int cpu, struct cpu_desc *desc)
UPDATE_CAP(has_sscofpmf, (desc->smode_extensions & SV_SSCOFPMF) != 0);
UPDATE_CAP(has_svpbmt, (desc->smode_extensions & SV_SVPBMT) != 0);
+ /* Z extension support. */
+ UPDATE_CAP(has_zicbom, (desc->z_extensions & Z_ZICBOM) != 0);
+ UPDATE_CAP(has_zicboz, (desc->z_extensions & Z_ZICBOZ) != 0);
+ UPDATE_CAP(has_zicbop, (desc->z_extensions & Z_ZICBOP) != 0);
+
#undef UPDATE_CAP
}
@@ -506,6 +555,9 @@ identify_cpu(u_int cpu)
update_global_capabilities(cpu, desc);
handle_cpu_quirks(cpu, desc);
+
+ if (has_zicbom && cpu == 0)
+ cbo_zicbom_setup_cache(desc->cbom_block_size);
}
void