git: 5b701ed19c2e - main - kinst: start moving towards per-probe trampolines

The branch main has been updated by christos:

URL: https://cgit.FreeBSD.org/src/commit/?id=5b701ed19c2ed439457b0177681472aac21fde94

commit 5b701ed19c2ed439457b0177681472aac21fde94
Author:     Christos Margiolis <christos@FreeBSD.org>
AuthorDate: 2023-07-19 14:57:21 +0000
Commit:     Christos Margiolis <christos@FreeBSD.org>
CommitDate: 2023-07-19 14:57:21 +0000

    kinst: start moving towards per-probe trampolines
    
    Using per-CPU and per-thread trampolines is expensive and error-prone,
    since we're rewriting the same memory blocks constantly. Per-probe
    trampolines solve this problem by giving each probe its own block of
    executable memory, which more or less remains the same after the initial
    write.
    
    What this patch does, is get rid of the initialization code which
    allocates a trampoline for each thread, and instead let each port of
    kinst allocate a trampoline for each new probe created. It also sets up
    the infrastructure needed to support the new trampoline scheme.
    
    This change is not currently supported on amd64, as the amd64 port needs
    further changes to work, so this is a temporary/gradual patch to fix the
    riscv and arm64 ports.
    
    Reviewed by:    markj
    Approved by:    markj (mentor)
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D40962
---
 sys/cddl/dev/kinst/kinst.c      |  3 +++
 sys/cddl/dev/kinst/kinst.h      | 28 ++++++++++++++++++++++++++++
 sys/cddl/dev/kinst/trampoline.c | 35 ++++++++++++++++++++++++++++++-----
 3 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/sys/cddl/dev/kinst/kinst.c b/sys/cddl/dev/kinst/kinst.c
index 4bd3047f49db..60400a452b95 100644
--- a/sys/cddl/dev/kinst/kinst.c
+++ b/sys/cddl/dev/kinst/kinst.c
@@ -228,6 +228,9 @@ kinst_destroy(void *arg, dtrace_id_t id, void *parg)
 	struct kinst_probe *kp = parg;
 
 	LIST_REMOVE(kp, kp_hashnext);
+#ifndef __amd64__
+	kinst_trampoline_dealloc(kp->kp_tramp);
+#endif
 	free(kp, M_KINST);
 }
 
diff --git a/sys/cddl/dev/kinst/kinst.h b/sys/cddl/dev/kinst/kinst.h
index 0a47eb4f3583..390a2d1c13bf 100644
--- a/sys/cddl/dev/kinst/kinst.h
+++ b/sys/cddl/dev/kinst/kinst.h
@@ -35,10 +35,38 @@ struct kinst_probe {
 	kinst_patchval_t	kp_patchval;
 	kinst_patchval_t	kp_savedval;
 	kinst_patchval_t	*kp_patchpoint;
+	uint8_t			*kp_tramp;
 
 	struct kinst_probe_md	kp_md;
 };
 
+struct kinst_cpu_state {
+	/*
+	 * kinst uses a breakpoint to return from the trampoline and resume
+	 * execution. To do this safely, kinst implements a per-CPU state
+	 * machine; the state is set to KINST_PROBE_FIRED for the duration of
+	 * the trampoline execution (i.e from the time we transfer execution to
+	 * it, until we return). Upon return, the state is set to
+	 * KINST_PROBE_ARMED to indicate that a probe is not currently firing.
+	 * All CPUs have their state initialized to KINST_PROBE_ARMED when
+	 * kinst is loaded.
+	 */
+	enum {
+		KINST_PROBE_ARMED,
+		KINST_PROBE_FIRED,
+	} state;
+	/*
+	 * Points to the probe whose trampoline we're currently executing.
+	 */
+	struct kinst_probe *kp;
+	/*
+	 * Because we execute trampolines with interrupts disabled, we have to
+	 * cache the CPU's status in order to restore it when we return from
+	 * the trampoline.
+	 */
+	uint64_t status;
+};
+
 LIST_HEAD(kinst_probe_list, kinst_probe);
 
 extern struct kinst_probe_list	*kinst_probetab;
diff --git a/sys/cddl/dev/kinst/trampoline.c b/sys/cddl/dev/kinst/trampoline.c
index 87c01e39745b..adc4eaa7fceb 100644
--- a/sys/cddl/dev/kinst/trampoline.c
+++ b/sys/cddl/dev/kinst/trampoline.c
@@ -49,8 +49,10 @@ static TAILQ_HEAD(, trampchunk)	kinst_trampchunks =
     TAILQ_HEAD_INITIALIZER(kinst_trampchunks);
 static struct sx		kinst_tramp_sx;
 SX_SYSINIT(kinst_tramp_sx, &kinst_tramp_sx, "kinst tramp");
+#ifdef __amd64__
 static eventhandler_tag		kinst_thread_ctor_handler;
 static eventhandler_tag		kinst_thread_dtor_handler;
+#endif
 
 /*
  * Fill the trampolines with KINST_TRAMP_FILL_PATTERN so that the kernel will
@@ -150,12 +152,14 @@ kinst_trampoline_alloc_locked(int how)
 		if ((how & M_NOWAIT) != 0)
 			return (NULL);
 
-		/*
-		 * We didn't find any free trampoline in the current list,
-		 * allocate a new one.  If that fails the provider will no
-		 * longer be reliable, so try to warn the user.
-		 */
 		if ((chunk = kinst_trampchunk_alloc()) == NULL) {
+#ifdef __amd64__
+			/*
+			 * We didn't find any free trampoline in the current
+			 * list, allocate a new one.  If that fails the
+			 * provider will no longer be reliable, so try to warn
+			 * the user.
+			 */
 			static bool once = true;
 
 			if (once) {
@@ -164,6 +168,7 @@ kinst_trampoline_alloc_locked(int how)
 				    "kinst: failed to allocate trampoline, "
 				    "probes may not fire");
 			}
+#endif
 			return (NULL);
 		}
 		off = 0;
@@ -220,6 +225,7 @@ kinst_trampoline_dealloc(uint8_t *tramp)
 	sx_xunlock(&kinst_tramp_sx);
 }
 
+#ifdef __amd64__
 static void
 kinst_thread_ctor(void *arg __unused, struct thread *td)
 {
@@ -240,10 +246,12 @@ kinst_thread_dtor(void *arg __unused, struct thread *td)
 	 */
 	kinst_trampoline_dealloc(tramp);
 }
+#endif
 
 int
 kinst_trampoline_init(void)
 {
+#ifdef __amd64__
 	struct proc *p;
 	struct thread *td;
 	void *tramp;
@@ -296,12 +304,21 @@ retry:
 out:
 	sx_xunlock(&kinst_tramp_sx);
 	sx_sunlock(&allproc_lock);
+#else
+	int error = 0;
+
+	sx_xlock(&kinst_tramp_sx);
+	TAILQ_INIT(&kinst_trampchunks);
+	sx_xunlock(&kinst_tramp_sx);
+#endif
+
 	return (error);
 }
 
 int
 kinst_trampoline_deinit(void)
 {
+#ifdef __amd64__
 	struct trampchunk *chunk, *tmp;
 	struct proc *p;
 	struct thread *td;
@@ -324,6 +341,14 @@ kinst_trampoline_deinit(void)
 	TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp)
 		kinst_trampchunk_free(chunk);
 	sx_xunlock(&kinst_tramp_sx);
+#else
+	struct trampchunk *chunk, *tmp;
+
+	sx_xlock(&kinst_tramp_sx);
+	TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp)
+		kinst_trampchunk_free(chunk);
+	sx_xunlock(&kinst_tramp_sx);
+#endif
 
 	return (0);
 }