kinst: start moving towards per-probe trampolines

Using per-CPU and per-thread trampolines is expensive and error-prone,
since we're rewriting the same memory blocks constantly. Per-probe
trampolines solve this problem by giving each probe its own block of
executable memory, which more or less remains the same after the initial
write.

What this patch does, is get rid of the initialization code which
allocates a trampoline for each thread, and instead let each port of
kinst allocate a trampoline for each new probe created. It also sets up
the infrastructure needed to support the new trampoline scheme.

This change is not currently supported on amd64, as the amd64 port needs
further changes to work, so this is a temporary/gradual patch to fix the
riscv and arm64 ports.

Reviewed by:	markj
Approved by:	markj (mentor)
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D40962

sys/cddl/dev/kinst/kinst.c

@@ -228,6 +228,9 @@ kinst_destroy(void *arg, dtrace_id_t id, void *parg)
 	struct kinst_probe *kp = parg;
 
 	LIST_REMOVE(kp, kp_hashnext);
+#ifndef __amd64__
+	kinst_trampoline_dealloc(kp->kp_tramp);
+#endif
 	free(kp, M_KINST);
 }
 

sys/cddl/dev/kinst/kinst.h

@@ -35,10 +35,38 @@ struct kinst_probe {
 	kinst_patchval_t	kp_patchval;
 	kinst_patchval_t	kp_savedval;
 	kinst_patchval_t	*kp_patchpoint;
+	uint8_t			*kp_tramp;
 
 	struct kinst_probe_md	kp_md;
 };
 
+struct kinst_cpu_state {
+	/*
+	 * kinst uses a breakpoint to return from the trampoline and resume
+	 * execution. To do this safely, kinst implements a per-CPU state
+	 * machine; the state is set to KINST_PROBE_FIRED for the duration of
+	 * the trampoline execution (i.e from the time we transfer execution to
+	 * it, until we return). Upon return, the state is set to
+	 * KINST_PROBE_ARMED to indicate that a probe is not currently firing.
+	 * All CPUs have their state initialized to KINST_PROBE_ARMED when
+	 * kinst is loaded.
+	 */
+	enum {
+		KINST_PROBE_ARMED,
+		KINST_PROBE_FIRED,
+	} state;
+	/*
+	 * Points to the probe whose trampoline we're currently executing.
+	 */
+	struct kinst_probe *kp;
+	/*
+	 * Because we execute trampolines with interrupts disabled, we have to
+	 * cache the CPU's status in order to restore it when we return from
+	 * the trampoline.
+	 */
+	uint64_t status;
+};
+
 LIST_HEAD(kinst_probe_list, kinst_probe);
 
 extern struct kinst_probe_list	*kinst_probetab;

sys/cddl/dev/kinst/trampoline.c

@@ -49,8 +49,10 @@ static TAILQ_HEAD(, trampchunk)	kinst_trampchunks =
     TAILQ_HEAD_INITIALIZER(kinst_trampchunks);
 static struct sx		kinst_tramp_sx;
 SX_SYSINIT(kinst_tramp_sx, &kinst_tramp_sx, "kinst tramp");
+#ifdef __amd64__
 static eventhandler_tag		kinst_thread_ctor_handler;
 static eventhandler_tag		kinst_thread_dtor_handler;
+#endif
 
 /*
  * Fill the trampolines with KINST_TRAMP_FILL_PATTERN so that the kernel will
@@ -150,12 +152,14 @@ kinst_trampoline_alloc_locked(int how)
 		if ((how & M_NOWAIT) != 0)
 			return (NULL);
 
-		/*
-		 * We didn't find any free trampoline in the current list,
-		 * allocate a new one.  If that fails the provider will no
-		 * longer be reliable, so try to warn the user.
-		 */
 		if ((chunk = kinst_trampchunk_alloc()) == NULL) {
+#ifdef __amd64__
+			/*
+			 * We didn't find any free trampoline in the current
+			 * list, allocate a new one.  If that fails the
+			 * provider will no longer be reliable, so try to warn
+			 * the user.
+			 */
 			static bool once = true;
 
 			if (once) {
@@ -164,6 +168,7 @@ kinst_trampoline_alloc_locked(int how)
 				    "kinst: failed to allocate trampoline, "
 				    "probes may not fire");
 			}
+#endif
 			return (NULL);
 		}
 		off = 0;
@@ -220,6 +225,7 @@ kinst_trampoline_dealloc(uint8_t *tramp)
 	sx_xunlock(&kinst_tramp_sx);
 }
 
+#ifdef __amd64__
 static void
 kinst_thread_ctor(void *arg __unused, struct thread *td)
 {
@@ -240,10 +246,12 @@ kinst_thread_dtor(void *arg __unused, struct thread *td)
 	 */
 	kinst_trampoline_dealloc(tramp);
 }
+#endif
 
 int
 kinst_trampoline_init(void)
 {
+#ifdef __amd64__
 	struct proc *p;
 	struct thread *td;
 	void *tramp;
@@ -296,12 +304,21 @@ kinst_trampoline_init(void)
 out:
 	sx_xunlock(&kinst_tramp_sx);
 	sx_sunlock(&allproc_lock);
+#else
+	int error = 0;
+
+	sx_xlock(&kinst_tramp_sx);
+	TAILQ_INIT(&kinst_trampchunks);
+	sx_xunlock(&kinst_tramp_sx);
+#endif
+
 	return (error);
 }
 
 int
 kinst_trampoline_deinit(void)
 {
+#ifdef __amd64__
 	struct trampchunk *chunk, *tmp;
 	struct proc *p;
 	struct thread *td;
@@ -324,6 +341,14 @@ kinst_trampoline_deinit(void)
 	TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp)
 		kinst_trampchunk_free(chunk);
 	sx_xunlock(&kinst_tramp_sx);
+#else
+	struct trampchunk *chunk, *tmp;
+
+	sx_xlock(&kinst_tramp_sx);
+	TAILQ_FOREACH_SAFE(chunk, &kinst_trampchunks, next, tmp)
+		kinst_trampchunk_free(chunk);
+	sx_xunlock(&kinst_tramp_sx);
+#endif
 
 	return (0);
 }