mirror of
https://github.com/torvalds/linux
synced 2024-09-06 18:01:05 +00:00
![Song Liu](/assets/img/avatar_default.png)
Instead of blindly allocating PAGE_SIZE for each trampoline, check the size of the trampoline with arch_bpf_trampoline_size(). This size is saved in bpf_tramp_image->size, and used for modmem charge/uncharge. The fallback arch_alloc_bpf_trampoline() still allocates a whole page because we need to use set_memory_* to protect the memory. struct_ops trampoline still uses a whole page for multiple trampolines. With this size check at caller (regular trampoline and struct_ops trampoline), remove arch_bpf_trampoline_size() from arch_prepare_bpf_trampoline() in archs. Also, update bpf_image_ksym_add() to handle symbol of different sizes. Signed-off-by: Song Liu <song@kernel.org> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> # on s390x Acked-by: Jiri Olsa <jolsa@kernel.org> Acked-by: Björn Töpel <bjorn@rivosinc.com> Tested-by: Björn Töpel <bjorn@rivosinc.com> # on riscv Link: https://lore.kernel.org/r/20231206224054.492250-7-song@kernel.org Signed-off-by: Alexei Starovoitov <ast@kernel.org>
171 lines
4.2 KiB
C
171 lines
4.2 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/* Copyright(c) 2019 Intel Corporation. */
|
|
|
|
#include <linux/hash.h>
|
|
#include <linux/bpf.h>
|
|
#include <linux/filter.h>
|
|
#include <linux/static_call.h>
|
|
|
|
/* The BPF dispatcher is a multiway branch code generator. The
|
|
* dispatcher is a mechanism to avoid the performance penalty of an
|
|
* indirect call, which is expensive when retpolines are enabled. A
|
|
* dispatch client registers a BPF program into the dispatcher, and if
|
|
* there is available room in the dispatcher a direct call to the BPF
|
|
* program will be generated. All calls to the BPF programs called via
|
|
* the dispatcher will then be a direct call, instead of an
|
|
* indirect. The dispatcher hijacks a trampoline function it via the
|
|
* __fentry__ of the trampoline. The trampoline function has the
|
|
* following signature:
|
|
*
|
|
* unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
|
|
* unsigned int (*bpf_func)(const void *,
|
|
* const struct bpf_insn *));
|
|
*/
|
|
|
|
static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
|
|
struct bpf_dispatcher *d, struct bpf_prog *prog)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
|
|
if (prog == d->progs[i].prog)
|
|
return &d->progs[i];
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
|
|
struct bpf_dispatcher *d)
|
|
{
|
|
return bpf_dispatcher_find_prog(d, NULL);
|
|
}
|
|
|
|
static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
|
|
struct bpf_prog *prog)
|
|
{
|
|
struct bpf_dispatcher_prog *entry;
|
|
|
|
if (!prog)
|
|
return false;
|
|
|
|
entry = bpf_dispatcher_find_prog(d, prog);
|
|
if (entry) {
|
|
refcount_inc(&entry->users);
|
|
return false;
|
|
}
|
|
|
|
entry = bpf_dispatcher_find_free(d);
|
|
if (!entry)
|
|
return false;
|
|
|
|
bpf_prog_inc(prog);
|
|
entry->prog = prog;
|
|
refcount_set(&entry->users, 1);
|
|
d->num_progs++;
|
|
return true;
|
|
}
|
|
|
|
static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
|
|
struct bpf_prog *prog)
|
|
{
|
|
struct bpf_dispatcher_prog *entry;
|
|
|
|
if (!prog)
|
|
return false;
|
|
|
|
entry = bpf_dispatcher_find_prog(d, prog);
|
|
if (!entry)
|
|
return false;
|
|
|
|
if (refcount_dec_and_test(&entry->users)) {
|
|
entry->prog = NULL;
|
|
bpf_prog_put(prog);
|
|
d->num_progs--;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs)
|
|
{
|
|
return -ENOTSUPP;
|
|
}
|
|
|
|
static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf)
|
|
{
|
|
s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
|
|
int i;
|
|
|
|
for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
|
|
if (d->progs[i].prog)
|
|
*ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
|
|
}
|
|
return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs);
|
|
}
|
|
|
|
static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
|
|
{
|
|
void *new, *tmp;
|
|
u32 noff = 0;
|
|
|
|
if (prev_num_progs)
|
|
noff = d->image_off ^ (PAGE_SIZE / 2);
|
|
|
|
new = d->num_progs ? d->image + noff : NULL;
|
|
tmp = d->num_progs ? d->rw_image + noff : NULL;
|
|
if (new) {
|
|
/* Prepare the dispatcher in d->rw_image. Then use
|
|
* bpf_arch_text_copy to update d->image, which is RO+X.
|
|
*/
|
|
if (bpf_dispatcher_prepare(d, new, tmp))
|
|
return;
|
|
if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2)))
|
|
return;
|
|
}
|
|
|
|
__BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func);
|
|
|
|
/* Make sure all the callers executing the previous/old half of the
|
|
* image leave it, so following update call can modify it safely.
|
|
*/
|
|
synchronize_rcu();
|
|
|
|
if (new)
|
|
d->image_off = noff;
|
|
}
|
|
|
|
void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
|
|
struct bpf_prog *to)
|
|
{
|
|
bool changed = false;
|
|
int prev_num_progs;
|
|
|
|
if (from == to)
|
|
return;
|
|
|
|
mutex_lock(&d->mutex);
|
|
if (!d->image) {
|
|
d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);
|
|
if (!d->image)
|
|
goto out;
|
|
d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);
|
|
if (!d->rw_image) {
|
|
bpf_prog_pack_free(d->image, PAGE_SIZE);
|
|
d->image = NULL;
|
|
goto out;
|
|
}
|
|
bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym);
|
|
}
|
|
|
|
prev_num_progs = d->num_progs;
|
|
changed |= bpf_dispatcher_remove_prog(d, from);
|
|
changed |= bpf_dispatcher_add_prog(d, to);
|
|
|
|
if (!changed)
|
|
goto out;
|
|
|
|
bpf_dispatcher_update(d, prev_num_progs);
|
|
out:
|
|
mutex_unlock(&d->mutex);
|
|
}
|