mirror of
https://github.com/torvalds/linux
synced 2024-10-15 15:59:15 +00:00
bpf: Add arch_bpf_trampoline_size()
This helper will be used to calculate the size of the trampoline before allocating the memory. arch_prepare_bpf_trampoline() for arm64 and riscv64 can use arch_bpf_trampoline_size() to check the trampoline fits in the image. OTOH, arch_prepare_bpf_trampoline() for s390 has to call the JIT process twice, so it cannot use arch_bpf_trampoline_size(). Signed-off-by: Song Liu <song@kernel.org> Acked-by: Ilya Leoshkevich <iii@linux.ibm.com> Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> # on s390x Acked-by: Jiri Olsa <jolsa@kernel.org> Acked-by: Björn Töpel <bjorn@rivosinc.com> Tested-by: Björn Töpel <bjorn@rivosinc.com> # on riscv Link: https://lore.kernel.org/r/20231206224054.492250-6-song@kernel.org Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
parent
38b8b58ae7
commit
96d1b7c081
|
@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
|
|||
return ctx->idx;
|
||||
}
|
||||
|
||||
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
||||
void *image_end, const struct btf_func_model *m,
|
||||
u32 flags, struct bpf_tramp_links *tlinks,
|
||||
void *func_addr)
|
||||
static int btf_func_model_nregs(const struct btf_func_model *m)
|
||||
{
|
||||
int i, ret;
|
||||
int nregs = m->nr_args;
|
||||
int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE;
|
||||
struct jit_ctx ctx = {
|
||||
.image = NULL,
|
||||
.idx = 0,
|
||||
};
|
||||
int i;
|
||||
|
||||
/* extra registers needed for struct argument */
|
||||
for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) {
|
||||
|
@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
|||
nregs += (m->arg_size[i] + 7) / 8 - 1;
|
||||
}
|
||||
|
||||
return nregs;
|
||||
}
|
||||
|
||||
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks, void *func_addr)
|
||||
{
|
||||
struct jit_ctx ctx = {
|
||||
.image = NULL,
|
||||
.idx = 0,
|
||||
};
|
||||
struct bpf_tramp_image im;
|
||||
int nregs, ret;
|
||||
|
||||
nregs = btf_func_model_nregs(m);
|
||||
/* the first 8 registers are used for arguments */
|
||||
if (nregs > 8)
|
||||
return -ENOTSUPP;
|
||||
|
||||
ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags);
|
||||
ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, nregs, flags);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
if (ret > max_insns)
|
||||
return -EFBIG;
|
||||
return ret < 0 ? ret : ret * AARCH64_INSN_SIZE;
|
||||
}
|
||||
|
||||
ctx.image = image;
|
||||
ctx.idx = 0;
|
||||
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
||||
void *image_end, const struct btf_func_model *m,
|
||||
u32 flags, struct bpf_tramp_links *tlinks,
|
||||
void *func_addr)
|
||||
{
|
||||
int ret, nregs;
|
||||
struct jit_ctx ctx = {
|
||||
.image = image,
|
||||
.idx = 0,
|
||||
};
|
||||
|
||||
nregs = btf_func_model_nregs(m);
|
||||
/* the first 8 registers are used for arguments */
|
||||
if (nregs > 8)
|
||||
return -ENOTSUPP;
|
||||
|
||||
ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
if (ret > ((long)image_end - (long)image))
|
||||
return -EFBIG;
|
||||
|
||||
jit_fill_hole(image, (unsigned int)(image_end - image));
|
||||
ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags);
|
||||
|
|
|
@ -1029,6 +1029,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
|
|||
return ret;
|
||||
}
|
||||
|
||||
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks, void *func_addr)
|
||||
{
|
||||
struct bpf_tramp_image im;
|
||||
struct rv_jit_context ctx;
|
||||
int ret;
|
||||
|
||||
ctx.ninsns = 0;
|
||||
ctx.insns = NULL;
|
||||
ctx.ro_insns = NULL;
|
||||
ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx);
|
||||
|
||||
return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns);
|
||||
}
|
||||
|
||||
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
||||
void *image_end, const struct btf_func_model *m,
|
||||
u32 flags, struct bpf_tramp_links *tlinks,
|
||||
|
@ -1037,14 +1052,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
|||
int ret;
|
||||
struct rv_jit_context ctx;
|
||||
|
||||
ctx.ninsns = 0;
|
||||
ctx.insns = NULL;
|
||||
ctx.ro_insns = NULL;
|
||||
ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
|
||||
ret = arch_bpf_trampoline_size(im, m, flags, tlinks, func_addr);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
if (ninsns_rvoff(ret) > (long)image_end - (long)image)
|
||||
if (ret > (long)image_end - (long)image)
|
||||
return -EFBIG;
|
||||
|
||||
ctx.ninsns = 0;
|
||||
|
|
|
@ -2637,6 +2637,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
|
|||
return 0;
|
||||
}
|
||||
|
||||
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks, void *orig_call)
|
||||
{
|
||||
struct bpf_tramp_image im;
|
||||
struct bpf_tramp_jit tjit;
|
||||
int ret;
|
||||
|
||||
memset(&tjit, 0, sizeof(tjit));
|
||||
|
||||
ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags,
|
||||
tlinks, orig_call);
|
||||
|
||||
return ret < 0 ? ret : tjit.common.prg;
|
||||
}
|
||||
|
||||
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
||||
void *image_end, const struct btf_func_model *m,
|
||||
u32 flags, struct bpf_tramp_links *tlinks,
|
||||
|
@ -2644,19 +2659,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
|||
{
|
||||
struct bpf_tramp_jit tjit;
|
||||
int ret;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 2; i++) {
|
||||
if (i == 0) {
|
||||
/* Compute offsets, check whether the code fits. */
|
||||
memset(&tjit, 0, sizeof(tjit));
|
||||
} else {
|
||||
/* Generate the code. */
|
||||
tjit.common.prg = 0;
|
||||
tjit.common.prg_buf = image;
|
||||
}
|
||||
ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags,
|
||||
tlinks, func_addr);
|
||||
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
if (tjit.common.prg > (char *)image_end - (char *)image)
|
||||
|
@ -2665,9 +2673,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
|||
* BPF_MAX_TRAMP_LINKS.
|
||||
*/
|
||||
return -E2BIG;
|
||||
}
|
||||
|
||||
return tjit.common.prg;
|
||||
tjit.common.prg = 0;
|
||||
tjit.common.prg_buf = image;
|
||||
ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags,
|
||||
tlinks, func_addr);
|
||||
|
||||
return ret < 0 ? ret : tjit.common.prg;
|
||||
}
|
||||
|
||||
bool bpf_jit_supports_subprog_tailcalls(void)
|
||||
|
|
|
@ -2422,7 +2422,7 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
|
|||
* add rsp, 8 // skip eth_type_trans's frame
|
||||
* ret // return to its caller
|
||||
*/
|
||||
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
|
||||
static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
|
||||
const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks,
|
||||
void *func_addr)
|
||||
|
@ -2678,6 +2678,38 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
|
|||
return ret;
|
||||
}
|
||||
|
||||
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
|
||||
const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks,
|
||||
void *func_addr)
|
||||
{
|
||||
return __arch_prepare_bpf_trampoline(im, image, image_end, m, flags, tlinks, func_addr);
|
||||
}
|
||||
|
||||
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks, void *func_addr)
|
||||
{
|
||||
struct bpf_tramp_image im;
|
||||
void *image;
|
||||
int ret;
|
||||
|
||||
/* Allocate a temporary buffer for __arch_prepare_bpf_trampoline().
|
||||
* This will NOT cause fragmentation in direct map, as we do not
|
||||
* call set_memory_*() on this buffer.
|
||||
*
|
||||
* We cannot use kvmalloc here, because we need image to be in
|
||||
* module memory range.
|
||||
*/
|
||||
image = bpf_jit_alloc_exec(PAGE_SIZE);
|
||||
if (!image)
|
||||
return -ENOMEM;
|
||||
|
||||
ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, m, flags,
|
||||
tlinks, func_addr);
|
||||
bpf_jit_free_exec(image);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf)
|
||||
{
|
||||
u8 *jg_reloc, *prog = *pprog;
|
||||
|
|
|
@ -1106,6 +1106,8 @@ void *arch_alloc_bpf_trampoline(unsigned int size);
|
|||
void arch_free_bpf_trampoline(void *image, unsigned int size);
|
||||
void arch_protect_bpf_trampoline(void *image, unsigned int size);
|
||||
void arch_unprotect_bpf_trampoline(void *image, unsigned int size);
|
||||
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks, void *func_addr);
|
||||
|
||||
u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
|
||||
struct bpf_tramp_run_ctx *run_ctx);
|
||||
|
|
|
@ -1072,6 +1072,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, unsigned int size)
|
|||
set_memory_rw((long)image, 1);
|
||||
}
|
||||
|
||||
int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks, void *func_addr)
|
||||
{
|
||||
return -ENOTSUPP;
|
||||
}
|
||||
|
||||
static int __init init_trampolines(void)
|
||||
{
|
||||
int i;
|
||||
|
|
Loading…
Reference in a new issue