mirror of
https://github.com/torvalds/linux
synced 2024-09-06 18:01:05 +00:00
bpf: Avoid retpoline for bpf_for_each_map_elem
Similarly to09772d92cd
("bpf: avoid retpoline for lookup/update/delete calls on maps") and84430d4232
("bpf, verifier: avoid retpoline for map push/pop/peek operation") avoid indirect call while calling bpf_for_each_map_elem. Before (a program fragment): ; if (rules_map) { 142: (15) if r4 == 0x0 goto pc+8 143: (bf) r3 = r10 ; bpf_for_each_map_elem(rules_map, process_each_rule, &ctx, 0); 144: (07) r3 += -24 145: (bf) r1 = r4 146: (18) r2 = subprog[+5] 148: (b7) r4 = 0 149: (85) call bpf_for_each_map_elem#143680 <-- indirect call via helper After (same program fragment): ; if (rules_map) { 142: (15) if r4 == 0x0 goto pc+8 143: (bf) r3 = r10 ; bpf_for_each_map_elem(rules_map, process_each_rule, &ctx, 0); 144: (07) r3 += -24 145: (bf) r1 = r4 146: (18) r2 = subprog[+5] 148: (b7) r4 = 0 149: (85) call bpf_for_each_array_elem#170336 <-- direct call On a benchmark that calls bpf_for_each_map_elem() once and does many other things (mostly checking fields in skb) with CONFIG_RETPOLINE=y it makes program faster. Before: ============================================================================ Benchmark.cpp time/iter iters/s ============================================================================ IngressMatchByRemoteEndpoint 80.78ns 12.38M IngressMatchByRemoteIP 80.66ns 12.40M IngressMatchByRemotePort 80.87ns 12.37M After: ============================================================================ Benchmark.cpp time/iter iters/s ============================================================================ IngressMatchByRemoteEndpoint 73.49ns 13.61M IngressMatchByRemoteIP 71.48ns 13.99M IngressMatchByRemotePort 70.39ns 14.21M Signed-off-by: Andrey Ignatov <rdna@fb.com> Signed-off-by: Alexei Starovoitov <ast@kernel.org> Link: https://lore.kernel.org/bpf/20211006001838.75607-1-rdna@fb.com
This commit is contained in:
parent
32a16f6bfe
commit
0640c77c46
|
@ -13112,7 +13112,8 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
|
|||
insn->imm == BPF_FUNC_map_push_elem ||
|
||||
insn->imm == BPF_FUNC_map_pop_elem ||
|
||||
insn->imm == BPF_FUNC_map_peek_elem ||
|
||||
insn->imm == BPF_FUNC_redirect_map)) {
|
||||
insn->imm == BPF_FUNC_redirect_map ||
|
||||
insn->imm == BPF_FUNC_for_each_map_elem)) {
|
||||
aux = &env->insn_aux_data[i + delta];
|
||||
if (bpf_map_ptr_poisoned(aux))
|
||||
goto patch_call_imm;
|
||||
|
@ -13156,6 +13157,11 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
|
|||
(int (*)(struct bpf_map *map, void *value))NULL));
|
||||
BUILD_BUG_ON(!__same_type(ops->map_redirect,
|
||||
(int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL));
|
||||
BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
|
||||
(int (*)(struct bpf_map *map,
|
||||
bpf_callback_t callback_fn,
|
||||
void *callback_ctx,
|
||||
u64 flags))NULL));
|
||||
|
||||
patch_map_ops_generic:
|
||||
switch (insn->imm) {
|
||||
|
@ -13180,6 +13186,9 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
|
|||
case BPF_FUNC_redirect_map:
|
||||
insn->imm = BPF_CALL_IMM(ops->map_redirect);
|
||||
continue;
|
||||
case BPF_FUNC_for_each_map_elem:
|
||||
insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
|
||||
continue;
|
||||
}
|
||||
|
||||
goto patch_call_imm;
|
||||
|
|
Loading…
Reference in a new issue