From afe897e89918434be8600250dbdda977aa99f53d Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:01 -0700 Subject: [PATCH 01/15] bpf: Read a kfunc's __sz argument only when it is in a register Commit e0b7b91c72db ("bpf: Support stack arguments for kfunc calls") supported stack arguments for kfunc's. In bpf_kfunc_stack_access_bytes(), the size of a ptr + __sz pair is read from const_reg_vals[] at index 'BPF_REG_1 + arg + 1'. Past the fifth argument that index leaves the argument registers and reaches 6 through 9, which are the callee saved registers R6 through R9. The verifier does record constants for those, so a __sz argument passed on the stack can take the value of an unrelated register as its size. Fix it by guard size_reg which has to be less than or equal to MAX_BPF_FUNC_REG_ARGS. Fixes: e0b7b91c72db ("bpf: Support stack arguments for kfunc calls") Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195201.982215-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 617a277c3558..0c6a404167ca 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13641,13 +13641,14 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * goto out; } - /* ptr + __sz/__szk pair: size is in the next register */ + /* ptr + __sz/__szk pair: the size follows the pointer */ if (arg + 1 < nargs && (btf_param_match_suffix(btf, &args[arg + 1], "__sz") || btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) { int size_reg = BPF_REG_1 + arg + 1; - if (aux->const_reg_mask & BIT(size_reg)) { + if (size_reg <= MAX_BPF_FUNC_REG_ARGS && + (aux->const_reg_mask & BIT(size_reg))) { size = (s64)aux->const_reg_vals[size_reg]; goto out; } -- knotmirror 0.0.1 From 4196b1064bf26dd02665750b84ab91d2ad93a83b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:06 -0700 Subject: [PATCH 02/15] selftests/bpf: Add a test for an __int128 by-value argument A 128-bit integer is passed in two consecutive argument registers, but the verifier counts one argument register per parameter whatever its size. For __u64 take_i128_global(int a, u128 v, int c) the compiler passes a in R1, v in R2:R3 and c in R4, while the verifier marks only R1 through R3 at the entry of the global function. The callee then reads its own third parameter out of a register the verifier considers uninitialized, and the program is rejected for a register the source never names: Validating take_i128_global() func#1... 20: R1=scalar() R2=scalar() R3=scalar() R10=fp0 ; __noinline __u64 take_i128_global(int a, u128 v, int c) @ verifier_aggregate_arg.c:12 20: (bf) r0 = r2 ; R0=scalar(id=4) R2=scalar(id=4) ; return (__u64)a + (__u64)(v >> 64) + (__u64)v + c; @ verifier_aggregate_arg.c:14 21: (bc) w1 = w1 ; R1=scalar(smin=0,smax=umax=0xffffffff,var_off=(0x0; 0xffffffff)) 22: (67) r1 <<= 32 ; R1=scalar(smax=0x7fffffff00000000,smin32=0,smax32=umax32=0,var_off=(0x0; 0xffffffff00000000)) 23: (c7) r1 s>>= 32 ; R1=scalar(smin=0xffffffff80000000,smax=0x7fffffff) 24: (0f) r0 += r1 ; R0=scalar() R1=scalar(smin=0xffffffff80000000,smax=0x7fffffff) 25: (0f) r0 += r3 ; R0=scalar() R3=scalar() 26: (bc) w1 = w4 R4 !read_ok The log is from clang 23. LLVM 21/22 place the argument in the same registers. Add the test with the failure it produces now. A later patch, "bpf: Support __int128 as a by-value function argument", places the two slots and flips this test to __success. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195206.984633-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_aggregate_arg.c | 40 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index b97381448248..7732df9bc870 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -5,6 +5,7 @@ #include "arena_kfunc.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "verifier_aggregate_arg.skel.h" #include "verifier_aggregate_ret.skel.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" @@ -172,6 +173,7 @@ void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } +void test_verifier_aggregate_arg(void) { RUN_TESTS(verifier_aggregate_arg); } void test_verifier_aggregate_ret(void) { RUN_TESTS(verifier_aggregate_ret); } void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c new file mode 100644 index 000000000000..d90f754396d0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +#ifdef __SIZEOF_INT128__ + +typedef unsigned __int128 u128; + +__noinline __u64 take_i128_global(int a, u128 v, int c) +{ + return (__u64)a + (__u64)(v >> 64) + (__u64)v + c; +} + +SEC("tc") +/* + * The verifier counts one argument register for the __int128 and marks only + * R1 through R3 at the entry of take_i128_global(), while the compiler passed + * a in R1, v in R2:R3 and c in R4. + */ +__failure __msg("R4 !read_ok") +int aggregate_arg_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (take_i128_global(1, v, 2) != a + b + 3) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +char _license[] SEC("license") = "GPL"; -- knotmirror 0.0.1 From 3bd9d96710f00b9e5a7218d1d1cbb93d29b0595b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:11 -0700 Subject: [PATCH 03/15] bpf: Rename bpf_subprog_info::arg_cnt to arg_slot_cnt Rename arg_cnt to arg_slot_cnt, as a later patch gives a parameter that takes two argument registers, an __int128 or a 16-byte aggregate, two slots. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195211.986081-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 6 +++--- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 19 +++++++++++-------- .../bpf/progs/verifier_stack_arg_order.c | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 1e7593e8d5c5..120a32bdd451 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -828,7 +828,7 @@ struct bpf_subprog_info { bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; bool might_sleep: 1; - u8 arg_cnt:4; + u8 arg_slot_cnt:4; enum priv_stack_mode priv_stack_mode; struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS]; @@ -838,8 +838,8 @@ struct bpf_subprog_info { static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) { - if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) - return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS; + if (sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) + return sub->arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; return 0; } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c693f2c345f2..9b0bd3898882 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8081,7 +8081,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) } args = (const struct btf_param *)(t + 1); nargs = btf_type_vlen(t); - sub->arg_cnt = nargs; + sub->arg_slot_cnt = nargs; if (nargs > MAX_BPF_FUNC_ARGS) { bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n", MAX_BPF_FUNC_ARGS, tname, nargs); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c6a404167ca..cf2696d7b778 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10247,7 +10247,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + err = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), NULL, NULL); if (err) @@ -10259,7 +10259,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); args = btf_params(func_proto); - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, + ret = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), btf, args); if (ret) return ret; @@ -10267,7 +10267,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, /* check that BTF function arguments match actual types that the * verifier sees. */ - for (i = 0; i < sub->arg_cnt; i++) { + for (i = 0; i < sub->arg_slot_cnt; i++) { argno_t argno = argno_from_arg(i + 1); struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); struct bpf_subprog_arg_info *arg = &sub->args[i]; @@ -19647,13 +19647,14 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) } /* Also ensure the callback only has a single scalar argument. */ - if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { + if (sub->arg_slot_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { verbose(env, "exception cb only supports single integer argument\n"); ret = -EINVAL; goto out; } } - for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = BPF_REG_1; + i <= min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { arg = &sub->args[i - BPF_REG_1]; reg = ®s[i]; @@ -19696,7 +19697,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) goto out; } } - if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) { + if (env->prog->type == BPF_PROG_TYPE_EXT && + sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) { verbose(env, "freplace programs with >%d args not supported yet\n", MAX_BPF_FUNC_REG_ARGS); ret = -EINVAL; @@ -19709,9 +19711,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) */ if (env->prog->aux->func_info_aux) { ret = btf_prepare_func_args(env, 0); - if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) { + if (ret || sub->arg_slot_cnt != 1 || + sub->args[0].arg_type != ARG_PTR_TO_CTX) { env->prog->aux->func_info_aux[0].unreliable = true; - sub->arg_cnt = 1; + sub->arg_slot_cnt = 1; sub->stack_arg_cnt = 0; } } diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index 57f22691744a..ab1955852233 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -116,8 +116,8 @@ __naked void stack_arg_pruning_load_after_call(void) /* * "bad_ptr": the first arg is 'long *', which is not a recognized pointer * type for static subprogs (not ctx, dynptr, or tagged). btf_prepare_func_args() - * sets arg_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The subprog - * is marked unreliable but the call still proceeds for static subprogs. + * sets arg_slot_cnt = 7 / stack_arg_cnt = 2, then fails with -EINVAL. The + * subprog is marked unreliable but the call still proceeds for static subprogs. */ __noinline __used __naked static void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) -- knotmirror 0.0.1 From 3f1023cab8130e8d43271b57d36139273a6c48a0 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:16 -0700 Subject: [PATCH 04/15] bpf: Index global function arguments by argument slot btf_prepare_func_args() indexes sub->args[] by BTF parameter, so a parameter can only ever stand for a single argument register. Give the loop a second index: 'i' keeps walking the BTF parameters while 'slots_used' walks the argument slots, and sub->arg_slot_cnt becomes the number of slots, so that a later patch can give a parameter two of them. No functional change: every parameter still takes exactly one slot. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195216.987189-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9b0bd3898882..cdbae53bb94d 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8037,7 +8037,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) const struct btf_param *args; const struct btf_type *t, *ref_t, *fn_t; int err; - u32 i, nargs, btf_id; + u32 i, slots_used, nargs, btf_id; const char *tname; if (sub->args_cached) @@ -8122,8 +8122,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) /* Convert BTF function arguments into verifier types. * Only PTR_TO_CTX and SCALAR are supported atm. */ - for (i = 0; i < nargs; i++) { + for (i = 0, slots_used = 0; i < nargs; i++) { u32 tags = 0; + err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); if (err) return err; @@ -8145,7 +8146,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) btf_validate_prog_ctx_type(log, btf, t, i, prog_type, prog->expected_attach_type)) return -EINVAL; - sub->args[i].arg_type = ARG_PTR_TO_CTX; + sub->args[slots_used++].arg_type = ARG_PTR_TO_CTX; continue; } if (btf_is_dynptr_ptr(btf, t)) { @@ -8153,7 +8154,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d has invalid combination of tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_DYNPTR; + sub->args[slots_used++].arg_type = ARG_PTR_TO_DYNPTR; continue; } if (tags & ARG_TAG_TRUSTED) { @@ -8168,10 +8169,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (kern_type_id < 0) return kern_type_id; - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; if (tags & ARG_TAG_NULLABLE) - sub->args[i].arg_type |= PTR_MAYBE_NULL; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type |= PTR_MAYBE_NULL; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_UNTRUSTED) { @@ -8185,8 +8187,10 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) ref_t = btf_type_skip_modifiers(btf, t->type, NULL); if (btf_type_is_void(ref_t) || btf_type_is_primitive(ref_t)) { - sub->args[i].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED; - sub->args[i].mem_size = 0; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | + PTR_UNTRUSTED; + sub->args[slots_used].mem_size = 0; + slots_used++; continue; } @@ -8202,8 +8206,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) i, btf_type_str(ref_t), tname); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_ARENA) { @@ -8211,7 +8216,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_ARENA; + sub->args[slots_used++].arg_type = ARG_PTR_TO_ARENA; continue; } if (is_global) { /* generic user data pointer */ @@ -8231,10 +8236,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; if (tags & ARG_TAG_NONNULL) - sub->args[i].arg_type &= ~PTR_MAYBE_NULL; - sub->args[i].mem_size = mem_size; + sub->args[slots_used].arg_type &= ~PTR_MAYBE_NULL; + sub->args[slots_used].mem_size = mem_size; + slots_used++; continue; } @@ -8244,7 +8250,7 @@ skip_pointer: return -EINVAL; } if (btf_type_is_int(t) || btf_is_any_enum(t)) { - sub->args[i].arg_type = ARG_SCALAR; + sub->args[slots_used++].arg_type = ARG_SCALAR; continue; } if (!is_global) @@ -8254,6 +8260,8 @@ skip_pointer: return -EINVAL; } + sub->arg_slot_cnt = slots_used; + sub->args_cached = true; return 0; -- knotmirror 0.0.1 From 4dd676ca5b7a3bdc18e217bdd618c5e73016aee4 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:21 -0700 Subject: [PATCH 05/15] bpf: Support by-value struct arguments up to 16 bytes A global function taking a struct or union by value is rejected today: Arg#1 type STRUCT in tar() is not supported yet. Accept one of at most 16 bytes, which arrives in one or two consecutive argument registers. Only structs composed entirely of scalars are taken for now; btf_struct_is_composed_of() enforces that. The slots of a value are independent of each other, so the compiler may split one across the last argument register and the stack, as in static void f(int a, int b, int c, int d, struct pair p); or place it wholly past the registers, and the verifier describes either the same way. What has to follow the slots is stack_arg_cnt, recomputed from the slots consumed, together with the "no stack args in global functions" and JIT support checks, and the MAX_BPF_FUNC_ARGS bound, which now has to account for a parameter that takes two slots at once. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195221.987808-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 80 +++++++++++++++++++++++++++++++++++-------- kernel/bpf/verifier.c | 2 ++ 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index cdbae53bb94d..c560ee0c4347 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8018,6 +8018,29 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt return -EOPNOTSUPP; } +static int btf_check_arg_slots(struct bpf_verifier_log *log, const char *tname, + bool is_global, u32 slot_cnt, + struct bpf_subprog_info *sub) +{ + if (slot_cnt <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + if (is_global) { + bpf_log(log, + "global function %s() needs %d > %d argument slots, " + "stack args not supported\n", + tname, slot_cnt, MAX_BPF_FUNC_REG_ARGS); + return -EINVAL; + } + if (!bpf_jit_supports_stack_args()) { + bpf_log(log, "JIT does not support function %s() with %d argument slots\n", + tname, slot_cnt); + return -EFAULT; + } + sub->stack_arg_cnt = slot_cnt - MAX_BPF_FUNC_REG_ARGS; + return 0; +} + /* Process BTF of a function to produce high-level expectation of function * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information * is cached in subprog info for reuse. @@ -8087,20 +8110,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) MAX_BPF_FUNC_ARGS, tname, nargs); return -EFAULT; } - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - if (!bpf_jit_supports_stack_args()) { - bpf_log(log, "JIT does not support function %s() with %d args\n", - tname, nargs); - return -EFAULT; - } - sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; - } - - if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) { - bpf_log(log, "global function %s has %d > %d args, stack args not supported\n", - tname, nargs, MAX_BPF_FUNC_REG_ARGS); - return -EINVAL; - } + err = btf_check_arg_slots(log, tname, is_global, nargs, sub); + if (err) + return err; err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { @@ -8125,6 +8137,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) for (i = 0, slots_used = 0; i < nargs; i++) { u32 tags = 0; + if (slots_used >= MAX_BPF_FUNC_ARGS) + goto too_many_slots; + err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); if (err) return err; @@ -8253,6 +8268,33 @@ skip_pointer: sub->args[slots_used++].arg_type = ARG_SCALAR; continue; } + if (btf_type_is_struct(t)) { + u32 nslots; + + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + if (!is_global) + return -EINVAL; + bpf_log(log, + "Arg#%d type %s in %s() has size %u, only 1 to %d bytes " + "can be passed by value\n", + i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (!btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { + if (!is_global) + return -EINVAL; + bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n", + i, btf_type_str(t), tname); + return -EINVAL; + } + + nslots = (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + if (slots_used + nslots > MAX_BPF_FUNC_ARGS) + goto too_many_slots; + while (nslots--) + sub->args[slots_used++].arg_type = ARG_SCALAR; + continue; + } if (!is_global) return -EINVAL; bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n", @@ -8260,11 +8302,21 @@ skip_pointer: return -EINVAL; } + err = btf_check_arg_slots(log, tname, is_global, slots_used, sub); + if (err) + return err; sub->arg_slot_cnt = slots_used; sub->args_cached = true; return 0; + +too_many_slots: + if (!is_global) + return -EINVAL; + bpf_log(log, "Arguments of %s() need more than %d argument slots\n", + tname, MAX_BPF_FUNC_ARGS); + return -EINVAL; } static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index cf2696d7b778..d8963d50d85b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10259,6 +10259,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); args = btf_params(func_proto); + if (sub->arg_slot_cnt != btf_type_vlen(func_proto)) + args = NULL; ret = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), btf, args); if (ret) -- knotmirror 0.0.1 From 00cfeb9554e2513cc721acdf5679bd26dcf24e02 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:27 -0700 Subject: [PATCH 06/15] bpf: Support __int128 as a by-value function argument A 128-bit integer follows the same calling convention as a 16-byte by-value struct: LLVM emits it as a 16-byte BTF_KIND_INT and passes it in two consecutive argument registers. So a __int128 gets its two slots. The test added at the start of the series, which recorded the wrong register map as an R4 !read_ok rejection, now passes and is flipped to __success. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195227.987991-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 9 +++------ .../testing/selftests/bpf/progs/verifier_aggregate_arg.c | 7 +------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c560ee0c4347..9f8a4e4aac3b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8264,11 +8264,7 @@ skip_pointer: bpf_log(log, "arg#%d has pointer tag, but is not a pointer type\n", i); return -EINVAL; } - if (btf_type_is_int(t) || btf_is_any_enum(t)) { - sub->args[slots_used++].arg_type = ARG_SCALAR; - continue; - } - if (btf_type_is_struct(t)) { + if (btf_type_is_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) { u32 nslots; if (!t->size || t->size > 2 * BPF_REG_SIZE) { @@ -8280,7 +8276,8 @@ skip_pointer: i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE); return -EINVAL; } - if (!btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { + if (btf_type_is_struct(t) && + !btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { if (!is_global) return -EINVAL; bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n", diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c index d90f754396d0..fc7c1b18bc40 100644 --- a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c @@ -17,12 +17,7 @@ __noinline __u64 take_i128_global(int a, u128 v, int c) } SEC("tc") -/* - * The verifier counts one argument register for the __int128 and marks only - * R1 through R3 at the entry of take_i128_global(), while the compiler passed - * a in R1, v in R2:R3 and c in R4. - */ -__failure __msg("R4 !read_ok") +__success __retval(0) int aggregate_arg_int128_c_test(struct __sk_buff *skb) { __u64 a = skb->len ^ MIX_A; -- knotmirror 0.0.1 From c6ff4e06033fede2fd1d41232ae942add649b759 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:32 -0700 Subject: [PATCH 07/15] bpf: Rename bpf_call_summary::num_params to arg_slot_cnt The field 'num_params' counts the argument registers and outgoing stack slots a helper or kfunc call takes. The next patch gives a 16-byte parameter two slots, so the name stops describing what the field holds. Rename 'num_params' to 'arg_slot_cnt'. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195232.988528-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 2 +- kernel/bpf/liveness.c | 10 +++++----- kernel/bpf/verifier.c | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 120a32bdd451..1b836c6d570f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1066,7 +1066,7 @@ static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) } struct bpf_call_summary { - u8 num_params; + u8 arg_slot_cnt; bool is_void; bool fastcall; }; diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 7165ea325961..44ecdc5b4ec2 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -1434,21 +1434,21 @@ static int record_call_access(struct bpf_verifier_env *env, { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_call_summary cs; - int r, err, num_params = 5; + int r, err, arg_slot_cnt = 5; if (bpf_pseudo_call(insn)) return 0; if (bpf_get_call_summary(env, insn, &cs)) - num_params = cs.num_params; + arg_slot_cnt = cs.arg_slot_cnt; - for (r = BPF_REG_1; r < BPF_REG_1 + min(num_params, MAX_BPF_FUNC_REG_ARGS); r++) { + for (r = BPF_REG_1; r < BPF_REG_1 + min(arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); r++) { err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx); if (err) return err; } - for (r = 0; r < MAX_STACK_ARG_SLOTS && r < num_params - MAX_BPF_FUNC_REG_ARGS; r++) { + for (r = 0; r < MAX_STACK_ARG_SLOTS && r < arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; r++) { err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r], r + MAX_BPF_FUNC_REG_ARGS, insn_idx); if (err) @@ -2199,7 +2199,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, def = ALL_CALLER_SAVED_REGS; use = def & ~BIT(BPF_REG_0); if (bpf_get_call_summary(env, insn, &cs)) - use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); + use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1); def = mask_widen(def); use = mask_widen(use); break; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d8963d50d85b..8c784e9fd8b7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17843,11 +17843,11 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, (bpf_verifier_inlines_helper_call(env, call->imm) || bpf_jit_inlines_helper_call(call->imm)); cs->is_void = fn->ret_type == RET_VOID; - cs->num_params = 0; + cs->arg_slot_cnt = 0; for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) { if (fn->arg_type[i] == ARG_UNUSED) break; - cs->num_params++; + cs->arg_slot_cnt++; } return true; } @@ -17859,7 +17859,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, if (err < 0) /* error would be reported later */ return false; - cs->num_params = btf_type_vlen(meta.func_proto); + cs->arg_slot_cnt = btf_type_vlen(meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; @@ -17968,7 +17968,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, * - includes R1-R5 if corresponding parameter has is described * in the function prototype. */ - clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0); + clobbered_regs_mask = GENMASK(cs.arg_slot_cnt, cs.is_void ? 1 : 0); /* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */ expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS; -- knotmirror 0.0.1 From ae6abae582b750418ae1b09d3b295c416c61e082 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:37 -0700 Subject: [PATCH 08/15] bpf: Recognize by-value struct and __int128 kfunc arguments A kfunc taking a struct or union by value is rejected today, and one taking an __int128 is accepted but mis-counted: Unrecognized R2 type STRUCT The kfunc arguments walk the same slot as a BPF-to-BPF call: one argument register per eightbyte, and a 16-byte value takes two. The outgoing stack argument count at the call site follows the slots for the same reason. Similar to BPF-to-BPF aggregate handling, a kfunc aggregate argument is only supported when it is composed of scalars. Everything that maps a kfunc argument to a register has to follow the slots too. An argument of a single eightbyte lands in the same place under every calling convention, so those are taken. The conventions the JIT has to reconcile do not agree on where a larger one goes, so refuse it for now with Function f arg#1 type INT cannot be passed at argument slot 1 on this architecture which the next patch turns into an answer from the JIT. The paths that handle a two-slot argument are therefore unreachable until then. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195237.988712-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 189 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 36 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8c784e9fd8b7..c33f1e1d1a1c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -8238,7 +8238,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, } static int resolve_func_arg_type(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 arg, + struct bpf_reg_state *reg, u32 arg, argno_t argno, struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type, u32 *arg_size); static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, @@ -8885,7 +8885,7 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat return 0; } -static int check_func_arg(struct bpf_verifier_env *env, u32 arg, +static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot, struct bpf_call_arg_meta *meta, int insn_idx) { @@ -8893,8 +8893,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, const struct bpf_func_proto *fn = meta->fn; struct bpf_func_state *caller = cur_func(env); struct bpf_reg_state *regs = cur_regs(env); - argno_t argno = argno_from_arg(arg + 1); - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, arg); + argno_t argno = argno_from_arg(slot + 1); + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot); enum bpf_arg_type arg_type = fn->arg_type[arg]; int regno = reg_from_argno(argno); u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; @@ -8925,7 +8925,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return 0; } - err = resolve_func_arg_type(env, reg, arg, meta, &arg_type, &arg_size); + err = resolve_func_arg_type(env, reg, arg, argno, meta, &arg_type, &arg_size); if (err) return err; @@ -9220,8 +9220,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, case ARG_MEM_SIZE: case ARG_MEM_SIZE_OR_ZERO: { - struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, arg - 1); - argno_t buff_argno = argno_from_arg(arg); + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, prev_slot); + argno_t buff_argno = argno_from_arg(prev_slot + 1); enum bpf_mem_size_failure failure; const char *buff_arg, *size_arg; bool zero_size_allowed; @@ -9448,9 +9448,52 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; } +/* + * The slots a parameter takes, from its BTF type. This has to agree with + * btf_func_model_arg_slots(), which answers the same from the size the + * func model recorded, or the verifier would check an argument at a slot + * the JIT does not place it at. + */ +static u32 kfunc_arg_slots(const struct btf_type *t) +{ + if (btf_type_is_int(t) || btf_type_is_struct(t)) + return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + return 1; +} + +static u32 kfunc_proto_slots(const struct btf *btf, const struct btf_type *func_proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0; + + for (i = 0; i < nargs; i++) + slots_used += kfunc_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + + return slots_used; +} + +/* The argument slot @slot holds an eightbyte of a by-value argument. */ +static int check_arg_extra_slot(struct bpf_verifier_env *env, struct bpf_func_state *caller, + u32 slot, struct bpf_call_arg_meta *meta) +{ + struct bpf_reg_state *reg = get_func_arg_reg(caller, cur_regs(env), slot); + argno_t argno = argno_from_arg(slot + 1); + int regno = reg_from_argno(argno); + int err; + + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } + + return check_reg_type(env, reg, argno, ARG_SCALAR, meta); +} + static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int insn_idx) { + u32 slot = 0, prev_slot = 0, proto_slots, nslots; struct bpf_func_state *caller = cur_func(env); const struct btf_param *args = NULL; u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS; @@ -9461,19 +9504,47 @@ static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_met nargs = btf_type_vlen(meta->func_proto); } - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - err = check_outgoing_stack_args(env, caller, nargs, meta->func_name, - meta->btf, args); + /* + * A by-value argument of more than one eightbyte takes a slot per + * eightbyte, so the slots a call occupies are no longer its parameter + * count. Only a proto whose parameters take a slot each can name the + * argument a stack slot belongs to. + */ + proto_slots = meta->btf ? kfunc_proto_slots(meta->btf, meta->func_proto) : nargs; + + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { + err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name, + meta->btf, proto_slots == nargs ? args : NULL); if (err) return err; } - for (arg = 0; arg < nargs; arg++) { + for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) { + const struct btf_type *t; + u32 k; + + nslots = 1; + if (args) { + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); + nslots = kfunc_arg_slots(t); + } + if (meta->fn->arg_type[arg] == ARG_UNUSED) break; - err = check_func_arg(env, arg, meta, insn_idx); + err = check_func_arg(env, arg, slot, prev_slot, meta, insn_idx); if (err) return err; + + /* + * check_func_arg() took the first slot. A parameter of more + * than one eightbyte is always a scalar, so every slot it + * takes beyond the first holds one too. + */ + for (k = 1; k < nslots; k++) { + err = check_arg_extra_slot(env, caller, slot + k, meta); + if (err) + return err; + } } return 0; @@ -12329,11 +12400,10 @@ static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, } static int resolve_func_arg_type(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, u32 arg, + struct bpf_reg_state *reg, u32 arg, argno_t argno, struct bpf_call_arg_meta *meta, enum bpf_arg_type *arg_type, u32 *arg_size) { - argno_t argno = argno_from_arg(arg + 1); const struct btf_param *args; const struct btf_type *ref_t, *resolve_ret; const struct btf *btf; @@ -12661,12 +12731,12 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) static int get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - const struct btf_param *args, int arg, int nargs, + const struct btf_param *args, int arg, int nargs, u32 slot, struct bpf_func_proto *proto) { const struct btf_type *t, *ref_t = NULL, *resolve_ret; const u32 *ref_id_ptr = NULL; - argno_t argno = argno_from_arg(arg + 1); + argno_t argno = argno_from_arg(slot + 1); const char *ref_tname = NULL; u32 ref_id, type_size; int arg_type; @@ -12695,6 +12765,23 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, return ARG_SCALAR; } + if (btf_type_is_struct(t)) { + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + verbose(env, + "%s type %s has size %u, only 1 to %d bytes " + "can be passed by value\n", + reg_arg_name(env, argno), btf_type_str(t), t->size, + 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (!btf_type_is_scalar_struct(env, meta->btf, t)) { + verbose(env, "%s type %s is not composed of scalars\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + return ARG_SCALAR; + } + if (!btf_type_is_ptr(t)) { verbose(env, "Unrecognized %s type %s\n", reg_arg_name(env, argno), btf_type_str(t)); @@ -12858,7 +12945,7 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg struct bpf_func_proto *proto) { const struct btf_param *args; - u32 i, nargs; + u32 i, nargs, slots_used; int arg_type; args = (const struct btf_param *)(meta->func_proto + 1); @@ -12868,20 +12955,41 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg nargs, MAX_BPF_FUNC_ARGS); return -EINVAL; } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - meta->func_name, nargs); - return -ENOTSUPP; - } - for (i = 0; i < nargs; i++) { - arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, proto); + for (i = 0, slots_used = 0; i < nargs; i++) { + const struct btf_type *t; + u32 nslots; + + t = btf_type_skip_modifiers(meta->btf, args[i].type, NULL); + nslots = kfunc_arg_slots(t); + /* + * The calling conventions the JIT has to reconcile do not + * agree on where an argument of more than one eightbyte goes, + * so refuse one until the JIT can say where this arch puts it. + */ + if (nslots > 1) { + verbose(env, + "Function %s arg#%d type %s cannot be passed at " + "argument slot %d on this architecture\n", + meta->func_name, i, btf_type_str(t), slots_used); + return -EINVAL; + } + slots_used += nslots; + + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, + slots_used - nslots, proto); if (arg_type < 0) return arg_type; proto->arg_type[i] = arg_type; } + if (slots_used > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d argument slots\n", + meta->func_name, slots_used); + return -ENOTSUPP; + } + return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } @@ -13621,7 +13729,7 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; - u32 nargs, type_size; + u32 i, slot, nargs, type_size; s64 size; if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0) @@ -13630,23 +13738,32 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * btf = meta.btf; args = btf_params(meta.func_proto); nargs = btf_type_vlen(meta.func_proto); - if (arg >= nargs) + + /* + * @arg is an argument slot and a 16-byte parameter takes two of them, + * so walk the parameters to find the one that starts at this slot. A + * slot holding the upper eightbyte of such a parameter belongs to no + * pointer, and neither does a slot past the last parameter. + */ + for (i = 0, slot = 0; i < nargs && slot < arg; i++) + slot += kfunc_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + if (i >= nargs || slot != arg) return 0; - t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + t = btf_type_skip_modifiers(btf, args[i].type, NULL); if (!btf_type_is_ptr(t)) return 0; /* dynptr: fixed 16-byte on-stack representation */ - if (is_kfunc_arg_dynptr(btf, &args[arg])) { + if (is_kfunc_arg_dynptr(btf, &args[i])) { size = BPF_DYNPTR_SIZE; goto out; } /* ptr + __sz/__szk pair: the size follows the pointer */ - if (arg + 1 < nargs && - (btf_param_match_suffix(btf, &args[arg + 1], "__sz") || - btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) { + if (i + 1 < nargs && + (btf_param_match_suffix(btf, &args[i + 1], "__sz") || + btf_param_match_suffix(btf, &args[i + 1], "__szk"))) { int size_reg = BPF_REG_1 + arg + 1; if (size_reg <= MAX_BPF_FUNC_REG_ARGS && @@ -13862,7 +13979,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id, ret_nregs = 1; + u32 i, proto_slots, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -14288,11 +14405,11 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); - nargs = btf_type_vlen(meta.func_proto); - if (nargs > MAX_BPF_FUNC_REG_ARGS) { + proto_slots = kfunc_proto_slots(desc_btf, meta.func_proto); + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; - u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; + u16 out_stack_arg_cnt = proto_slots - MAX_BPF_FUNC_REG_ARGS; u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt; if (stack_arg_cnt > caller_info->stack_arg_cnt) @@ -17859,7 +17976,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, if (err < 0) /* error would be reported later */ return false; - cs->arg_slot_cnt = btf_type_vlen(meta.func_proto); + cs->arg_slot_cnt = kfunc_proto_slots(meta.btf, meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; -- knotmirror 0.0.1 From 1e6c598b02e72ddd62b4d4e0c05159794b8bddbf Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:42 -0700 Subject: [PATCH 09/15] bpf: Prepare kfunc arguments for the JIT from an ABI description The previous patch refuses a kfunc argument of more than one eightbyte. This patch allows up to 16 byte kfunc arguments. But different architectures have different ways to map the BPF calling convention (no gap, no backfill) to the native one. Rather than have each arch open-code where it wants an argument, describe the convention with a register count and four booleans, and let each arch set what applies to it: struct bpf_jit_arg_abi { u8 nr_arg_regs; bool even_reg_align; bool even_stack_align; bool split_at_boundary; bool backfill_after_stack; }; The four booleans are meant to cover x86-64, arm64, RISC-V LP64 and PowerPC64 ELFv2, although only x86-64 and arm64 fill the struct in here. bpf_jit_place_args() and bpf_jit_plan_arg_moves() use that description to work out where each argument belongs and which slots the JIT then has to move, to be used in the JIT later on. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195242.989416-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 15 ++++++ include/linux/bpf_verifier.h | 1 + include/linux/filter.h | 32 ++++++++++++ kernel/bpf/btf.c | 3 ++ kernel/bpf/core.c | 94 ++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 99 +++++++++++++++++++++++++++++++----- 6 files changed, 231 insertions(+), 13 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d0066d744ceb..2a5fa346aada 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -993,6 +993,13 @@ static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); */ #define MAX_BPF_FUNC_REG_ARGS 5 +/* A by-value argument takes two eightbytes at most, so the maximum number of + * argument slots of any function is 2 * MAX_BPF_FUNC_ARGS. A local array may + * need that size for processing, although eventually the maximum slots will + * be capped at MAX_BPF_FUNC_ARGS. + */ +#define MAX_BPF_FUNC_ARG_SLOTS (2 * MAX_BPF_FUNC_ARGS) + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -1210,6 +1217,9 @@ struct bpf_prog_offload { u32 jited_len; }; +/* The argument is aligned to 16 bytes. */ +#define BTF_FMODEL_ALIGN16_ARG BIT(0) + /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) @@ -1227,6 +1237,11 @@ struct btf_func_model { u8 arg_flags[MAX_BPF_FUNC_ARGS]; }; +static inline u32 btf_func_model_arg_slots(const struct btf_func_model *m, u32 arg) +{ + return (m->arg_size[arg] + sizeof(u64) - 1) / sizeof(u64); +} + /* Restore arguments before returning from trampoline to let original function * continue executing. This flag is used for fentry progs when there are no * fexit progs. diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 1b836c6d570f..cf85141ea167 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1512,6 +1512,7 @@ enum btf_member_kind { bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, const struct btf_type *t, u32 member_kinds); +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/include/linux/filter.h b/include/linux/filter.h index 00ad8b63aa47..b17222db2efc 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1248,6 +1248,38 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); bool bpf_jit_supports_timed_may_goto(void); bool bpf_jit_supports_fsession(void); + +struct bpf_jit_arg_abi { + /* Argument registers of the kernel convention. */ + u8 nr_arg_regs; + /* Round the register number up to an even one for 16-byte alignment. */ + bool even_reg_align; + /* Round the stack slot up to an even one for 16-byte alignment. */ + bool even_stack_align; + /* An argument may straddle the last register and the stack. */ + bool split_at_boundary; + /* A later argument may reuse a register a stack-passed one skipped. */ + bool backfill_after_stack; +}; + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void); +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot); + +/* The JIT's scratch register, in place of an argument slot. */ +#define BPF_JIT_ARG_TMP 0xff + +/* Every argument slot moves at most once, and the scratch goes out and back. */ +#define BPF_JIT_MAX_ARG_MOVES (MAX_BPF_FUNC_ARG_SLOTS + 2) + +struct bpf_jit_arg_move { + u8 dst; + u8 src; +}; + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves); u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9f8a4e4aac3b..7daf4c286c9b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7579,6 +7579,9 @@ static u8 __get_arg_fmodel_flags(const struct btf *btf, { u8 flags = __get_type_fmodel_flags(t); + if (btf_func_arg_align(btf, t) > sizeof(u64)) + flags |= BTF_FMODEL_ALIGN16_ARG; + if (btf_param_match_suffix(btf, arg, "__arena__nullable")) flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; else if (btf_param_match_suffix(btf, arg, "__arena")) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index c673b02d55a6..4e208cc94752 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3287,6 +3287,100 @@ bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) return false; } +/* + * How this arch places a by-value kfunc argument, or NULL for one that has + * not opted in and so only takes an argument of a single eightbyte, which + * every convention places in slot order. + */ +const struct bpf_jit_arg_abi * __weak bpf_jit_arg_abi(void) +{ + return NULL; +} + +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot) +{ + u32 i, k, nslots, slot = 0, nregs_used = 0, stack_off = 0; + bool on_stack = false; + + for (i = 0; i < fm->nr_args; i++) { + bool align16 = fm->arg_flags[i] & BTF_FMODEL_ALIGN16_ARG; + u32 pos; + + nslots = btf_func_model_arg_slots(fm, i); + + if (align16 && abi->even_reg_align) + nregs_used = round_up(nregs_used, 2); + + if (!on_stack && nregs_used + nslots <= abi->nr_arg_regs) { + /* wholly in registers */ + pos = nregs_used; + nregs_used += nslots; + } else if (!on_stack && abi->split_at_boundary) { + /* the last registers hold what fits, the stack the rest */ + pos = nregs_used; + stack_off = (nregs_used + nslots - abi->nr_arg_regs) * BPF_REG_SIZE; + nregs_used = abi->nr_arg_regs; + on_stack = true; + } else { + /* wholly on the stack */ + if (align16 && abi->even_stack_align) + stack_off = round_up(stack_off, 2 * BPF_REG_SIZE); + pos = abi->nr_arg_regs + stack_off / BPF_REG_SIZE; + stack_off += nslots * BPF_REG_SIZE; + if (!abi->backfill_after_stack) + on_stack = true; + } + + for (k = 0; k < nslots; k++) + pos_of_slot[slot + k] = pos + k; + slot += nslots; + } + + return slot; +} + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves) +{ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 nslots, n = 0, s, back; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + back = nslots; + + /* + * An argument is two eightbytes at most, so it frees one register at + * most and only one argument ever moves down. Its destination is + * still in use, so carry it in the scratch. Only a lower slot can + * take the one it leaves, so the walk reaches it first. + */ + for (s = nslots; s > 0; s--) { + u8 slot = s - 1, pos = pos_of_slot[slot]; + + if (pos == slot) + continue; + + if (pos < slot) { + moves[n].dst = BPF_JIT_ARG_TMP; + back = slot; + } else { + moves[n].dst = pos; + } + moves[n].src = slot; + n++; + } + + if (back < nslots) { + moves[n].dst = pos_of_slot[back]; + moves[n].src = BPF_JIT_ARG_TMP; + n++; + } + + return n; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c33f1e1d1a1c..6c6b8d8520cd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2844,7 +2844,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto); + const struct btf_func_model *fm, struct bpf_func_proto *proto); int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { @@ -2961,7 +2961,7 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc = &tab->descs[tab->nr_descs]; memset(desc, 0, sizeof(*desc)); - err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + err = gen_kfunc_arg_proto(env, &meta, &func_model, &desc->proto); if (err) return err; @@ -12729,6 +12729,58 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]); } +static u32 kfunc_abi_slots(const struct btf_func_model *fm) +{ + const struct bpf_jit_arg_abi *abi = bpf_jit_arg_abi(); + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 i, nslots, slots = 0; + + for (i = 0; i < fm->nr_args; i++) + slots += btf_func_model_arg_slots(fm, i); + + if (!abi) + return slots; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + for (i = 0; i < nslots; i++) + if (pos_of_slot[i] + 1 > slots) + slots = pos_of_slot[i] + 1; + + return slots; +} + +static u32 __btf_func_arg_align(const struct btf *btf, const struct btf_type *t, int rec) +{ + const struct btf_member *member; + const struct btf_type *mt; + u32 align, i; + + while (btf_type_is_array(t)) + t = btf_type_skip_modifiers(btf, btf_array(t)->type, NULL); + + if (btf_type_is_int(t)) + return t->size > BPF_REG_SIZE ? t->size : BPF_REG_SIZE; + if (!btf_type_is_struct(t)) + return BPF_REG_SIZE; + if (rec >= BTF_MEMBER_MAX_DEPTH) + return 0; + + for_each_member(i, t, member) { + mt = btf_type_skip_modifiers(btf, member->type, NULL); + align = __btf_func_arg_align(btf, mt, rec + 1); + if (!align) + return 0; + if (align > BPF_REG_SIZE) + return 2 * BPF_REG_SIZE; + } + return BPF_REG_SIZE; +} + +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t) +{ + return __btf_func_arg_align(btf, t, 0); +} + static int get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, const struct btf_param *args, int arg, int nargs, u32 slot, @@ -12942,9 +12994,12 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto) + const struct btf_func_model *fm, struct bpf_func_proto *proto) { + const struct bpf_jit_arg_abi *abi; + const struct btf *btf = meta->btf; const struct btf_param *args; + const struct btf_type *t; u32 i, nargs, slots_used; int arg_type; @@ -12957,17 +13012,35 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg } for (i = 0, slots_used = 0; i < nargs; i++) { - const struct btf_type *t; - u32 nslots; + u32 nslots = btf_func_model_arg_slots(fm, i); - t = btf_type_skip_modifiers(meta->btf, args[i].type, NULL); - nslots = kfunc_arg_slots(t); - /* - * The calling conventions the JIT has to reconcile do not - * agree on where an argument of more than one eightbyte goes, - * so refuse one until the JIT can say where this arch puts it. - */ if (nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + if (!btf_func_arg_align(btf, t)) { + verbose(env, + "Function %s arg#%d type %s nests structs more than " + "%d levels deep\n", + meta->func_name, i, btf_type_str(t), + BTF_MEMBER_MAX_DEPTH); + return -EINVAL; + } + } + slots_used += nslots; + } + + if (slots_used > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s needs %d > %d argument slots\n", meta->func_name, + slots_used, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } + + abi = bpf_jit_arg_abi(); + + for (i = 0, slots_used = 0; i < nargs; i++) { + u32 nslots = btf_func_model_arg_slots(fm, i); + + if (!abi && nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); verbose(env, "Function %s arg#%d type %s cannot be passed at " "argument slot %d on this architecture\n", @@ -14405,7 +14478,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); - proto_slots = kfunc_proto_slots(desc_btf, meta.func_proto); + proto_slots = kfunc_abi_slots(&desc->func_model); if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; -- knotmirror 0.0.1 From a3195057406fbea09c784e8c97e86765989dee3b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:47 -0700 Subject: [PATCH 10/15] bpf, x86: Move kfunc arguments into the x86-64 calling convention The x86-64 allows 6 register arguments and then stack arguments, but backfilling the 6th register is possible, as in the following example: /* s is a 16-byte struct */ void kfunc(u64 a, u64 b, u64 c, u64 d, u64 e, struct big s, u64 f) BPF puts s in slots 5 and 6, R9 and the first stack slot, and f in slot 7, while x86-64 puts s in the two stack slots and f in R9. Emit the moves bpf_jit_plan_arg_moves() plans to bridge the two. Only one argument ever moves down, as an argument frees one register at most, so carrying that single value in the scratch register past its own destination is enough. In addition, the arena argument walk counts eightbytes rather than parameters, as an argument may take two registers. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195247.989857-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 78 +++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index bba351944202..2671e4118d00 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1839,6 +1839,65 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, return 0; } +static const struct bpf_jit_arg_abi x86_arg_abi = { + .nr_arg_regs = 6, + .backfill_after_stack = true, + .even_stack_align = true, +}; + +static const u8 x86_arg_reg[] = { + BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5, X86_REG_R9, +}; + +/* + * Move the arguments the x86-64 ABI places somewhere other than the argument + * slot the BPF calling convention gave them. @stack_base addresses the + * outgoing stack argument area from RBP. Return the number of emitted bytes. + */ +static int emit_kfunc_arg_moves(const struct btf_func_model *fm, s32 stack_base, u8 **pprog) +{ + struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES]; + const u8 nreg = x86_arg_abi.nr_arg_regs; + u8 *prog = *pprog, *start = prog; + u32 i, n; + + n = bpf_jit_plan_arg_moves(&x86_arg_abi, fm, moves); + + for (i = 0; i < n; i++) { + u8 dst = moves[i].dst, src = moves[i].src, reg; + bool dst_mem = dst != BPF_JIT_ARG_TMP && dst >= nreg; + bool src_mem = src != BPF_JIT_ARG_TMP && src >= nreg; + + /* + * Take the value into a register: the one it belongs in, the + * scratch when it is carried past its own destination, and + * BPF_REG_AX only to pass one stack slot to another. + */ + if (src == BPF_JIT_ARG_TMP) { + reg = AUX_REG; + } else if (src_mem) { + reg = dst == BPF_JIT_ARG_TMP ? AUX_REG : + dst_mem ? BPF_REG_AX : x86_arg_reg[dst]; + emit_ldx(&prog, BPF_DW, reg, BPF_REG_FP, + stack_base + (src - nreg) * 8); + } else { + reg = x86_arg_reg[src]; + } + + /* And leave it where the argument belongs. */ + if (dst == BPF_JIT_ARG_TMP) + emit_mov_reg(&prog, true, AUX_REG, reg); + else if (dst_mem) + emit_stx(&prog, BPF_DW, BPF_REG_FP, reg, + stack_base + (dst - nreg) * 8); + else if (reg != x86_arg_reg[dst]) + emit_mov_reg(&prog, true, x86_arg_reg[dst], reg); + } + + *pprog = prog; + return prog - start; +} + /* * Rebase the __arena args of a kfunc call to arena kernel addresses, * rN = kern_vm_start + (u32)rN, with R12 holding kern_vm_start. A nullable @@ -1850,11 +1909,17 @@ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, { u8 *prog = *pprog; u8 *start = prog; - int i; + int i, slot; - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = 0, slot = 0; i < fm->nr_args; i++) { + u32 arg_regs = (fm->arg_size[i] + 7) / 8; u8 flags = fm->arg_flags[i]; - u32 reg = BPF_REG_1 + i; + u32 reg; + + if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS) + break; + reg = BPF_REG_1 + slot; + slot += arg_regs; if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; @@ -2837,6 +2902,8 @@ populate_extable: if (err < 0) return err; ip += err; + ip += emit_kfunc_arg_moves(fm, outgoing_arg_base - + outgoing_rsp, &prog); } if (priv_frame_ptr) { push_r9(&prog); @@ -4351,6 +4418,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void) return true; } +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) +{ + return &x86_arg_abi; +} + bool bpf_jit_supports_stack_args(void) { return true; -- knotmirror 0.0.1 From afdbb4cdd3d2714c58c68aab131bd2a43b14feb4 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:52 -0700 Subject: [PATCH 11/15] bpf, arm64: Place trampoline arguments by the arm64 calling convention calc_arg_aux(), save_args() and restore_args() number argument registers and stack slots consecutively, while AAPCS64 rounds both up to an even one for an argument aligned to 16 bytes: an __int128, or an aggregate holding one. The argument that takes the hole, and every argument after it, is saved from and restored to the wrong place, so a bpf program attached to such a function reads a neighbouring eightbyte, and the arguments handed on to the original function are shifted. Take the position of each argument slot from bpf_jit_place_args() and keep it in struct arg_aux. save_args() then reads a slot from the register, or the incoming stack slot, and restore_args() puts it back there; the outgoing area built for the original function mirrors the incoming one, hole and all. A bpf program still cannot read an __int128 argument itself: it is an integer wider than eight bytes, which btf_ctx_access() refuses, so the program is rejected whatever the trampoline does with the argument. What a program can read, and what the placement puts right, is an aggregate holding an __int128, allowed as a struct, and any argument that follows a 16-byte aligned one. Both were taken from the wrong place before. Bound the argument slots rather than the arguments while here. btf_distill_func_proto() only limits the count although a by-value argument may take two slots, so, similar to x86-64, support up to MAX_BPF_FUNC_ARGS argument slots and refuse a function with more. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195252.990578-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit_comp.c | 78 ++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 3aa3ea0bc30b..25a7657a6710 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1220,6 +1220,12 @@ static int add_exception_handler(const struct bpf_insn *insn, return 0; } +static const struct bpf_jit_arg_abi arm64_arg_abi = { + .nr_arg_regs = 8, + .even_reg_align = true, + .even_stack_align = true, +}; + static const u8 stack_arg_reg[] = { A64_R(5), A64_R(6), A64_R(7) }; #define NR_STACK_ARG_REGS ARRAY_SIZE(stack_arg_reg) @@ -1293,6 +1299,16 @@ static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *ins return 0; } +static bool a64_arg_on_stack(u8 slot) +{ + return slot >= arm64_arg_abi.nr_arg_regs; +} + +static s32 a64_arg_stack_off(u8 slot) +{ + return (slot - arm64_arg_abi.nr_arg_regs) * sizeof(u64); +} + /* JITs an eBPF instruction. * Returns: * 0 - successfully JITed an 8-byte eBPF instruction. @@ -2529,33 +2545,41 @@ struct arg_aux { * arguments to be properly aligned) */ int ostack_for_args; + /* where AAPCS64 puts each argument slot: an argument register below + * the eighth, an on-stack argument slot from it up + */ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; }; static int calc_arg_aux(const struct btf_func_model *m, struct arg_aux *a) { - int stack_slots, nregs, slots, i; + int slots, i, slot, total; + + total = bpf_jit_place_args(&arm64_arg_abi, m, a->pos_of_slot); + if (total > MAX_BPF_FUNC_ARGS) + return -ENOTSUPP; /* verifier ensures m->nr_args <= MAX_BPF_FUNC_ARGS */ - for (i = 0, nregs = 0; i < m->nr_args; i++) { + for (i = 0, slot = 0; i < m->nr_args; i++) { slots = (m->arg_size[i] + 7) / 8; - if (nregs + slots <= 8) /* passed through register ? */ - nregs += slots; - else + if (a64_arg_on_stack(a->pos_of_slot[slot])) /* passed through register ? */ break; + slot += slots; } a->args_in_regs = i; - a->regs_for_args = nregs; + a->regs_for_args = slot; a->ostack_for_args = 0; a->bstack_for_args = 0; /* the rest arguments are passed through stack */ - for (; i < m->nr_args; i++) { - stack_slots = (m->arg_size[i] + 7) / 8; - a->bstack_for_args += stack_slots * 8; - a->ostack_for_args = a->ostack_for_args + stack_slots * 8; - } + for (; i < m->nr_args; i++) + a->bstack_for_args += ((m->arg_size[i] + 7) / 8) * 8; + + /* the outgoing area reaches the last slot, over any alignment hole */ + if (a->bstack_for_args) + a->ostack_for_args = a64_arg_stack_off(a->pos_of_slot[total - 1]) + 8; return 0; } @@ -2602,7 +2626,7 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, { u8 tmp = bpf2a64[TMP_REG_1]; u8 base_lo = bpf2a64[TMP_REG_2]; - int i, reg, doff, soff, slots; + int i, reg, slot, soff, slots; /* only the low 32 bits of the base take part in the subtraction */ if (arena_base) @@ -2611,12 +2635,13 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, /* store arguments to the stack for the bpf program, or restore * arguments from stack for the original function */ - for (i = 0, reg = 0; i < a->args_in_regs; i++) { + for (i = 0, slot = 0; i < a->args_in_regs; i++) { bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; slots = (m->arg_size[i] + 7) / 8; while (slots-- > 0) { + reg = a->pos_of_slot[slot++]; if (for_call_origin) { emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); } else if (arena_arg) { @@ -2625,7 +2650,6 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, } else { emit(A64_STR64I(reg, A64_SP, bargs_off), ctx); } - reg++; bargs_off += 8; } } @@ -2637,9 +2661,11 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops * callback is called indirectly and only the FP/LR frame is saved, so * they start at FP + 16. + * + * The outgoing area mirrors the incoming one, hole and all; only the + * bpf program takes the arguments packed. */ soff = is_struct_ops ? 16 : 32; - doff = (for_call_origin ? oargs_off : bargs_off); /* save on stack arguments */ for (i = a->args_in_regs; i < m->nr_args; i++) { @@ -2649,7 +2675,9 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, slots = (m->arg_size[i] + 7) / 8; /* verifier ensures arg_size <= 16, so slots equals 1 or 2 */ while (slots-- > 0) { - emit(A64_LDR64I(tmp, A64_FP, soff), ctx); + int off = a64_arg_stack_off(a->pos_of_slot[slot++]); + + emit(A64_LDR64I(tmp, A64_FP, soff + off), ctx); /* if there is unused space in the last slot, clear * the garbage contained in the space. */ @@ -2664,19 +2692,21 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, */ if (arena_arg) emit_arena_arg_conv(ctx, tmp, tmp, nullable, base_lo); - emit(A64_STR64I(tmp, A64_SP, doff), ctx); - soff += 8; - doff += 8; + if (for_call_origin) + emit(A64_STR64I(tmp, A64_SP, oargs_off + off), ctx); + else + emit(A64_STR64I(tmp, A64_SP, bargs_off), ctx); + bargs_off += 8; } } } -static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs) +static void restore_args(struct jit_ctx *ctx, int bargs_off, const struct arg_aux *a) { - int reg; + int slot; - for (reg = 0; reg < nregs; reg++) { - emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); + for (slot = 0; slot < a->regs_for_args; slot++) { + emit(A64_LDR64I(a->pos_of_slot[slot], A64_SP, bargs_off), ctx); bargs_off += 8; } } @@ -2948,7 +2978,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_args(ctx, bargs_off, a->regs_for_args); + restore_args(ctx, bargs_off, a); /* restore callee saved register x19 and x20 */ emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx); -- knotmirror 0.0.1 From e1891a2c64747fbe9ee236a6aa45e9203b02f99d Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:52:57 -0700 Subject: [PATCH 12/15] bpf, arm64: Move kfunc arguments into the arm64 calling convention Do the proper move from the BPF calling convention to the arm64 calling convention to satisfy the native requirement. AAPCS64 only ever moves an argument to a higher slot, so the moves need one scratch register to carry an eightbyte from one stack slot to another, and never the one a convention moving an argument down would need. In addition, the arena argument walk counts eightbytes rather than parameters, as an argument may take two registers. The walk takes the func model from the caller now, as the moves need it too, and runs first so that they carry the rebased value. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195257.990814-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit_comp.c | 68 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 25a7657a6710..6c04fee46876 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1268,19 +1268,20 @@ static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct * kern_vm_start. A nullable arg preserves NULL by skipping the add, tested * on the truncated value as arena NULL is offset 0. */ -static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *insn) +static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct btf_func_model *fm) { const u8 arena_vm_base = bpf2a64[ARENA_VM_START]; - const struct btf_func_model *fm; - int i; - - fm = bpf_jit_find_kfunc_model(ctx->prog, insn); - if (!fm) - return -EINVAL; + int i, slot; - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { - const u8 reg = bpf2a64[BPF_REG_1 + i]; + for (i = 0, slot = 0; i < fm->nr_args; i++) { + u32 arg_regs = (fm->arg_size[i] + 7) / 8; u8 flags = fm->arg_flags[i]; + u8 reg; + + if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS) + break; + reg = bpf2a64[BPF_REG_1 + slot]; + slot += arg_regs; if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; @@ -1309,6 +1310,42 @@ static s32 a64_arg_stack_off(u8 slot) return (slot - arm64_arg_abi.nr_arg_regs) * sizeof(u64); } +/* + * Move the arguments AAPCS64 places somewhere other than the argument slot the + * BPF calling convention gave them. Slot N is X(N) up to the eighth, and the + * outgoing stack argument area from SP beyond it, both for the slot an + * argument comes from and for the one it goes to. + * + * AAPCS64 only ever moves an argument to a higher slot, so no move here ever + * takes BPF_JIT_ARG_TMP: bpf_jit_plan_arg_moves() hands out the scratch only + * for a convention that moves one down, which needs a register to carry the + * value past its own destination. + */ +static void emit_kfunc_arg_moves(struct jit_ctx *ctx, const struct btf_func_model *fm) +{ + struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES]; + const u8 tmp = bpf2a64[TMP_REG_1]; + u32 i, n; + + n = bpf_jit_plan_arg_moves(&arm64_arg_abi, fm, moves); + + for (i = 0; i < n; i++) { + u8 dst = moves[i].dst, src = moves[i].src, reg; + + if (a64_arg_on_stack(src)) { + reg = tmp; + emit(A64_LDR64I(reg, A64_SP, a64_arg_stack_off(src)), ctx); + } else { + reg = src; + } + + if (a64_arg_on_stack(dst)) + emit(A64_STR64I(reg, A64_SP, a64_arg_stack_off(dst)), ctx); + else if (reg != dst) + emit(A64_MOV(1, dst, reg), ctx); + } +} + /* JITs an eBPF instruction. * Returns: * 0 - successfully JITed an 8-byte eBPF instruction. @@ -1732,9 +1769,15 @@ emit_cond_jmp: if (ret < 0) return ret; if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { - ret = emit_kfunc_arena_args(ctx, insn); + const struct btf_func_model *fm; + + fm = bpf_jit_find_kfunc_model(ctx->prog, insn); + if (!fm) + return -EINVAL; + ret = emit_kfunc_arena_args(ctx, fm); if (ret < 0) return ret; + emit_kfunc_arg_moves(ctx, fm); } emit_call(func_addr, ctx); /* @@ -2409,6 +2452,11 @@ bool bpf_jit_supports_kfunc_ret_reg_pair(void) return true; } +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) +{ + return &arm64_arg_abi; +} + bool bpf_jit_supports_stack_args(void) { return true; -- knotmirror 0.0.1 From 9b55be31d16d962bb5b1fa063a2a4ba780214743 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:53:02 -0700 Subject: [PATCH 13/15] selftests/bpf: Add C tests for by-value arguments up to 16 bytes Extend the by-value argument test with the aggregate cases, written in C so that they depend on the compiler lowering the argument into a pair of argument registers rather than on a hand-written register layout. The programs cover a struct and a union that fill two registers, a smaller struct that fills one, and two struct arguments in a row, alongside the __int128 already there. Each has an int argument around it so that a wrong slot count shows up as a wrong value in the parameters beside it; two pairs leave room for only one, which follows them. A global function taking a struct with a pointer member is rejected: the callee would receive the pointer as an opaque scalar. A struct the argument registers cannot hold reaches the callee partly on the stack, which the interpreter does not implement, so that case is loaded only when the JIT is on. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195302.991273-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../bpf/progs/verifier_aggregate_arg.c | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c index fc7c1b18bc40..b0ecccede47e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c @@ -7,6 +7,131 @@ #define MIX_A 0xdeadbeefcafef00dULL #define MIX_B 0x0123456789abcdefULL +struct pair { + __u64 lo; + __u64 hi; +}; + +struct small { + __u32 a; + __u32 b; +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; + __u64 hi; + } parts; +}; + +struct with_ptr { + void *p; + __u64 x; +}; + +static __noinline __u64 take_pair(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_static_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + + if (take_pair(1, p, 2) != a + b + 3) + return 1; + + return 0; +} + +#if defined(__clang__) + +__noinline __u64 take_pair_global(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + + if (take_pair_global(1, p, 2) != a + b + 3) + return 1; + + return 0; +} + +__noinline __u64 take_two_pairs_global(struct pair p, struct pair q, int d) +{ + return p.lo + p.hi + q.lo + q.hi + d; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_two_structs_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + struct pair q = { .lo = a + 1, .hi = b + 2 }; + + if (take_two_pairs_global(p, q, 3) != 2 * a + 2 * b + 6) + return 1; + + return 0; +} + +__noinline __u64 take_small_global(int a, struct small s, int c) +{ + return (__u64)a + s.a + s.b + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_small_struct_c_test(struct __sk_buff *skb) +{ + __u32 a = skb->len ^ (__u32)MIX_A; + __u32 b = skb->len ^ (__u32)MIX_B; + struct small s = { .a = a, .b = b }; + + if (take_small_global(1, s, 2) != (__u64)a + b + 3) + return 1; + + return 0; +} + +__noinline __u64 take_upair_global(int a, union upair u, int c) +{ + return (__u64)a + u.parts.lo + u.parts.hi + c; +} + +SEC("tc") +__success __retval(0) +int aggregate_arg_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair u; + + u.halves[0] = a; + u.halves[1] = b; + if (take_upair_global(1, u, 2) != a + b + 3) + return 1; + + return 0; +} + +#endif + #ifdef __SIZEOF_INT128__ typedef unsigned __int128 u128; @@ -32,4 +157,44 @@ int aggregate_arg_int128_c_test(struct __sk_buff *skb) #endif /* __SIZEOF_INT128__ */ +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +static __noinline __u64 take_spilled_pair(int a, int b, int c, int d, struct pair p) +{ + return (__u64)a + b + c + d + p.lo + p.hi; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_spilled_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p = { .lo = a, .hi = b }; + int n = skb->len; + + if (take_spilled_pair(n, n + 1, n + 2, n + 3, p) != a + b + 4 * n + 6) + return 1; + + return 0; +} + +#endif + +__noinline __u64 take_with_ptr_global(struct with_ptr s) +{ + return s.x; +} + +SEC("tc") +__failure __msg("type STRUCT in take_with_ptr_global() is not composed of scalars") +int aggregate_arg_ptr_member_fail(struct __sk_buff *skb) +{ + struct with_ptr s = { .p = skb, .x = skb->len }; + + return take_with_ptr_global(s); +} + char _license[] SEC("license") = "GPL"; -- knotmirror 0.0.1 From a7a97c37c58fea0b0329b808d44dc7b0403db187 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:53:07 -0700 Subject: [PATCH 14/15] selftests/bpf: Add inline-asm tests for by-value arguments The tests cover a struct passed in a register pair, a pointer in one half of it refused at the call site, a struct too large to pass by value, and four placements a global function cannot have: six scalars, a struct split between the last argument register and the stack, one wholly past the registers, and an __int128 whose two slots push the last parameter out. The six-scalar case is the one whose slot count is known from the parameters alone, so it takes the check btf_prepare_func_args() makes before it walks them, while the other three take the one it makes after. Both report the same way. GCC passes an aggregate by invisible reference, so a callee it compiles expects a pointer where BTF says the halves of the struct are, and those tests are left to clang. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195308.992139-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/aggregate_arg.c | 9 + .../selftests/bpf/progs/aggregate_arg_func.c | 188 ++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/aggregate_arg.c create mode 100644 tools/testing/selftests/bpf/progs/aggregate_arg_func.c diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c new file mode 100644 index 000000000000..b230f3bd3b2a --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include "aggregate_arg_func.skel.h" + +void test_aggregate_arg(void) +{ + RUN_TESTS(aggregate_arg_func); +} diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_func.c b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c new file mode 100644 index 000000000000..d0a4f84a6fbf --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_func.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +#ifdef __SIZEOF_INT128__ +typedef unsigned __int128 u128; +#endif + +struct pair { + __u64 lo; + __u64 hi; +}; + +struct too_big { + __u64 a; + __u64 b; + __u64 c; +}; + +#if defined(__clang__) + +__noinline __u64 global_arg_pair(int a, struct pair p, int c) +{ + return (__u64)a + p.lo + p.hi + c; +} + +SEC("tc") +__success __retval(0x33) +__naked int aggregate_arg_pair_asm(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 0x10;" /* p.lo */ + "r3 = 0x20;" /* p.hi */ + "r4 = 2;" + "call %[global_arg_pair];" + "exit;" + : + : __imm(global_arg_pair) + : __clobber_all); +} + +SEC("tc") +__failure __msg("R2 is not a scalar") +__naked int aggregate_arg_pair_ptr_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = r10;" /* a stack pointer where p.lo belongs */ + "r3 = 0x20;" + "r4 = 2;" + "call %[global_arg_pair];" + "exit;" + : + : __imm(global_arg_pair) + : __clobber_all); +} + +#endif + +__noinline __u64 global_arg_too_big(struct too_big s) +{ + return s.a + s.b + s.c; +} + +SEC("tc") +__failure __msg("in global_arg_too_big() has size 24, only 1 to 16 bytes can be passed by value") +__naked int aggregate_arg_too_big_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "call %[global_arg_too_big];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_too_big) + : __clobber_all); +} + +#if defined(__BPF_FEATURE_STACK_ARGUMENT) + +/* + * One slot per parameter, so the slot count is settled before the parameters + * are walked. The cases below reach the same count only once they have been. + */ +__noinline __u64 global_arg_six_scalars(int a, int b, int c, int d, int e, int f) +{ + return (__u64)a + b + c + d + e + f; +} + +SEC("tc") +__failure __msg("global function global_arg_six_scalars() needs 6 > 5 argument slots") +__naked int aggregate_arg_six_scalars_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_six_scalars];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_six_scalars) + : __clobber_all); +} + +__noinline __u64 global_arg_split(int a, int b, int c, int d, struct pair p) +{ + return (__u64)a + b + c + d + p.lo + p.hi; +} + +SEC("tc") +__failure __msg("global function global_arg_split() needs 6 > 5 argument slots") +__naked int aggregate_arg_split_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_split];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_split) + : __clobber_all); +} + +__noinline __u64 global_arg_past_regs(struct pair p, struct pair q, int a, struct pair r) +{ + return p.lo + p.hi + q.lo + q.hi + a + r.lo + r.hi; +} + +SEC("tc") +__failure __msg("global function global_arg_past_regs() needs 7 > 5 argument slots") +__naked int aggregate_arg_past_regs_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_past_regs];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_past_regs) + : __clobber_all); +} + +#ifdef __SIZEOF_INT128__ + +__noinline __u64 global_arg_i128_slots(u128 v, int a, int b, int c, int d) +{ + return (__u64)v + a + b + c + d; +} + +SEC("tc") +__failure __msg("global function global_arg_i128_slots() needs 6 > 5 argument slots") +__naked int aggregate_arg_i128_slots_fail(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call %[global_arg_i128_slots];" + "r0 = 0;" + "exit;" + : + : __imm(global_arg_i128_slots) + : __clobber_all); +} + +#endif /* __SIZEOF_INT128__ */ + +#endif + +char _license[] SEC("license") = "GPL"; -- knotmirror 0.0.1 From 4006c4b06c2cb3c00c67688bd0da2cd3c95be210 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Sat, 12 Sep 2026 12:53:13 -0700 Subject: [PATCH 15/15] selftests/bpf: Add tests for by-value kfunc arguments Add kfuncs taking a 16-byte struct and an __int128 by value, with combinations of <= 8 byte arguments and '> 8 && <= 16' byte arguments. Each kfunc weighs its parameters by argument slot, the first by one, the second by two and so on, and every test checks the value it returns. A plain sum would be the same whichever slot each value reached, so an argument that lands in the wrong one, or a 16-byte argument whose halves arrive the other way round, would pass quietly; a weighted one differs. An __int128 takes two argument slots. One test passes it in registers and one past them, where both conventions pad the stack to align it although the BPF convention does not, so both JITs move it up an eightbyte. Two more cover a rejection. An aggregate holding a pointer is refused everywhere, and in arena_kfunc.c a two-slot struct pushes an arena pointer past the argument registers, which is refused too. An aggregate too large to pass by value is already covered in aggregate_arg_func.c. test_stack_arg_big() in stack_arg_fail.c passed a 16-byte struct as the sixth argument and asserted the unrecognized stack argument type it used to be reported as. The JIT places that argument now, so the test is removed. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260912195313.992803-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/aggregate_arg.c | 2 + .../selftests/bpf/progs/aggregate_arg_kfunc.c | 168 ++++++++++++++++++ .../testing/selftests/bpf/progs/arena_kfunc.c | 15 ++ .../selftests/bpf/progs/stack_arg_fail.c | 10 -- .../selftests/bpf/test_kmods/bpf_testmod.c | 59 +++++- .../bpf/test_kmods/bpf_testmod_kfunc.h | 27 +++ 6 files changed, 270 insertions(+), 11 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c index b230f3bd3b2a..aa7562f48737 100644 --- a/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_arg.c @@ -2,8 +2,10 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include #include "aggregate_arg_func.skel.h" +#include "aggregate_arg_kfunc.skel.h" void test_aggregate_arg(void) { RUN_TESTS(aggregate_arg_func); + RUN_TESTS(aggregate_arg_kfunc); } diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c new file mode 100644 index 000000000000..52d3d5d53946 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#ifdef __SIZEOF_INT128__ +typedef unsigned __int128 u128; +#endif + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +#if defined(__clang__) + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_struct(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg(1, s, 2) != 2 * a + 3 * b + 9) + return 1; + + return 0; +} + +#endif + +#ifdef __SIZEOF_INT128__ + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg(1, 2, v) != 4 * a + 3 * b + 5) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_last_regs(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_nofit(1, 2, 3, 4, s) != 5 * a + 6 * b + 30) + return 1; + + return 0; +} + +/* + * The x86-64 ABI moves an argument its six remaining registers cannot hold + * wholly onto the stack. arm64, with eight argument registers, still has a + * pair for it. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_straddle(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_big_arg s = { .a = a, .b = b }; + + if (bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s) != 6 * a + 7 * b + 55) + return 1; + + return 0; +} + +/* The same, with an argument after the struct to take the eighth register. */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_tail(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_tail(1, 2, 3, 4, 5, s, 6) != 6 * a + 7 * b + 103) + return 1; + + return 0; +} + +/* + * arm64 gives no register to an argument its eight registers cannot hold, + * nor to anything after it. Past its six registers the x86-64 ABI has both + * eightbytes on the stack either way. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_split8(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + + if (bpf_kfunc_call_test_pair_arg_split8(1, 2, 3, 4, 5, 6, 7, s) != 8 * a + 9 * b + 140) + return 1; + + return 0; +} + +#ifdef __SIZEOF_INT128__ + +/* + * Both conventions pad the stack to align this __int128, and the BPF + * convention pads for neither, so both JITs move it up an eightbyte. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_pad(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg_pad(1, 2, 3, 4, 5, 6, 7, v) != 9 * a + 8 * b + 140) + return 1; + + return 0; +} + +#endif /* __SIZEOF_INT128__ */ + +#endif + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R1 type STRUCT is not composed of scalars") +int aggregate_arg_kfunc_ptr_member(struct __sk_buff *skb) +{ + struct prog_test_ptr_arg s = { .p = skb, .x = 1 }; + + return bpf_kfunc_call_test_ptr_arg(s); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index 6578cf12fa27..96f7aacf4d9f 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -228,6 +228,21 @@ int arena_arg_stack(void *ctx) bpf_kfunc_arena_stack_arg_test(1, 2, 3, 4, 5, (u64 *)1); return 0; } + +#if defined(__clang__) +/* The struct takes two slots, so the arena pointer is the sixth. */ +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("arena pointer cannot be a stack argument") +int arena_arg_stack_after_pair(void *ctx) +{ + struct prog_test_pair_arg s = { .lo = 1, .hi = 2 }; + + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_call_test_pair_arena_arg(1, 2, 3, s, (u64 *)1); + return 0; +} +#endif #else SEC("syscall") __arch_x86_64 diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c index eed97d582515..fff2e947ea33 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_fail.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c @@ -3,20 +3,10 @@ #include #include -#include "../test_kmods/bpf_testmod_kfunc.h" #include "bpf_misc.h" #if defined(__BPF_FEATURE_STACK_ARGUMENT) -SEC("tc") -__failure __msg("Unrecognized *(R11-8) type STRUCT") -int test_stack_arg_big(struct __sk_buff *skb) -{ - struct prog_test_big_arg s = { .a = 1, .b = 2 }; - - return bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s); -} - SEC("socket") __description("r11 in ALU instruction") __failure __msg("R11 is invalid") diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index f798bbbb4d13..66b14168566a 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -1029,6 +1029,55 @@ __bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(u64 a, u6 return r; } +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg(u64 a, struct prog_test_pair_arg s, u64 b) +{ + return a + s.lo * 2 + s.hi * 3 + b * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg(u64 a, u64 b, __int128 v) +{ + return a + b * 2 + (u64)v * 3 + (u64)((unsigned __int128)v >> 64) * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_pad(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, __int128 v) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + (u64)v * 8 + (u64)((unsigned __int128)v >> 64) * 9; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_nofit(u64 a, u64 b, u64 c, u64 d, + struct prog_test_pair_arg s) +{ + return a + b * 2 + c * 3 + d * 4 + s.lo * 5 + s.hi * 6; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_tail(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_pair_arg s, u64 f) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 + f * 8; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arg_split8(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 f, u64 g, + struct prog_test_pair_arg s) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + s.lo * 8 + s.hi * 9; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_ptr_arg(struct prog_test_ptr_arg s) +{ + return s.x; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_pair_arena_arg(u64 a, u64 b, u64 c, + struct prog_test_pair_arg s, + u64 *f__arena) +{ + return a + b + c + s.lo + s.hi + *f__arena; +} + __bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) { struct prog_test_ret_ptr r = { .p = NULL, .tag = tag }; @@ -1186,7 +1235,7 @@ __bpf_kfunc u64 bpf_kfunc_call_stack_arg_timer(u64 a, u64 b, u64 c, u64 d, u64 e __bpf_kfunc u64 bpf_kfunc_call_stack_arg_big(u64 a, u64 b, u64 c, u64 d, u64 e, struct prog_test_big_arg s) { - return a + b + c + d + e + s.a + s.b; + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.a * 6 + s.b * 7; } static struct prog_test_ref_kfunc prog_test_struct = { @@ -1667,6 +1716,14 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr2d) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_pad) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_nofit) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_tail) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_split8) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ptr_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arena_arg) #endif BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index b213ef14848b..d3696d5254c9 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -61,6 +61,16 @@ struct prog_test_big_arg { __u64 b; }; +struct prog_test_pair_arg { /* 16 bytes: two argument registers */ + __u64 lo; + __u64 hi; +}; + +struct prog_test_ptr_arg { /* 16 bytes, but holds a pointer */ + void *p; + __u64 x; +}; + struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ __u64 lo; __u64 hi; @@ -221,6 +231,23 @@ __int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; +__u64 bpf_kfunc_call_test_pair_arg(__u64 a, struct prog_test_pair_arg s, __u64 b) __ksym; +#ifdef __SIZEOF_INT128__ +__u64 bpf_kfunc_call_test_i128_arg(__u64 a, __u64 b, __int128 v) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_pad(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, __int128 v) __ksym; +#endif +__u64 bpf_kfunc_call_test_pair_arg_nofit(__u64 a, __u64 b, __u64 c, __u64 d, + struct prog_test_pair_arg s) __ksym; +__u64 bpf_kfunc_call_test_pair_arg_tail(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + struct prog_test_pair_arg s, __u64 f) __ksym; +__u64 bpf_kfunc_call_test_pair_arg_split8(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 f, __u64 g, + struct prog_test_pair_arg s) __ksym; +__u64 bpf_kfunc_call_test_ptr_arg(struct prog_test_ptr_arg s) __ksym; +__u64 bpf_kfunc_call_test_pair_arena_arg(__u64 a, __u64 b, __u64 c, + struct prog_test_pair_arg s, + __u64 *f__arena) __ksym; struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; -- knotmirror 0.0.1