From 72c0681c0e75aa4fe6ce46ac0bb63fb5be9132d7 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:00 +0000 Subject: [PATCH 01/15] bpf: Allow bitwise ops, shifts and mul/div on pointers with CAP_PERFMON Rust's core::fmt keeps a flag in the low bit of a pointer: r2 = *(u64 *)(r1 + 0) r3 = r2 r3 &= 1 r2 >>= 1 The verifier rejects it: r0 &= 8 R0 bitwise operator &= on pointer prohibited Negation and byte swap of a pointer already produce a number when allow_ptr_leaks is set. Do the same for bitwise ops, shifts, *=, /= and %=, 64-bit and 32-bit. The result is an unknown number. With CAP_PERFMON the program can store the pointer and load it back as a number already. += and -= are not changed: they keep the pointer, 32-bit += is rejected. Pointers that allow no arithmetic and pointers that may be NULL are still rejected. Two tests in verifier_value_illegal_alu store through the result of &= and /=. They are rejected at the store now. Update expected messages. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-2-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 13 ++++++++++++- .../bpf/progs/verifier_value_illegal_alu.c | 8 ++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c1fc00eb8487..866c0540c68a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -15571,8 +15571,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn u32 dst = insn->dst_reg; const char *reason; int ret, bounds_ret; + bool to_scalar; dst_reg = ®s[dst]; + /* + * Only += and -= keep a pointer. Any other op makes a number of it, + * which is fine when the program may see values of pointers anyway. + */ + to_scalar = opcode != BPF_ADD && opcode != BPF_SUB && env->allow_ptr_leaks; if ((known && (smin_val != smax_val || umin_val != umax_val)) || smin_val > smax_val || umin_val > umax_val) { @@ -15583,7 +15589,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return 0; } - if (BPF_CLASS(insn->code) != BPF_ALU64) { + if (BPF_CLASS(insn->code) != BPF_ALU64 && !to_scalar) { /* 32-bit ALU ops on pointers produce (meaningless) scalars */ if (opcode == BPF_SUB && env->allow_ptr_leaks) { __mark_reg_unknown(env, dst_reg); @@ -15649,6 +15655,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return -EACCES; } + if (to_scalar) { + __mark_reg_unknown(env, dst_reg); + return 0; + } + /* For 'scalar += pointer', dst_reg inherits the complete pointer * register state. Individual fields may be adjusted later by pointer * arithmetic. Callers guarantee that below does not overwrite off_reg. diff --git a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c index 4d8273c258d5..9f669cf85c59 100644 --- a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c +++ b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c @@ -22,8 +22,8 @@ struct { SEC("socket") __description("map element value illegal alu op, 1") -__failure __msg("R0 bitwise operator &= on pointer") -__failure_unpriv +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 bitwise operator &= on pointer") __naked void value_illegal_alu_op_1(void) { asm volatile (" \ @@ -70,8 +70,8 @@ l0_%=: exit; \ SEC("socket") __description("map element value illegal alu op, 3") -__failure __msg("R0 pointer arithmetic with /= operator") -__failure_unpriv +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 pointer arithmetic with /= operator") __naked void value_illegal_alu_op_3(void) { asm volatile (" \ -- knotmirror 0.0.1 From 7ba7c5b3989ed28617399a47fc70c292ff606b23 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:01 +0000 Subject: [PATCH 02/15] selftests/bpf: Add tests for ALU on pointers with CAP_PERFMON Check that &=, |=, ^=, >>=, *= and 32-bit &= on a pointer to map value produce a number for a program with CAP_PERFMON and that the number can't be dereferenced. They are rejected without CAP_PERFMON, with CAP_BPF or without it. Pointer that may be NULL and pointer to map are rejected as before. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-3-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/progs/verifier_value_illegal_alu.c | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c index 9f669cf85c59..31663338866d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c +++ b/tools/testing/selftests/bpf/progs/verifier_value_illegal_alu.c @@ -165,6 +165,171 @@ __naked void map_ptr_illegal_alu_op(void) : __clobber_all); } +SEC("socket") +__description("tag in the low bit of a pointer, and, shift") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 bitwise operator &= on pointer") +__naked void ptr_tag_and_shift(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = r0; \ + r1 &= 1; \ + r2 = r0; \ + r2 >>= 1; \ + r3 = r0; \ + r3 |= 1; \ + r3 ^= 1; \ + r0 = *(u32*)(r0 + 0); \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("tag in the low bit of a pointer, CAP_BPF without CAP_PERFMON") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 bitwise operator &= on pointer") +__caps_unpriv(CAP_BPF) +__naked void ptr_tag_cap_bpf(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = r0; \ + r1 &= 1; \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("number op= pointer") +__success __retval(0) +__failure_unpriv __msg_unpriv("R1 pointer arithmetic with *= operator") +__naked void number_mul_ptr(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = 7; \ + r1 *= r0; \ + r0 = 0; \ +l0_%=: exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("pointer with the tag cleared is a number") +__failure __msg("R0 invalid mem access 'scalar'") +__failure_unpriv __msg_unpriv("R0 bitwise operator |= on pointer") +__naked void ptr_tag_cleared_deref(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r0 |= 1; \ + r0 ^= 1; \ + r0 = *(u32*)(r0 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("shift of a pointer that may be NULL") +__failure __msg("R0 pointer arithmetic on map_value_or_null prohibited, null-check it first") +__failure_unpriv +__naked void ptr_or_null_shift(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + r0 >>= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("and of a pointer to map") +__failure __msg("R0 pointer arithmetic on map_ptr prohibited") +__failure_unpriv +__naked void map_ptr_and(void) +{ + asm volatile (" \ + r0 = %[map_hash_48b] ll; \ + r0 &= 1; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(map_hash_48b) + : __clobber_all); +} + +SEC("socket") +__description("32-bit and of a pointer") +__success __retval(0) +__failure_unpriv __msg_unpriv("R0 32-bit pointer arithmetic prohibited") +__naked void ptr_and32(void) +{ + asm volatile (" \ + r2 = r10; \ + r2 += -8; \ + r1 = 0; \ + *(u64*)(r2 + 0) = r1; \ + r1 = %[map_hash_48b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + w0 &= 1; \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_48b) + : __clobber_all); +} + SEC("flow_dissector") __description("flow_keys illegal alu op with variable offset") __failure __msg("R7 pointer arithmetic on flow_keys prohibited") -- knotmirror 0.0.1 From 4c651a91bdfcded7a775a177d8918a4127555c78 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:02 +0000 Subject: [PATCH 03/15] bpf: Treat load and store through a number as arena access There are no address spaces in Rust. LLVM emits plain loads and stores for arena memory, without cast_kern: r3 = 0x20 ll // R_BPF_64_64 .bss, libbpf puts it into arena r2 = *(u64 *)(r3 + 0) // R3 invalid mem access 'scalar' lock *(u64 *)(r2 + 8) += r1 Add BPF_F_ARENA_SCALAR flag of BPF_PROG_LOAD. When the program is loaded with it and has an arena treat ldx, stx, st and atomics through a number as arena access. It's as safe as access through PTR_TO_ARENA: JIT adds the base of arena to the low 32 bits of the address. The program has an arena only with CAP_BPF and CAP_PERFMON. Registers of the program are not changed, since Rust compares and stores the address after the access. bpf_do_misc_fixups() copies the low 32 bits into BPF_REG_AX and the access goes through it: r2 = *(u64 *)(r3 + 0) -> w12 = w3 r2 = *(u64 *)(r12 + 0) Constant blinding needs BPF_REG_AX for immediates and leaves insns that use BPF_REG_AX alone. So the immediate of st through a number is not blinded, the rest of the program is: *(u64 *)(r3 + 0) = 1 -> w12 = w3 *(u64 *)(r12 + 0) = 1 The same insn may see PTR_TO_ARENA on another path. It works for both. It's a flag, since a number is also what the verifier makes of a pointer that went away when the program has CAP_PERFMON: a ringbuf record after bpf_ringbuf_submit(), a pointer to the packet after bpf_skb_pull_data(). Programs in C that have an arena keep "invalid mem access 'scalar'" for such bugs. JIT tells with bpf_jit_supports_arena_scalar() that it takes BPF_REG_AX as the address of arena access. With other JITs the flag is rejected with -EOPNOTSUPP. x86 and arm64 do: - x86: the fault handler finds the register that holds the address through reg2pt_regs[]. Add BPF_REG_AX, r10 of x86, there. - arm64: nothing else is needed. The JIT takes any register as the address and the fault handler reads it by its number. Compile tested only. Any number is an address of arena, NULL and small numbers included, like it is for PTR_TO_ARENA: arena code in C dereferences NULL and relies on the fault being handled. Still rejected: - insn that sees a number on one path and a pointer that is not PTR_TO_ARENA on another. - numbers passed to helpers and kfuncs, except __arena arguments. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-4-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/arm64/net/bpf_jit_comp.c | 5 +++ arch/x86/net/bpf_jit_comp.c | 6 ++++ include/linux/bpf_verifier.h | 2 ++ include/linux/filter.h | 1 + include/uapi/linux/bpf.h | 6 ++++ kernel/bpf/core.c | 6 ++++ kernel/bpf/fixups.c | 39 ++++++++++++++++++++++ kernel/bpf/syscall.c | 4 +++ kernel/bpf/verifier.c | 61 +++++++++++++++++++++++++++------- tools/include/uapi/linux/bpf.h | 6 ++++ 10 files changed, 124 insertions(+), 12 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 17134138fb5c..08725cde0c5a 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -3420,6 +3420,11 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_arena_scalar(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index e38466ad44b2..083fcd6cf15b 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -236,6 +236,7 @@ static const int reg2pt_regs[] = { [BPF_REG_7] = offsetof(struct pt_regs, r13), [BPF_REG_8] = offsetof(struct pt_regs, r14), [BPF_REG_9] = offsetof(struct pt_regs, r15), + [BPF_REG_AX] = offsetof(struct pt_regs, r10), }; /* @@ -4673,6 +4674,11 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_arena_scalar(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 79c4be5c33f4..c51083c761cf 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -672,6 +672,7 @@ struct bpf_insn_aux_data { bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ bool is_iter_next; /* bpf_iter__next() kfunc call */ bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ + bool arena_scalar; /* ldx/stx/st/atomic through a number, it's an address in arena */ u8 alu_state; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. @@ -943,6 +944,7 @@ struct bpf_verifier_env { bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ bool test_reg_invariants; /* fail verification on register invariants violations */ + bool arena_scalar; /* load and store through a number is arena access */ struct bpf_verifier_state *cur_state; /* current verifier state */ /* Search pruning optimization, array of list_heads for * lists of struct bpf_verifier_state_list. diff --git a/include/linux/filter.h b/include/linux/filter.h index e42eccb0990e..9339c6131f8f 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1250,6 +1250,7 @@ bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); +bool bpf_jit_supports_arena_scalar(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); bool bpf_jit_supports_large_stack(void); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 4687c3310996..e0ed44b1bbcb 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1344,6 +1344,12 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) +/* + * Load and store through a number is an access to the arena of the program + * at the low 32 bits of the number. It's for programs written in Rust. + */ +#define BPF_F_ARENA_SCALAR (1U << 8) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index a67c895ce734..92a87c73aa44 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3435,6 +3435,12 @@ bool __weak bpf_jit_supports_arena(void) return false; } +/* Whether JIT takes BPF_REG_AX as the address of arena access */ +bool __weak bpf_jit_supports_arena_scalar(void) +{ + return false; +} + bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return false; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index cb7219ff68bd..206cc9a614a9 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -44,6 +44,31 @@ static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *i return false; } +/* + * The insn accesses arena through a number. JITs add the base of arena + * to the register as it is, so the access goes through the low 32 bits of + * the number in BPF_REG_AX. Registers of the program are not changed: the + * number may be compared or stored later, and on another path the register + * may be PTR_TO_ARENA. + * + * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate + * of st through a number is not blinded. + */ +static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) +{ + bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); + struct bpf_insn *patch = buf; + + *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); + *patch = *insn; + if (load) + patch->src_reg = BPF_REG_AX; + else + patch->dst_reg = BPF_REG_AX; + patch++; + return patch - buf; +} + /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -1736,6 +1761,20 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); + if (env->insn_aux_data[i + delta].arena_scalar) { + cnt = arena_scalar_access(insn, insn_buf); + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + prog = new_prog; + env->prog = prog; + insn = prog->insnsi + i + delta; + goto next_insn; + } + /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 4e5377447e1b..654f896af265 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2972,9 +2972,13 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_TEST_REG_INVARIANTS | + BPF_F_ARENA_SCALAR | BPF_F_TOKEN_FD)) return -EINVAL; + if ((attr->prog_flags & BPF_F_ARENA_SCALAR) && !bpf_jit_supports_arena_scalar()) + return -EOPNOTSUPP; + bpf_prog_load_fixup_attach_type(attr); if (attr->prog_flags & BPF_F_TOKEN_FD) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 866c0540c68a..fd3c0206bd67 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5260,6 +5260,19 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +/* + * There are no address spaces in Rust, addresses of arena are plain numbers. + * When the program is loaded with BPF_F_ARENA_SCALAR and has an arena a load or + * a store through a number is an access to arena at the low 32 bits of it, + * like the access through PTR_TO_ARENA is. The register stays a number. + */ +static bool is_arena_scalar(struct bpf_verifier_env *env, int regno) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + return reg->type == SCALAR_VALUE && env->arena_scalar && env->prog->aux->arena; +} + static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn) { @@ -5290,7 +5303,7 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_sk_reg(env, regno)) return false; - if (is_arena_reg(env, regno)) + if (is_arena_reg(env, regno) || is_arena_scalar(env, regno)) return bpf_jit_supports_insn(insn, true); if (is_load_acq_unsafe(env, regno, insn)) return false; @@ -7171,6 +7184,22 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch); +/* + * Returns the register to check the access of the current insn with. + * For a number in a program with an arena that is 'arena'. + */ +static struct bpf_reg_state *mem_access_reg(struct bpf_verifier_env *env, int regno, + struct bpf_reg_state *arena) +{ + if (!is_arena_scalar(env, regno)) + return cur_regs(env) + regno; + + memset(arena, 0, sizeof(*arena)); + arena->type = PTR_TO_ARENA; + env->insn_aux_data[env->insn_idx].arena_scalar = true; + return arena; +} + static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) @@ -7178,6 +7207,7 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *src_reg; enum bpf_reg_type src_reg_type; int err; @@ -7199,15 +7229,16 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - src_reg_type = regs[insn->src_reg].type; + src_reg = mem_access_reg(env, insn->src_reg, &arena); + src_reg_type = src_reg->type; /* * check_stack_read_fixed_off() may refine the modification's origin to * the source stack slot. */ bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); - err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, - BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), + insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); @@ -7224,6 +7255,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; int err; @@ -7245,11 +7277,12 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - dst_reg_type = regs[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, - BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), + insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -7259,7 +7292,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { - struct bpf_reg_state *dst_reg; + struct bpf_reg_state arena, *dst_reg; int load_reg; int err; @@ -7304,6 +7337,9 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } + /* load_reg may be dst_reg. Look at dst_reg before it's marked as unknown. */ + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + load_reg = bpf_atomic_load_reg(insn); if (load_reg >= 0) { /* check and record load of old value */ @@ -7312,8 +7348,6 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } - dst_reg = cur_regs(env) + insn->dst_reg; - /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ @@ -19369,15 +19403,17 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_stack_arg_write(env, state, insn->off, NULL); } + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; - dst_reg_type = cur_regs(env)[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; - err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg), + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -22617,6 +22653,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; + env->arena_scalar = attr->prog_flags & BPF_F_ARENA_SCALAR; env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 4687c3310996..e0ed44b1bbcb 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1344,6 +1344,12 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) +/* + * Load and store through a number is an access to the arena of the program + * at the low 32 bits of the number. It's for programs written in Rust. + */ +#define BPF_F_ARENA_SCALAR (1U << 8) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ -- knotmirror 0.0.1 From b233f937b98f8e5c88e2978ef36321f888076274 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:03 +0000 Subject: [PATCH 04/15] selftests/bpf: Add tests for arena access through numbers Add tests for load and store through the address that bpf_arena_alloc_pages() returned, without cast_kern: - all sizes, sign extending load, atomics, load-acquire, store-release. - no register is changed by the access, whatever the upper half of the address is, and 64-bit math on the address stays 64-bit. - a number that is not an address of arena, address of the stack included, reads zeroes and writes nowhere at both ends of the range of insn offset. - insn that sees a number on one path and PTR_TO_ARENA on another, followed by access through PTR_TO_ARENA. - store in a callback of bpf_loop(). - NULL that a helper returned and a number that is less than a page are addresses of arena too. - xchg that loads into the register that holds the address. - rejected: prog without arena, prog without BPF_F_ARENA_SCALAR, pointer to stack at the same insn, xchg that loads into the register that holds a pointer to stack, numbers passed to helpers. The tests are for x86 and arm64, other JITs don't take the flag. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-5-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/prog_tests/arena_scalar_blinded.c | 21 + .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_arena_scalar.c | 912 ++++++++++++++++++ tools/testing/selftests/bpf/test_loader.c | 2 + 4 files changed, 937 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c create mode 100644 tools/testing/selftests/bpf/progs/verifier_arena_scalar.c diff --git a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c new file mode 100644 index 000000000000..2ac2e9a652fe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "sysctl_helpers.h" +#include "verifier_arena_scalar.skel.h" + +/* The same tests with constants of the programs blinded */ +void serial_test_arena_scalar_blinded(void) +{ + const char *harden = "/proc/sys/net/core/bpf_jit_harden"; + char old[16] = {}; + + if (!is_jit_enabled()) { + test__skip(); + return; + } + if (sysctl_set_or_fail(harden, old, "2")) + return; + RUN_TESTS(verifier_arena_scalar); + sysctl_set_or_fail(harden, NULL, old); +} diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 8a6d341b754a..460ad10ddc02 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -12,6 +12,7 @@ #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" +#include "verifier_arena_scalar.skel.h" #include "verifier_arena_globals1.skel.h" #include "verifier_arena_globals2.skel.h" #include "verifier_array_access.skel.h" @@ -198,6 +199,7 @@ void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } +void test_verifier_arena_scalar(void) { RUN(verifier_arena_scalar); } void test_verifier_arena_globals1(void) { RUN(verifier_arena_globals1); } void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c new file mode 100644 index 000000000000..bebc37f501b7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c @@ -0,0 +1,912 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "../../../include/linux/filter.h" +#include "bpf_misc.h" + +void *bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, int node_id, + __u64 flags) __ksym; + +#ifdef __TARGET_ARCH_arm64 +#define ARENA_VM_START (1ull << 32) +#else +#define ARENA_VM_START (1ull << 44) +#endif + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 4); + __ulong(map_extra, ARENA_VM_START); +} arena SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, long long); +} hash SEC(".maps"); + +/* JITs that take BPF_F_ARENA_SCALAR */ +#define __arena_scalar __flag(BPF_F_ARENA_SCALAR) __arch_x86_64 __arch_arm64 + +/* BTF FUNC records are not generated for kfuncs referenced from inline assembly */ +void __kfunc_btf_root(void) +{ + bpf_arena_alloc_pages(0, 0, 0, 0, 0); +} + +/* Tests start with r6 = address of a new page as the user space sees it, a number */ + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load and store of every size") +__success __retval(0) +__load_if_JITed() +__naked void ld_st_sizes(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + r1 = 0x1122334455667788 ll; \ + *(u64 *)(r6 + 0) = r1; \ + *(u32 *)(r6 + 8) = r1; \ + *(u16 *)(r6 + 12) = r1; \ + *(u8 *)(r6 + 14) = r1; \ + *(u64 *)(r6 + 16) = 0x1234; \ + *(u32 *)(r6 + 24) = 0x5678; \ + *(u16 *)(r6 + 28) = 0x9a; \ + *(u8 *)(r6 + 30) = 0xbc; \ + r0 = 1; \ + r2 = *(u64 *)(r6 + 0); \ + if r2 != r1 goto 9f; \ + r0 = 2; \ + r2 = *(u32 *)(r6 + 8); \ + if r2 != 0x55667788 goto 9f; \ + r0 = 3; \ + r2 = *(u16 *)(r6 + 12); \ + if r2 != 0x7788 goto 9f; \ + r0 = 4; \ + r2 = *(u8 *)(r6 + 14); \ + if r2 != 0x88 goto 9f; \ + r0 = 5; \ + r2 = *(u64 *)(r6 + 16); \ + if r2 != 0x1234 goto 9f; \ + r0 = 6; \ + r2 = *(u32 *)(r6 + 24); \ + if r2 != 0x5678 goto 9f; \ + r0 = 7; \ + r2 = *(u16 *)(r6 + 28); \ + if r2 != 0x9a goto 9f; \ + r0 = 8; \ + r2 = *(u8 *)(r6 + 30); \ + if r2 != 0xbc goto 9f; \ + /* the address is what it was */ \ + r0 = 9; \ + if r6 != r7 goto 9f; \ + r0 = 10; \ + r7 >>= 32; \ + if r7 == 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load into the register that holds the address") +__success __retval(0) +__load_if_JITed() +__naked void ld_into_base(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 77; \ + *(u64 *)(r6 + 0) = r1; \ + r6 = *(u64 *)(r6 + 0); \ + r0 = 1; \ + if r6 != 77 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: sign extending load") +__success __retval(0) +__load_if_JITed() +__naked void ldsx(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + *(u64 *)(r6 + 0) = 0x80; \ + .8byte %[ldsx_insn]; /* r2 = *(s8 *)(r6 + 0) */ \ + r0 = 1; \ + if r2 != -128 goto 9f; \ + r0 = 2; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(ldsx_insn, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_B, + BPF_REG_2, BPF_REG_6, 0, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: address loaded from arena") +__success __retval(0) +__load_if_JITed() +__naked void ptr_chase(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + /* page[0] = &page[64]; page[64] = 5; */ \ + r1 = r6; \ + r1 += 64; \ + *(u64 *)(r6 + 0) = r1; \ + *(u64 *)(r1 + 0) = 5; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 1; \ + if r2 != r1 goto 9f; \ + r3 = *(u64 *)(r2 + 0); \ + r0 = 2; \ + if r3 != 5 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: atomics") +__success __retval(0) +__load_if_JITed() +__naked void atomics(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + *(u64 *)(r6 + 8) = 1; \ + r1 = 2; \ + lock *(u64 *)(r6 + 8) += r1; \ + r1 = 4; \ + .8byte %[fetch_add_insn]; /* r1 = atomic_fetch_add((u64 *)(r6 + 8), r1) */ \ + r0 = 1; \ + if r1 != 3 goto 9f; \ + r1 = 8; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r6 + 8, r1) */ \ + r0 = 2; \ + if r1 != 7 goto 9f; \ + r0 = 8; \ + r1 = 16; \ + .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r6 + 8, r0, r1) */ \ + r2 = r0; \ + r0 = 3; \ + if r2 != 8 goto 9f; \ + r2 = *(u64 *)(r6 + 8); \ + r0 = 4; \ + if r2 != 16 goto 9f; \ + r0 = 5; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, + BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 8)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: cmpxchg through r0") +__success __retval(0) +__load_if_JITed() +__naked void cmpxchg_r0(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + /* The address is in r0. The value at the address is not equal to it. */ \ + *(u64 *)(r6 + 0) = 3; \ + r0 = r6; \ + r1 = 5; \ + .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r0 + 0, r0, r1) */ \ + r2 = r0; \ + r0 = 1; \ + if r2 != 3 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 2; \ + if r2 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_0, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: xchg into the register that holds the address") +__success __retval(0) +__load_if_JITed() +__naked void xchg_into_base(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 0) = 3; \ + r1 = r6; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ + r0 = 1; \ + if r1 != 3 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r0 = 2; \ + if r2 != r6 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: xchg into the register that holds a pointer to stack") +__failure __msg("misaligned access off (0x0; 0xffffffffffffffff)+0 size 8") +__naked void xchg_into_stack_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + r1 = r10; \ + r1 += -8; \ + .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(arena), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) + : __clobber_all); +} + +#ifdef CAN_USE_LOAD_ACQ_STORE_REL + +SEC("syscall") +__arena_scalar +__description("arena_scalar: load-acquire and store-release") +__success __retval(0) +__load_if_JITed() +__naked void load_acq_store_rel(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r7 = r6; \ + r1 = 0x1234; \ + .8byte %[store_release_insn]; /* store_release((u64 *)(r6 + 8), r1) */ \ + .8byte %[load_acquire_insn]; /* r2 = load_acquire((u64 *)(r6 + 8)) */ \ + r0 = 1; \ + if r2 != 0x1234 goto 9f; \ + .8byte %[load_acquire8_insn]; /* w2 = load_acquire((u8 *)(r6 + 8)) */ \ + r0 = 2; \ + if r2 != 0x34 goto 9f; \ + r0 = 3; \ + if r6 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(store_release_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_6, BPF_REG_1, 8)), + __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)), + __imm_insn(load_acquire8_insn, + BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)) + : __clobber_all); +} + +#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number that is not an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void not_in_arena(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + /* nothing is allocated: all loads read 0, stores are dropped */ \ + r6 = 0xdeadbeef00000000 ll; \ + r0 = 1; \ + r2 = *(u64 *)(r6 + 0); \ + if r2 != 0 goto 9f; \ + r0 = 2; \ + r2 = *(u8 *)(r6 - 32768); \ + if r2 != 0 goto 9f; \ + r6 = 0x12345678ffffffff ll; \ + r0 = 3; \ + r2 = *(u64 *)(r6 + 32760); \ + if r2 != 0 goto 9f; \ + *(u64 *)(r6 + 32760) = 1; \ + *(u8 *)(r6 + 32767) = r2; \ + r1 = 1; \ + lock *(u64 *)(r6 + 32760) += r1; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: store through the address of the stack as a number") +__success __retval(0) +__load_if_JITed() +__naked void stack_addr_as_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + *(u64 *)(r10 - 8) = 5; \ + r6 = r10; \ + r6 |= 0; \ + /* r6 is a number now. The store goes to arena, not to the stack. */ \ + *(u64 *)(r6 - 8) = 7; \ + r1 = 9; \ + *(u64 *)(r6 - 8) = r1; \ + lock *(u64 *)(r6 - 8) += r1; \ + r2 = *(u64 *)(r10 - 8); \ + r0 = 1; \ + if r2 != 5 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: 64-bit math on the address stays 64-bit") +__success __retval(0) +__load_if_JITed() +__naked void alu64_after_access(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r2 = *(u64 *)(r6 + 0); \ + r7 = r6; \ + r7 += 8; \ + r7 -= r6; \ + r0 = 1; \ + if r7 != 8 goto 9f; \ + r7 = r6; \ + r7 >>= 32; \ + r0 = 2; \ + if r7 == 0 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: st of an immediate changes no register") +__success __retval(0) +__load_if_JITed() +__naked void st_keeps_regs(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r0 = r6; \ + r1 = 0x1111; \ + r2 = 0x2222; \ + *(u64 *)(r0 + 0) = 5; \ + r3 = r0; \ + r0 = 1; \ + if r3 != r6 goto 9f; \ + r0 = 2; \ + if r1 != 0x1111 goto 9f; \ + r0 = 3; \ + if r2 != 0x2222 goto 9f; \ + r0 = 0x3333; \ + r1 = r6; \ + *(u32 *)(r1 + 8) = -7; \ + r3 = r0; \ + r0 = 4; \ + if r3 != 0x3333 goto 9f; \ + r0 = 5; \ + if r1 != r6 goto 9f; \ + r0 = 6; \ + r3 = *(u64 *)(r6 + 0); \ + if r3 != 5 goto 9f; \ + r0 = 7; \ + r3 = *(u64 *)(r6 + 8); \ + r4 = 0xfffffff9 ll; \ + if r3 != r4 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: access through a number with garbage in the upper half") +__success __retval(0) +__load_if_JITed() +__naked void st_value_garbage(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0xdeadbeef00001000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + *(u64 *)(r8 + 0) = 1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + *(u8 *)(r8 + 1) = 1; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + *(u32 *)(r8 + 4) = r1; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r2 = *(u16 *)(r8 + 2); \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 5; \ + if r1 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: access through a number without the upper half") +__success __retval(0) +__load_if_JITed() +__naked void st_value_small(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0x1000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + *(u64 *)(r8 + 0) = 1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + *(u8 *)(r8 + 1) = 1; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + *(u32 *)(r8 + 4) = r1; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r2 = *(u16 *)(r8 + 2); \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 5; \ + if r1 != 3 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: atomics through a number that is not an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void atomic_value(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = 0xdeadbeef00001000 ll; \ + *(u64 *)(r6 + 2048) = r1; \ + r7 = *(u64 *)(r6 + 2048); \ + r8 = *(u64 *)(r6 + 2048); \ + r1 = 3; \ + lock *(u64 *)(r8 + 0) += r1; \ + r0 = 1; \ + if r8 != r7 goto 9f; \ + .8byte %[fetch_add_insn]; \ + r0 = 2; \ + if r8 != r7 goto 9f; \ + .8byte %[xchg_insn]; \ + r0 = 3; \ + if r8 != r7 goto 9f; \ + r0 = 0; \ + .8byte %[cmpxchg_insn]; \ + r0 = 4; \ + if r8 != r7 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, + BPF_REG_8, BPF_REG_1, 0)), + __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_8, BPF_REG_1, 0)), + __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_8, BPF_REG_1, 0)) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number and pointer to arena at the same insn") +__success __retval(0) +__load_if_JITed() +__naked void mixed_number_arena(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r6 + 8) = 0x1234; \ + /* a number that the verifier does not know, 0 at run time */ \ + r7 = *(u64 *)(r6 + 16); \ + r8 = r6; \ + if r7 != 0 goto 1f; \ + .8byte %[cast_kern_insn]; \ +1: *(u8 *)(r8 + 0) = 1; \ + *(u8 *)(r8 + 1) = r7; \ + r2 = *(u8 *)(r8 + 1); \ + lock *(u64 *)(r8 + 24) += r7; \ + r0 = 0; \ + if r7 != 0 goto 9f; \ + /* pointer to arena only: JIT adds all 64 bits of r8 to the base */ \ + r0 = 2; \ + r2 = *(u64 *)(r8 + 8); \ + if r2 != 0x1234 goto 9f; \ + r0 = 3; \ + r2 = *(u8 *)(r6 + 0); \ + if r2 != 1 goto 9f; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm_insn(cast_kern_insn, BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, + BPF_REG_8, BPF_REG_8, 1, 1)) + : __clobber_all); +} + +SEC("syscall") +__description("arena_scalar: no flag, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_flag(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: no arena, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_arena(void) +{ + asm volatile (" \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: pointer that is NULL is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void null_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u32 *)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[hash] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r0 = 1; \ + if r1 != 0 goto 9f; \ + /* nothing is allocated: the load reads 0 */ \ + r0 = *(u64 *)(r1 + 0); \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: pointer that is NULL with an offset is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void null_ptr_off(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r1 = 0; \ + *(u32 *)(r10 - 4) = r1; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[hash] ll; \ + call %[bpf_map_lookup_elem]; \ + r1 = r0; \ + r0 = 1; \ + if r1 != 0 goto 9f; \ + r1 += 8; \ + *(u64 *)(r1 + 0) = 5; \ + r0 = *(u64 *)(r1 + 0); \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number that is less than a page is an address in arena") +__success __retval(0) +__load_if_JITed() +__naked void small_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + r1 &= 0xfff; \ + r0 = *(u64 *)(r1 + 0); \ + exit; \ +" : + : __imm_addr(arena), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number is not a pointer for a helper") +__failure __msg("R2 type=scalar expected=") +__naked void helper_arg(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + r1 = %[hash] ll; \ + r2 = r6; \ + call %[bpf_map_lookup_elem]; \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm_addr(hash), + __imm(bpf_arena_alloc_pages), + __imm(bpf_map_lookup_elem) + : __clobber_all); +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: number and pointer to stack at the same insn") +__failure __msg("same insn cannot be used with different pointers") +__load_if_JITed() +__naked void mixed_number_stack(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = -1; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r6 = r0; \ + r0 = 100; \ + if r6 == 0 goto 9f; \ + *(u64 *)(r10 - 8) = 0; \ + call %[bpf_get_prandom_u32]; \ + if w0 != 0 goto 1f; \ + r6 = r10; \ + r6 += -8; \ +1: r0 = *(u64 *)(r6 + 0); \ + r0 = 0; \ +9: exit; \ +" : + : __imm_addr(arena), + __imm(bpf_arena_alloc_pages), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +static int st_cb(__u64 idx, void *ctx) +{ + volatile long *p = *(volatile long **)ctx; + + p[idx] = 7; + return 0; +} + +SEC("syscall") +__arena_scalar +__description("arena_scalar: store through a number in a callback") +__success __retval(0) +__load_if_JITed() +int st_in_callback(void *unused) +{ + volatile long *p = bpf_arena_alloc_pages(&arena, NULL, 1, -1, 0); + + if (!p) + return 100; + bpf_loop(4, st_cb, &p, 0); + return p[0] + p[3] + p[4] - 14; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 25eeb1c1248b..a89890cd56d8 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -580,6 +580,8 @@ static int parse_test_spec(struct test_loader *tester, update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); + } else if (strcmp(val, "BPF_F_ARENA_SCALAR") == 0) { + update_flags(&spec->prog_flags, BPF_F_ARENA_SCALAR, clear); } else /* assume numeric value */ { err = parse_int(val, &flags, "test prog flags"); if (err) -- knotmirror 0.0.1 From bed1986a1932942e9db17fbdb2a19d0c4160a1f5 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:04 +0000 Subject: [PATCH 05/15] bpf: Allow names of Rust types and functions in BTF Names of types and functions in BTF that LLVM makes for a Rust program are not C identifiers: [69] STRUCT 'NonNull' size=16 vlen=1 [147] FUNC 'write_fmt' type_id=146 and the kernel rejects such BTF with "Invalid name". scx_simple and scx_cosmos schedulers written in Rust have them in STRUCT, FWD and FUNC, made of letters, digits and " #&()*,:;<>[]{}", 430 characters at most. Allow any printable character in btf_name_valid_identifier(), like it's done for DATASEC. It checks names of types, functions, members, enumerators, variables and arguments, so all of them can have such characters now. The limit of KSYM_NAME_LEN stays. The name of FUNC is a part of the name of the program in kallsyms, where a space would break the parsers. Replace what is not a character of an identifier with '_' there. Tests in prog_tests/btf.c expect "Invalid name" for names with '!' and '*', which are valid now. Put a character that is not printable there. The type name '?foo' is expected to load. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-6-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 15 ++---- kernel/bpf/core.c | 5 ++ tools/testing/selftests/bpf/prog_tests/btf.c | 52 ++++++++++---------- 3 files changed, 33 insertions(+), 39 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 8cc17a1cd25c..d27af5d8e495 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -901,16 +901,6 @@ static bool btf_name_offset_valid(const struct btf *btf, u32 offset) return offset < btf->hdr.str_len; } -static bool __btf_name_char_ok(char c, bool first) -{ - if ((first ? !isalpha(c) : - !isalnum(c)) && - c != '_' && - c != '.') - return false; - return true; -} - const char *btf_str_by_offset(const struct btf *btf, u32 offset) { while (offset < btf->start_str_off) @@ -923,20 +913,21 @@ const char *btf_str_by_offset(const struct btf *btf, u32 offset) return NULL; } +/* Names in BTF of Rust are not C identifiers. Allow any printable character */ static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) { /* offset must be valid */ const char *src = btf_str_by_offset(btf, offset); const char *src_limit; - if (!__btf_name_char_ok(*src, true)) + if (!isprint(*src)) return false; /* set a limit on identifier length */ src_limit = src + KSYM_NAME_LEN; src++; while (*src && src < src_limit) { - if (!__btf_name_char_ok(*src, false)) + if (!isprint(*src)) return false; src++; } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 92a87c73aa44..05c89396119a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -589,6 +590,10 @@ bpf_prog_ksym_set_name(struct bpf_prog *prog) prog->aux->func_info[prog->aux->func_idx].type_id); func_name = btf_name_by_offset(prog->aux->btf, type->name_off); snprintf(sym, (size_t)(end - sym), "_%s", func_name); + /* the name of a function of Rust is not an identifier */ + for (; *sym; sym++) + if (!isalnum(*sym) && *sym != '_' && *sym != '.') + *sym = '_'; return; } diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index df6ad38d287d..87b554067071 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -1987,14 +1987,14 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "typedef (invalid name, invalid identifier)", + .descr = "typedef (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPEDEF_ENC(NAME_TBD, 1), /* [2] */ BTF_END_RAW, }, - .str_sec = "\0__!int", - .str_sec_size = sizeof("\0__!int"), + .str_sec = "\0__\7int", + .str_sec_size = sizeof("\0__\7int"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "typedef_check_btf", .key_size = sizeof(int), @@ -2112,15 +2112,15 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "fwd type (invalid name, invalid identifier)", + .descr = "fwd type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_FWD, 0, 0), 0), /* [2] */ BTF_END_RAW, }, - .str_sec = "\0__!skb", - .str_sec_size = sizeof("\0__!skb"), + .str_sec = "\0__\7skb", + .str_sec_size = sizeof("\0__\7skb"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "fwd_type_check_btf", .key_size = sizeof(int), @@ -2175,7 +2175,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "struct type (invalid name, invalid identifier)", + .descr = "struct type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2183,8 +2183,8 @@ static struct btf_raw_test raw_tests[] = { BTF_MEMBER_ENC(NAME_TBD, 1, 0), BTF_END_RAW, }, - .str_sec = "\0A!\0B", - .str_sec_size = sizeof("\0A!\0B"), + .str_sec = "\0A\7\0B", + .str_sec_size = sizeof("\0A\7\0B"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "struct_type_check_btf", .key_size = sizeof(int), @@ -2217,7 +2217,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "struct member (invalid name, invalid identifier)", + .descr = "struct member (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2225,8 +2225,8 @@ static struct btf_raw_test raw_tests[] = { BTF_MEMBER_ENC(NAME_TBD, 1, 0), BTF_END_RAW, }, - .str_sec = "\0A\0B*", - .str_sec_size = sizeof("\0A\0B*"), + .str_sec = "\0A\0B\7", + .str_sec_size = sizeof("\0A\0B\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "struct_type_check_btf", .key_size = sizeof(int), @@ -2260,7 +2260,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "enum type (invalid name, invalid identifier)", + .descr = "enum type (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(NAME_TBD, @@ -2269,8 +2269,8 @@ static struct btf_raw_test raw_tests[] = { BTF_ENUM_ENC(NAME_TBD, 0), BTF_END_RAW, }, - .str_sec = "\0A!\0B", - .str_sec_size = sizeof("\0A!\0B"), + .str_sec = "\0A\7\0B", + .str_sec_size = sizeof("\0A\7\0B"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "enum_type_check_btf", .key_size = sizeof(int), @@ -2306,7 +2306,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "enum member (invalid name, invalid identifier)", + .descr = "enum member (invalid name, not printable)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_ENC(0, @@ -2315,8 +2315,8 @@ static struct btf_raw_test raw_tests[] = { BTF_ENUM_ENC(NAME_TBD, 0), BTF_END_RAW, }, - .str_sec = "\0A!", - .str_sec_size = sizeof("\0A!"), + .str_sec = "\0A\7", + .str_sec_size = sizeof("\0A\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "enum_type_check_btf", .key_size = sizeof(int), @@ -2625,14 +2625,14 @@ static struct btf_raw_test raw_tests[] = { .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ - /* void (*)(int a, unsigned int !!!) */ + /* void (*)(int a, unsigned int \7) */ BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), BTF_END_RAW, }, - .str_sec = "\0a\0!!!", - .str_sec_size = sizeof("\0a\0!!!"), + .str_sec = "\0a\0\7", + .str_sec_size = sizeof("\0a\0\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "func_proto_type_check_btf", .key_size = sizeof(int), @@ -2775,12 +2775,12 @@ static struct btf_raw_test raw_tests[] = { BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), - /* void !!!(int a, unsigned int b) */ + /* void \7(int a, unsigned int b) */ BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ BTF_END_RAW, }, - .str_sec = "\0a\0b\0!!!", - .str_sec_size = sizeof("\0a\0b\0!!!"), + .str_sec = "\0a\0b\0\7", + .str_sec_size = sizeof("\0a\0b\0\7"), .map_type = BPF_MAP_TYPE_ARRAY, .map_name = "func_type_check_btf", .key_size = sizeof(int), @@ -3585,15 +3585,13 @@ static struct btf_raw_test raw_tests[] = { .btf_load_err = true, }, { - .descr = "type name '?foo' is not ok", + .descr = "type name '?foo' is ok", .raw_types = { /* union ?foo; */ BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_FWD, 1, 0), 0), /* [1] */ BTF_END_RAW, }, BTF_STR_SEC("\0?foo"), - .err_str = "Invalid name", - .btf_load_err = true, }, { -- knotmirror 0.0.1 From 711d8a75c507cb843b4f73278d1f975bf15b11c8 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:05 +0000 Subject: [PATCH 06/15] selftests/bpf: Add tests for names of Rust types and functions in BTF Check that BTF with names of STRUCT, FWD, TYPEDEF and FUNC that Rust makes is loaded and that the name of FUNC is an identifier in kallsyms. The test is serial. Programs are not in kallsyms while bpf_jit_harden is set and there are tests that set it for a while. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-7-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/prog_tests/btf.c | 15 ++++ .../selftests/bpf/prog_tests/btf_rust.c | 69 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/btf_rust.c diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 87b554067071..207341f4bd99 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -3593,6 +3593,21 @@ static struct btf_raw_test raw_tests[] = { }, BTF_STR_SEC("\0?foo"), }, +{ + .descr = "names of Rust types and functions are ok", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_NTH(1), 0, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_NTH(2), 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_NTH(3), 1, 0), + BTF_FWD_ENC(NAME_NTH(4), 0), /* [3] */ + BTF_TYPEDEF_ENC(NAME_NTH(5), 2), /* [4] */ + BTF_FUNC_PROTO_ENC(0, 1), /* [5] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(6), 1), + BTF_FUNC_ENC(NAME_NTH(7), 5), /* [6] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0u32\0Option<&str>\0__0\0*const str\0{impl#9}<[u8; 4]>\0self\0fmt"), +}, { .descr = "float test #1, well-formed", diff --git a/tools/testing/selftests/bpf/prog_tests/btf_rust.c b/tools/testing/selftests/bpf/prog_tests/btf_rust.c new file mode 100644 index 000000000000..ce2b31087b59 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_rust.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#define FUNC_NAME "write_fmt" +#define KSYM_NAME "write_fmt_scx_simple__BpfStream_" + +/* The name of a function of Rust is a part of the name of the program in kallsyms */ +static void test_func_name(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int int_id, proto_id, prog_fd = -1, i; + struct bpf_func_info func_info = {}; + struct bpf_prog_info info = {}; + unsigned long long addr; + __u32 len = sizeof(info); + char sym[128], *p = sym; + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf")) + return; + int_id = btf__add_int(btf, "i32", 4, BTF_INT_SIGNED); + ASSERT_GT(int_id, 0, "int"); + proto_id = btf__add_func_proto(btf, int_id); + ASSERT_GT(proto_id, 0, "proto"); + ASSERT_OK(btf__add_func_param(btf, "ctx", int_id), "param"); + func_info.type_id = btf__add_func(btf, FUNC_NAME, BTF_FUNC_STATIC, proto_id); + ASSERT_GT(func_info.type_id, 0, "func"); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf load")) + goto out; + + opts.prog_btf_fd = btf__fd(btf); + opts.func_info = &func_info; + opts.func_info_cnt = 1; + opts.func_info_rec_size = sizeof(func_info); + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, + ARRAY_SIZE(insns), &opts); + if (!ASSERT_GE(prog_fd, 0, "prog load")) + goto out; + if (!ASSERT_OK(bpf_prog_get_info_by_fd(prog_fd, &info, &len), "prog info")) + goto out; + if (!info.jited_prog_len) { + test__skip(); + goto out; + } + + p += sprintf(p, "bpf_prog_"); + for (i = 0; i < BPF_TAG_SIZE; i++) + p += sprintf(p, "%02x", info.tag[i]); + sprintf(p, "_%s", KSYM_NAME); + ASSERT_OK(kallsyms_find(sym, &addr), sym); +out: + if (prog_fd >= 0) + close(prog_fd); + btf__free(btf); +} + +/* Serial: programs are not in kallsyms while another test sets bpf_jit_harden */ +void serial_test_btf_rust(void) +{ + if (test__start_subtest("func_name")) + test_func_name(); +} -- knotmirror 0.0.1 From 8e1b3e0945a6fa51c72339898d84fba9f0464b41 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:06 +0000 Subject: [PATCH 07/15] bpf: Allow arguments without names in static functions in BTF Some static functions in BTF of a Rust program have arguments without names: [71] FUNC_PROTO '(anon)' ret_type_id=0 vlen=1 '(anon)' type_id=72 [81] FUNC 'unwrap_failed' type_id=71 linkage=static and the kernel rejects such BTF with "Invalid arg#1". It's 5 of 25 static functions in scx_cosmos and 6 of 24 in scx_simple. The verifier looks at types of the arguments. The names are printed only, as "(anon)" when there is none. Allow such static FUNC. Global functions are checked as before. The test "func (Some arg has no name)" in prog_tests/btf.c has a static function. Make it global to keep the check. Signed-off-by: Alexei Starovoitov Acked-by: Alan Maguire Link: https://lore.kernel.org/bpf/20261002124714.180012-8-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 4 ++++ tools/testing/selftests/bpf/prog_tests/btf.c | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d27af5d8e495..c9d4b709380c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5752,6 +5752,10 @@ static int btf_func_check(struct btf_verifier_env *env, return -EINVAL; } + /* Rust leaves out names of some arguments of static functions */ + if (btf_func_linkage(t) == BTF_FUNC_STATIC) + return 0; + args = (const struct btf_param *)(proto_type + 1); nr_args = btf_type_vlen(proto_type); for (i = 0; i < nr_args; i++) { diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 207341f4bd99..21fdeeb23405 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -2793,7 +2793,7 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "func (Some arg has no name)", + .descr = "func (Some arg of global func has no name)", .raw_types = { BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ @@ -2802,7 +2802,8 @@ static struct btf_raw_test raw_tests[] = { BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), BTF_FUNC_PROTO_ARG_ENC(0, 2), /* void func(int a, unsigned int) */ - BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_TYPE_ENC(NAME_TBD, /* [4] */ + BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 3), BTF_END_RAW, }, .str_sec = "\0a\0func", -- knotmirror 0.0.1 From 3dc3a91ec9c7184eb5782c86083c7638e60675a0 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:07 +0000 Subject: [PATCH 08/15] selftests/bpf: Add test for arguments without names in static functions Check that BTF with a static function that has an argument without a name is loaded. Vararg has no name too. Check that a global function with it is loaded as before. Signed-off-by: Alexei Starovoitov Acked-by: Alan Maguire Link: https://lore.kernel.org/bpf/20261002124714.180012-9-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/prog_tests/btf.c | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 21fdeeb23405..642800b80c74 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -2819,6 +2819,57 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid arg#2", }, +{ + .descr = "func (Some arg of static func has no name)", + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* unsigned int */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + /* static void func(int a, unsigned int) */ + BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func (vararg of global func has no name)", + .raw_types = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* void (*)(int a, ...) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [2] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 0), + /* void func(int a, ...) */ + BTF_TYPE_ENC(NAME_TBD, /* [3] */ + BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + { .descr = "func (Non zero vlen)", .raw_types = { -- knotmirror 0.0.1 From 9094a158e842323b4487e32e88f01bd45a16af66 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:08 +0000 Subject: [PATCH 09/15] bpf: Allow a variable in DATASEC that is smaller than its type LLVM splits a static of a Rust program into pieces. Every piece is a VAR with the type of the whole static: [223] STRUCT 'BpfCell>' size=32 vlen=1 [236] VAR '..scx_cosmos9TASK_CTXS.0' type_id=223, linkage=static [248] DATASEC '.bss' size=0 vlen=9 type_id=236 offset=24648 size=1 (VAR '..TASK_CTXS.0') and the kernel rejects such BTF with "Invalid size". Allow it. The size of a variable in DATASEC is used in two places: - btf_find_datasec_var() looks for timers, spin locks, kptrs and other special fields. btf_find_field_one() skips a variable when its size is not the size of its type, so there are no special fields in a piece. - btf_datasec_show() prints variables by their types. It would read past the piece and past the end of the map value when the piece is the last one. Don't print the pieces. "global data test #8" and "#10" in prog_tests/btf.c expect "Invalid size". BTF of both is loaded now. The map of #10 is not created, since its value is larger than the section. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-10-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 31 +++++++++++++++----- tools/testing/selftests/bpf/prog_tests/btf.c | 9 ++---- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index c9d4b709380c..0630675377aa 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5397,7 +5397,7 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, env->resolve_mode = RESOLVE_TBD; for_each_vsi_from(i, v->next_member, v->t, vsi) { - u32 var_type_id = vsi->type, type_id, type_size = 0; + u32 var_type_id = vsi->type, type_id; const struct btf_type *var_type = btf_type_by_id(env->btf, var_type_id); if (!var_type || !btf_type_is_var(var_type)) { @@ -5412,16 +5412,16 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, return env_stack_push(env, var_type, var_type_id); } + /* + * The variable can be smaller than its type. It's a piece of + * a variable that the compiler split then, with the type of + * the whole variable. + */ type_id = var_type->type; - if (!btf_type_id_size(btf, &type_id, &type_size)) { + if (!btf_type_id_size(btf, &type_id, NULL)) { btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); return -EINVAL; } - - if (vsi->size < type_size) { - btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); - return -EINVAL; - } } env_stack_pop_resolved(env, 0, 0); @@ -5434,6 +5434,16 @@ static void btf_datasec_log(struct btf_verifier_env *env, btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); } +/* A piece of a variable is smaller than its type, which is the one of the whole variable */ +static bool btf_var_is_piece(const struct btf *btf, const struct btf_type *var, + const struct btf_var_secinfo *vsi) +{ + u32 size; + + return IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, var->type), &size)) || + vsi->size < size; +} + static void btf_datasec_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, @@ -5441,6 +5451,7 @@ static void btf_datasec_show(const struct btf *btf, { const struct btf_var_secinfo *vsi; const struct btf_type *var; + bool comma = false; u32 i; if (!btf_show_start_type(show, t, type_id, data)) @@ -5450,8 +5461,12 @@ static void btf_datasec_show(const struct btf *btf, __btf_name_by_offset(btf, t->name_off)); for_each_vsi(i, t, vsi) { var = btf_type_by_id(btf, vsi->type); - if (i) + /* there is less data than the type takes */ + if (btf_var_is_piece(btf, var, vsi)) + continue; + if (comma) btf_show(show, ","); + comma = true; btf_type_ops(var)->show(btf, var, vsi->type, data + vsi->offset, bits_offset, show); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 642800b80c74..24ab62b2834a 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -424,7 +424,7 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid type", }, { - .descr = "global data test #8, invalid var size", + .descr = "global data test #8, var is smaller than its type", .raw_types = { /* int */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -457,8 +457,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 0, .value_type_id = 7, .max_entries = 1, - .btf_load_err = true, - .err_str = "Invalid size", }, { .descr = "global data test #9, invalid var size", @@ -498,7 +496,7 @@ static struct btf_raw_test raw_tests[] = { .err_str = "Invalid size", }, { - .descr = "global data test #10, invalid var size", + .descr = "global data test #10, section is smaller than map value", .raw_types = { /* int */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ @@ -531,8 +529,7 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 0, .value_type_id = 7, .max_entries = 1, - .btf_load_err = true, - .err_str = "Invalid size", + .map_create_err = true, }, { .descr = "global data test #11, multiple section members", -- knotmirror 0.0.1 From 124a5a2bc0e28989aa007acf6543c826143cbf65 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:09 +0000 Subject: [PATCH 10/15] bpftool: Skip pieces of variables in DATASEC The kernel accepts a variable in DATASEC that is smaller than its type. It's a piece of a variable that the compiler split, with the type of the whole variable. bpftool prints a variable by its type, so it reads past the piece and, when the piece is the last in the section, past the buffer with the value of the map: $ bpftool map dump pinned /sys/fs/bpf/piece ".bss": [{ "whole": 67305985 },{ "piece.0": { "a": [134678021,1994496712,32592,32,0,833,0,414764080 Only the first number is in the map. Skip the pieces in btf_dumper_datasec(), like btf_datasec_show() in the kernel does, and in show_prog_metadata(). Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-11-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/bpf/bpftool/btf_dumper.c | 11 +++++++++++ tools/bpf/bpftool/main.h | 2 ++ tools/bpf/bpftool/prog.c | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index e4075824343f..3267163119dc 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -524,6 +524,15 @@ static int btf_dumper_var(const struct btf_dumper *d, __u32 type_id, return ret; } +/* + * The compiler splits a variable into pieces. Every piece is a VAR with the + * type of the whole variable, so there is less data than the type takes. + */ +bool btf_var_is_piece(const struct btf *btf, const struct btf_var_secinfo *vsi) +{ + return btf__resolve_size(btf, vsi->type) > vsi->size; +} + static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, const void *data) { @@ -542,6 +551,8 @@ static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, jsonw_name(d->jw, btf__name_by_offset(d->btf, t->name_off)); jsonw_start_array(d->jw); for (i = 0; i < vlen; i++) { + if (btf_var_is_piece(d->btf, &vsi[i])) + continue; ret = btf_dumper_do_type(d, vsi[i].type, 0, data + vsi[i].offset); if (ret) break; diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index fa2c877a3cbf..6851346a6030 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -121,6 +121,7 @@ struct obj_refs { }; struct btf; +struct btf_var_secinfo; struct bpf_line_info; int build_pinned_obj_table(struct hashmap *table, @@ -239,6 +240,7 @@ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id, const void *data); void btf_dumper_type_only(const struct btf *btf, __u32 func_type_id, char *func_only, int size); +bool btf_var_is_piece(const struct btf *btf, const struct btf_var_secinfo *vsi); void btf_dump_linfo_plain(const struct btf *btf, const struct bpf_line_info *linfo, diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index a60414aaa494..eb077236d2ea 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -348,7 +348,7 @@ static void show_prog_metadata(int fd, __u32 num_maps) t_var = btf__type_by_id(btf, vsi->type); name = btf__name_by_offset(btf, t_var->name_off); - if (!has_metadata_prefix(name)) + if (!has_metadata_prefix(name) || btf_var_is_piece(btf, vsi)) continue; if (!printed_header) { @@ -377,7 +377,7 @@ static void show_prog_metadata(int fd, __u32 num_maps) t_var = btf__type_by_id(btf, vsi->type); name = btf__name_by_offset(btf, t_var->name_off); - if (!has_metadata_prefix(name)) + if (!has_metadata_prefix(name) || btf_var_is_piece(btf, vsi)) continue; if (!printed_header) { -- knotmirror 0.0.1 From 87be058a3b6d32086ee84b0ce280092d82ef3541 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:10 +0000 Subject: [PATCH 11/15] selftests/bpf: Add tests for a variable that is smaller than its type Check that a variable in DATASEC that is smaller than its type is not printed when the map is read from bpffs and when bpftool dumps it. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-12-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/prog_tests/btf_rust.c | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_rust.c b/tools/testing/selftests/bpf/prog_tests/btf_rust.c index ce2b31087b59..daf777cadfda 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_rust.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_rust.c @@ -2,6 +2,7 @@ #include #include +#include "bpftool_helpers.h" #define FUNC_NAME "write_fmt" #define KSYM_NAME "write_fmt_scx_simple__BpfStream_" @@ -61,9 +62,81 @@ out: btf__free(btf); } +#define PIN_PATH "/sys/fs/bpf/btf_rust_piece" + +/* + * A piece of a static that LLVM split has the type of the whole static. + * It's the last variable in the section, so its type ends past the map value. + */ +static void test_piece(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int int_id, struct_id, var_id, piece_id, sec_id, map_fd = -1, key = 0; + __u32 value[2] = { 0x11111111, 0x22222222 }; + char line[256] = {}, out[1024] = {}; + struct btf *btf; + FILE *f = NULL; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf")) + return; + int_id = btf__add_int(btf, "u32", 4, 0); + ASSERT_GT(int_id, 0, "int"); + struct_id = btf__add_struct(btf, "Whole", 8); + ASSERT_GT(struct_id, 0, "struct"); + ASSERT_OK(btf__add_field(btf, "a", int_id, 0, 0), "field"); + ASSERT_OK(btf__add_field(btf, "b", int_id, 32, 0), "field"); + var_id = btf__add_var(btf, "CNT", BTF_VAR_STATIC, int_id); + ASSERT_GT(var_id, 0, "var"); + piece_id = btf__add_var(btf, "WHOLE.1", BTF_VAR_STATIC, struct_id); + ASSERT_GT(piece_id, 0, "piece"); + sec_id = btf__add_datasec(btf, ".bss", sizeof(value)); + ASSERT_GT(sec_id, 0, "datasec"); + ASSERT_OK(btf__add_datasec_var_info(btf, var_id, 0, 4), "var info"); + ASSERT_OK(btf__add_datasec_var_info(btf, piece_id, 4, 4), "piece info"); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf load")) + goto out; + + opts.btf_fd = btf__fd(btf); + opts.btf_value_type_id = sec_id; + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, ".bss", sizeof(key), sizeof(value), 1, &opts); + if (!ASSERT_GE(map_fd, 0, "map create")) + goto out; + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, value, 0), "map update")) + goto out; + + /* the variable is printed, the piece is not */ + unlink(PIN_PATH); + if (!ASSERT_OK(bpf_obj_pin(map_fd, PIN_PATH), "pin")) + goto out; + f = fopen(PIN_PATH, "r"); + if (!ASSERT_OK_PTR(f, "open")) + goto out; + while (fgets(line, sizeof(line), f) && line[0] == '#') + ; + ASSERT_HAS_SUBSTR(line, "286331153", "var"); + ASSERT_NULL(strstr(line, "572662306"), "piece"); + + /* the same for bpftool */ + if (!ASSERT_OK(get_bpftool_command_output("map dump pinned " PIN_PATH, out, sizeof(out)), + "bpftool")) + goto out; + ASSERT_HAS_SUBSTR(out, "CNT", "var"); + ASSERT_NULL(strstr(out, "WHOLE.1"), "piece"); +out: + if (f) + fclose(f); + unlink(PIN_PATH); + if (map_fd >= 0) + close(map_fd); + btf__free(btf); +} + /* Serial: programs are not in kallsyms while another test sets bpf_jit_harden */ void serial_test_btf_rust(void) { if (test__start_subtest("func_name")) test_func_name(); + if (test__start_subtest("piece")) + test_piece(); } -- knotmirror 0.0.1 From 8ab58ed42b086de4eb4035b4b6a75d29ac7f5e0f Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:11 +0000 Subject: [PATCH 12/15] libbpf: Keep global data in arena when the object has .arena.data All memory of a Rust program is arena, global data included. There are no address spaces in Rust to say so. LLVM also drops bounds checks of indexes into constant tables of core that it proved, so the verifier can't check the access as access to map value. The object asks for it with a section named .arena.data. What is in the section doesn't matter: #[used] #[link_section = ".arena.data"] static DATA_IN_ARENA: u8 = 0; or in C: char data_in_arena SEC(".arena.data"); There is nothing to tell to libbpf, so bpftool, veristat and other loaders work with such objects as they are. When the object has the section: - .data, .bss and .rodata sections are appended to arena data after __arena variables, at the alignment of the section. Like __arena variables the data is at the end of arena. - relocations of insns against the sections become relocations against the arena map. - array maps of the sections are not created. - when the object has no arena map libbpf creates one that is as large as the data. bpf_object__find_map_by_name(obj, "arena") finds it and bpf_map__set_max_entries() makes room for allocations. __arena variables still need the arena map to be declared. Read-only sections that stay frozen array maps: - .data.rel.ro and sections that have relocations in them. The kernel recognizes pointers to functions in them by value for callx. - .rodata.str*. They hold const strings for __str arguments of kfuncs. A copy of them is in arena too. The program dereferences pointers that it loads from data, so pointers to data that are stored in data become addresses of arena, wherever the pointer is. The arena map is created ahead of the other maps then, since its address goes into their data. A pointer to .rodata.str* points to the copy. A pointer to a section that is not in arena fails the load: sec '.data': pointer to 'tbl' at offset 8 can't be resolved: sec '.data.rel.ro' is not in arena So does an arena that is pinned or reused. libbpf doesn't mmap it, so its address is not known. Programs of the object are loaded with BPF_F_ARENA_SCALAR, since they access the data through plain numbers. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-13-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/lib/bpf/libbpf.c | 433 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 413 insertions(+), 20 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index fdd69aac39bd..cb09ded90773 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -552,6 +552,26 @@ struct bpf_struct_ops { #define STRUCT_OPS_SEC ".struct_ops" #define STRUCT_OPS_LINK_SEC ".struct_ops.link" #define ARENA_SEC ".addr_space.1" +/* + * An object with this section keeps its global data in arena instead of + * array maps. What is in the section doesn't matter. .data, .bss and .rodata + * sections become a part of the arena map of the object, like __arena global + * variables are. If the object doesn't declare an arena map libbpf creates + * one that is just large enough for the data. bpf_object__find_map_by_name() + * finds it by the name "arena" and bpf_map__set_max_entries() changes its + * size before the object is loaded. + * Read-only sections with pointers to functions and sections with constant + * strings stay frozen array maps. A constant that a helper or a kfunc takes + * by pointer has to be in such section. + * Pointers to data that are stored in data become addresses of arena. + * The load fails if such pointer points to a section that is not in arena, + * or if the arena is pinned or reused. + * + * It's for programs written in Rust. There are no address spaces in Rust, + * the program accesses arena through plain numbers, so programs of the object + * are loaded with BPF_F_ARENA_SCALAR. + */ +#define ARENA_DATA_SEC ".arena.data" enum libbpf_map_type { LIBBPF_MAP_UNSPEC, @@ -607,6 +627,13 @@ struct bpf_map { /* pointers to functions in the data of an internal map, see obj->func_ptrs */ struct func_ptr *func_ptrs; size_t func_ptr_cnt; + /* + * Data of the internal map is a part of arena data at arena_off. + * The map is not created, unless it's a copy that is in arena. + */ + bool in_arena; + bool arena_copy; + size_t arena_off; }; enum extern_type { @@ -775,6 +802,17 @@ struct bpf_object { void *arena_data; size_t arena_data_sz; size_t arena_data_off; + bool data_in_arena; + bool arena_mapped; + /* pointers to data in the data of internal maps, when data is in arena */ + struct data_ptr { + int sec_idx; + size_t sec_off; + int targ_sec_idx; + size_t targ_off; + char *sym_name; + } *data_ptrs; + size_t data_ptr_cnt; void *jumptables_data; size_t jumptables_data_sz; @@ -1600,6 +1638,7 @@ static struct bpf_object *bpf_object__new(const char *path, obj->efile.obj_buf = obj_buf; obj->efile.obj_buf_sz = obj_buf_sz; obj->efile.btf_maps_shndx = -1; + obj->efile.arena_data_shndx = -1; obj->kconfig_map_idx = -1; obj->arena_map_idx = -1; @@ -3081,6 +3120,136 @@ static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, return 0; } +static bool map_is_const_str(const struct bpf_map *map) +{ + return map->libbpf_type == LIBBPF_MAP_RODATA && + str_has_pfx(map->real_name, RODATA_SEC ".str"); +} + +static bool map_data_goes_to_arena(const struct bpf_object *obj, const struct bpf_map *map) +{ + int i; + + switch (map->libbpf_type) { + case LIBBPF_MAP_DATA: + case LIBBPF_MAP_BSS: + return true; + case LIBBPF_MAP_RODATA: + break; + default: + return false; + } + + /* + * Const strings that kfuncs and helpers take have to be in a frozen + * map. Pointers to them that are stored in data point to a copy of + * the strings that is in arena. + */ + if (map_is_const_str(map)) + return true; + /* + * Read-only data with pointers stays a frozen map: the kernel + * recognizes pointers to functions in it. + */ + if (str_has_pfx(map->real_name, DATA_REL_RO_SEC)) + return false; + for (i = 0; i < obj->efile.sec_cnt; i++) { + const struct elf_sec_desc *sec = &obj->efile.secs[i]; + + if (sec->sec_type == SEC_RELO && sec->shdr->sh_info == map->sec_idx) + return false; + } + return true; +} + +/* Returns the offset of the data of the map in arena data that is sz bytes so far */ +static size_t map_arena_off(const struct bpf_object *obj, const struct bpf_map *map, size_t sz) +{ + size_t align = obj->efile.secs[map->sec_idx].shdr->sh_addralign; + + return roundup(sz, max(align, sizeof(__u64))); +} + +/* + * Make .data, .bss and .rodata a part of arena data. They follow __arena + * variables of the object, if there are any. + */ +static int bpf_object__init_arena_data(struct bpf_object *obj) +{ + const size_t page_sz = sysconf(_SC_PAGE_SIZE); + size_t sz = obj->arena_data_sz; + struct bpf_map *map; + void *data; + int i; + + if (!obj->data_in_arena) + return 0; + + for (i = 0; i < obj->nr_maps; i++) { + map = &obj->maps[i]; + if (map_data_goes_to_arena(obj, map)) + sz = map_arena_off(obj, map, sz) + map->def.value_size; + } + if (sz == obj->arena_data_sz) + return 0; + + if (obj->arena_map_idx < 0) { + /* bpf_object__init_user_btf_maps() didn't look: there is no .maps section */ + if (obj->efile.arena_data) { + pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n", + ARENA_SEC); + return -ENOENT; + } + + map = bpf_object__add_map(obj); + if (IS_ERR(map)) + return PTR_ERR(map); + + map->real_name = strdup("arena"); + map->name = strdup("arena"); + if (!map->real_name || !map->name) { + zfree(&map->real_name); + zfree(&map->name); + return -ENOMEM; + } + map->sec_idx = -1; + map->def.type = BPF_MAP_TYPE_ARENA; + map->def.max_entries = roundup(sz, page_sz) / page_sz; + map->def.map_flags = BPF_F_MMAPABLE; + obj->arena_map_idx = map - obj->maps; + } + + data = realloc(obj->arena_data, sz); + if (!data) + return -ENOMEM; + memset(data + obj->arena_data_sz, 0, sz - obj->arena_data_sz); + sz = obj->arena_data_sz; + obj->arena_data = data; + obj->maps[obj->arena_map_idx].mmaped = data; + + for (i = 0; i < obj->nr_maps; i++) { + map = &obj->maps[i]; + if (!map_data_goes_to_arena(obj, map)) + continue; + sz = map_arena_off(obj, map, sz); + map->arena_off = sz; + memcpy(data + sz, map->mmaped, map->def.value_size); + sz += map->def.value_size; + pr_debug("map '%s': data is in arena at offset %zu\n", map->name, map->arena_off); + if (map_is_const_str(map)) { + map->arena_copy = true; + continue; + } + munmap(map->mmaped, bpf_map_mmap_sz(map)); + /* make bpf_map__initial_value() and bpf_map__set_initial_value() work */ + map->mmaped = data + map->arena_off; + map->autocreate = false; + map->in_arena = true; + } + obj->arena_data_sz = sz; + return 0; +} + static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, const char *pin_root_path) { @@ -3173,6 +3342,7 @@ static int bpf_object__init_maps(struct bpf_object *obj, err = err ?: bpf_object__init_global_data_maps(obj); err = err ?: bpf_object__init_kconfig_map(obj); err = err ?: bpf_object_init_struct_ops(obj); + err = err ?: bpf_object__init_arena_data(obj); return err; } @@ -3987,6 +4157,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!sh) return -LIBBPF_ERRNO__FORMAT; + /* the second pass has to know it when it sees relocations of data */ + name = elf_sec_str(obj, sh->sh_name); + if (name && strcmp(name, ARENA_DATA_SEC) == 0) + obj->data_in_arena = true; + if (sh->sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { pr_warn("elf: multiple symbol tables in %s\n", obj->path); @@ -4103,6 +4278,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj) } else if (strcmp(name, ARENA_SEC) == 0) { obj->efile.arena_data = data; obj->efile.arena_data_shndx = idx; + } else if (strcmp(name, ARENA_DATA_SEC) == 0) { + /* the marker, see the first pass */ } else if (strcmp(name, JUMPTABLES_SEC) == 0) { obj->jumptables_data = malloc(data->d_size); if (!obj->jumptables_data) @@ -4127,6 +4304,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj) * have pointers to functions. */ if (!section_have_execinstr(obj, targ_sec_idx) && + !(obj->data_in_arena && + (!strcmp(name, ".rel" DATA_SEC) || + str_has_pfx(name, ".rel" DATA_SEC "."))) && strcmp(name, ".rel" RODATA_SEC) && !str_has_pfx(name, ".rel" RODATA_SEC ".") && strcmp(name, ".rel" DATA_REL_RO_SEC) && @@ -4878,6 +5058,10 @@ static int bpf_program__record_reloc(struct bpf_program *prog, reloc_desc->insn_idx = insn_idx; reloc_desc->map_idx = map_idx; reloc_desc->sym_off = sym->st_value; + if (map->in_arena) { + reloc_desc->map_idx = obj->arena_map_idx; + reloc_desc->sym_off += map->arena_off; + } return 0; } @@ -5096,6 +5280,9 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) if (map_is_created(map)) return libbpf_err(-EBUSY); + if (map->in_arena) + return libbpf_err(-EOPNOTSUPP); + map->autocreate = autocreate; return 0; } @@ -5376,6 +5563,45 @@ bpf_object__reuse_map(struct bpf_map *map) return 0; } +static struct bpf_map *bpf_object__sec_map(struct bpf_object *obj, int sec_idx) +{ + int i; + + for (i = 0; i < obj->nr_maps; i++) + if (bpf_map__is_internal(&obj->maps[i]) && obj->maps[i].sec_idx == sec_idx) + return &obj->maps[i]; + return NULL; +} + +static int bpf_object__mmap_arena(struct bpf_object *obj, struct bpf_map *map) +{ + int i, err; + + map->mmaped = mmap((void *)(long)map->map_extra, + bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, + map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, + map->fd, 0); + if (map->mmaped == MAP_FAILED) { + err = -errno; + map->mmaped = NULL; + pr_warn("map '%s': failed to mmap arena: %s\n", + map->name, errstr(err)); + return err; + } + if (obj->arena_data) { + memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, + obj->arena_data_sz); + zfree(&obj->arena_data); + } + for (i = 0; i < obj->nr_maps; i++) { + struct bpf_map *m = &obj->maps[i]; + + if (m->in_arena) + m->mmaped = map->mmaped + obj->arena_data_off + m->arena_off; + } + return 0; +} + static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { @@ -5736,6 +5962,10 @@ bpf_object__create_maps(struct bpf_object *obj) continue; } + /* see bpf_object__relocate_data_ptrs() */ + if (map->def.type == BPF_MAP_TYPE_ARENA && obj->arena_mapped) + continue; + err = map_set_def_max_entries(map); if (err) goto err_out; @@ -5773,22 +6003,9 @@ retry: if (err < 0) goto err_out; } else if (map->def.type == BPF_MAP_TYPE_ARENA) { - map->mmaped = mmap((void *)(long)map->map_extra, - bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, - map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, - map->fd, 0); - if (map->mmaped == MAP_FAILED) { - err = -errno; - map->mmaped = NULL; - pr_warn("map '%s': failed to mmap arena: %s\n", - map->name, errstr(err)); + err = bpf_object__mmap_arena(obj, map); + if (err) return err; - } - if (obj->arena_data) { - memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, - obj->arena_data_sz); - zfree(&obj->arena_data); - } } if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { err = init_map_in_map_slots(obj, map); @@ -7803,6 +8020,81 @@ err_out: return err; } +/* + * Turn pointers to data that are stored in data into addresses of arena. + * The arena is created here, ahead of the other maps: the address has to be + * known before the maps that hold the pointers are created. + */ +static int bpf_object__relocate_data_ptrs(struct bpf_object *obj) +{ + struct bpf_map *arena, *map, *targ; + __u64 addr, val; + size_t i; + int err; + + if (!obj->data_ptr_cnt) + return 0; + + for (i = 0; i < obj->data_ptr_cnt; i++) { + struct data_ptr *p = &obj->data_ptrs[i]; + + if (p->targ_sec_idx < 0) + continue; + map = bpf_object__sec_map(obj, p->sec_idx); + targ = bpf_object__sec_map(obj, p->targ_sec_idx); + if (map && (map->in_arena || map->autocreate) && + (!targ || (!targ->in_arena && !targ->arena_copy))) { + pr_warn("sec '%s': pointer to '%s' at offset %zu can't be resolved: sec '%s' is not in arena\n", + map->real_name, p->sym_name, p->sec_off, + targ ? targ->real_name : ""); + return -LIBBPF_ERRNO__RELOC; + } + } + + if (obj->gen_loader) { + pr_warn("pointers to data in data are not supported by light skeleton\n"); + return -ENOTSUP; + } + + if (obj->arena_map_idx < 0) + return 0; + + arena = &obj->maps[obj->arena_map_idx]; + if (!arena->autocreate) + return 0; + /* libbpf doesn't mmap an arena that it didn't create, so the address is not known */ + if (arena->reused || arena->pin_path) { + pr_warn("map '%s': pointers to data in data are not supported with pinned or reused arena\n", + arena->name); + return -ENOTSUP; + } + + err = bpf_object__create_map(obj, arena, false); + err = err ?: bpf_object__mmap_arena(obj, arena); + if (err) { + pr_warn("map '%s': failed to create: %s\n", arena->name, errstr(err)); + return err; + } + obj->arena_mapped = true; + + for (i = 0; i < obj->data_ptr_cnt; i++) { + struct data_ptr *p = &obj->data_ptrs[i]; + + map = bpf_object__sec_map(obj, p->sec_idx); + if (!map || (!map->in_arena && !map->autocreate)) + continue; + + addr = (__u64)(unsigned long)arena->mmaped + obj->arena_data_off + p->targ_off; + if (p->targ_sec_idx >= 0) + addr += bpf_object__sec_map(obj, p->targ_sec_idx)->arena_off; + + memcpy(&val, map->mmaped + p->sec_off, sizeof(val)); + val += addr; + memcpy(map->mmaped + p->sec_off, &val, sizeof(val)); + } + return 0; +} + static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) { struct bpf_program *prog; @@ -7825,8 +8117,10 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat size_t mmap_sz = bpf_map_mmap_sz(arena_map); if (data_sz > mmap_sz) { - pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", - arena_map->name, mmap_sz, obj->arena_data_sz); + pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global %s of size %zu\n", + arena_map->name, mmap_sz, + obj->data_in_arena ? "data" : "__arena variables", + obj->arena_data_sz); return -E2BIG; } @@ -7835,6 +8129,10 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat obj->arena_data_off = mmap_sz - data_sz; } + err = bpf_object__relocate_data_ptrs(obj); + if (err) + return err; + /* Before relocating calls pre-process relocations and mark * few ld_imm64 instructions that points to subprogs. * Otherwise bpf_object__reloc_code() later would have to consider @@ -8067,12 +8365,85 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, return 0; } +/* + * A pointer to data that is stored in data. When data is in arena + * the program dereferences what it loads from data, so the pointer has + * to be the address of the target in arena. + */ +static int bpf_object__collect_data_ptr(struct bpf_object *obj, const char *relo_sec_name, + int relo_idx, size_t sec_idx, const Elf64_Rel *rel, + const Elf64_Sym *sym) +{ + Elf_Data *scn_data = obj->efile.secs[sec_idx].data; + const char *sym_name = elf_sym_str(obj, sym->st_name) ?: ""; + struct data_ptr *ptrs; + + if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_shndx < obj->efile.sec_cnt) + sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)) ?: ""; + + if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || + sym->st_shndx >= obj->efile.sec_cnt || + (sym->st_shndx != obj->efile.arena_data_shndx && + !bpf_object__shndx_is_data(obj, sym->st_shndx)) || + rel->r_offset >= scn_data->d_size || + scn_data->d_size - rel->r_offset < sizeof(__u64)) { + pr_warn("sec '%s': relo #%d: can't resolve pointer to '%s' at offset %zu when data is in arena\n", + relo_sec_name, relo_idx, sym_name, (size_t)rel->r_offset); + return -LIBBPF_ERRNO__RELOC; + } + + ptrs = libbpf_reallocarray(obj->data_ptrs, obj->data_ptr_cnt + 1, sizeof(*ptrs)); + if (!ptrs) + return -ENOMEM; + obj->data_ptrs = ptrs; + + ptrs[obj->data_ptr_cnt].sec_idx = sec_idx; + ptrs[obj->data_ptr_cnt].sec_off = rel->r_offset; + /* __arena variables are at the start of arena data */ + ptrs[obj->data_ptr_cnt].targ_sec_idx = + sym->st_shndx == obj->efile.arena_data_shndx ? -1 : sym->st_shndx; + ptrs[obj->data_ptr_cnt].targ_off = sym->st_value; + ptrs[obj->data_ptr_cnt].sym_name = strdup(sym_name); + if (!ptrs[obj->data_ptr_cnt].sym_name) + return -ENOMEM; + obj->data_ptr_cnt++; + + pr_debug("sec '%s': relo #%d: pointer at offset %zu to '%s'\n", + relo_sec_name, relo_idx, (size_t)rel->r_offset, sym_name); + return 0; +} + +/* Collect pointers in a data section that went to arena */ +static int bpf_object__collect_data_relos(struct bpf_object *obj, + Elf64_Shdr *shdr, Elf_Data *data) +{ + int i, err, nrels = shdr->sh_size / shdr->sh_entsize; + const char *relo_sec_name; + Elf64_Sym *sym; + Elf64_Rel *rel; + + relo_sec_name = elf_sec_str(obj, shdr->sh_name) ?: ""; + for (i = 0; i < nrels; i++) { + rel = elf_rel_by_idx(data, i); + sym = rel ? elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)) : NULL; + if (!sym) { + pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); + return -LIBBPF_ERRNO__FORMAT; + } + err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, shdr->sh_info, rel, sym); + if (err) + return err; + } + return 0; +} + /* * Collect pointers to functions in a read-only data section. They are * R_BPF_64_ABS64 relocations against .text section, where the offset of * a static function in the section is stored in place. Relocations in data * sections were ignored before pointers to functions were supported. Those - * that are something else, e.g. pointers to data, still are. + * that are something else, e.g. pointers to data, still are, unless data + * is in arena. */ static int bpf_object__collect_rodata_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) @@ -8108,6 +8479,15 @@ static int bpf_object__collect_rodata_relos(struct bpf_object *obj, if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || !sym_is_subprog(sym, obj->efile.text_shndx)) { + int err; + + if (obj->data_in_arena) { + err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, + sec_idx, rel, sym); + if (err) + return err; + continue; + } pr_debug("sec '%s': relo #%d: not a pointer to a function, skipping...\n", relo_sec_name, i); continue; @@ -8183,6 +8563,8 @@ static int bpf_object__collect_relos(struct bpf_object *obj) if (obj->efile.secs[idx].sec_type == SEC_RODATA) err = bpf_object__collect_rodata_relos(obj, shdr, data); + else if (obj->efile.secs[idx].sec_type == SEC_DATA) + err = bpf_object__collect_data_relos(obj, shdr, data); else if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) err = bpf_object__collect_st_ops_relos(obj, shdr, data); else if (idx == obj->efile.btf_maps_shndx) @@ -8476,6 +8858,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog } load_attr.log_level = log_level; load_attr.prog_flags = prog->prog_flags; + /* the program accesses its data in arena through plain numbers */ + if (obj->data_in_arena) + load_attr.prog_flags |= BPF_F_ARENA_SCALAR; load_attr.fd_array = obj->fd_array; load_attr.token_fd = obj->token_fd; @@ -8547,7 +8932,7 @@ retry_load: for (i = 0; i < obj->nr_maps; i++) { map = &prog->obj->maps[i]; - if (map->libbpf_type != LIBBPF_MAP_RODATA) + if (map->libbpf_type != LIBBPF_MAP_RODATA || map->in_arena) continue; if (bpf_prog_bind_map(ret, map->fd, NULL)) { @@ -10111,7 +10496,7 @@ static void bpf_map__destroy(struct bpf_map *map) zfree(&map->init_slots); map->init_slots_sz = 0; - if (map->mmaped && map->mmaped != map->obj->arena_data) + if (map->mmaped && map->mmaped != map->obj->arena_data && !map->in_arena) munmap(map->mmaped, bpf_map_mmap_sz(map)); map->mmaped = NULL; @@ -10196,6 +10581,9 @@ void bpf_object__close(struct bpf_object *obj) close(obj->func_ptr_maps[i].fd); zfree(&obj->func_ptr_maps); zfree(&obj->func_ptrs); + for (i = 0; i < obj->data_ptr_cnt; i++) + zfree(&obj->data_ptrs[i].sym_name); + zfree(&obj->data_ptrs); if (obj->btf_module_allowlist) { for (i = 0; i < obj->btf_module_allowlist_cnt; i++) @@ -10556,6 +10944,8 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; + if (obj->data_in_arena) + attr.prog_flags |= BPF_F_ARENA_SCALAR; prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); if (!prog_btf_fd && obj->btf) @@ -11562,6 +11952,9 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) if (map_is_created(map)) return libbpf_err(-EBUSY); + if (map->in_arena) + return libbpf_err(-EOPNOTSUPP); + if (map->mmaped) { size_t mmap_old_sz, mmap_new_sz; int err; -- knotmirror 0.0.1 From 373e1223ff00630dc28b2d0fb6ec5fb5cceaccce Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:12 +0000 Subject: [PATCH 13/15] libbpf: Keep format strings of bpf_printk() in .rodata.str bpf_printk(), BPF_SNPRINTF(), BPF_SEQ_PRINTF() and bpf_stream_printk() copy the format into a static const array, which the compiler puts into .rodata. When global data of the object is in arena .rodata is there too and the helper doesn't take the format: R1 type=scalar expected=fp, pkt, pkt_meta, map_key, map_value, mem, ringbuf_mem, buf, trusted_ptr_, ctx String literals are in .rodata.str1.1, which libbpf keeps in a read-only map. No compiler flag moves a named array there. Put the arrays into .rodata.str with a section attribute. An object that uses the macros gets one more map, .rodata.str. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-14-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/lib/bpf/bpf_helpers.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index 9d160b5b9c0e..b37b727ef216 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h @@ -248,13 +248,21 @@ enum libbpf_tristate { #define ___bpf_fill(arr, args...) \ ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) +/* + * Format strings are in a section of their own. When global data of the object + * is in arena (the object has .arena.data section) libbpf keeps .rodata.str* + * sections in read-only maps, which is where helpers and kfuncs take strings + * from. + */ +#define ___bpf_fmt_sec __attribute__((section(".rodata.str"))) + /* * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values * in a structure. */ #define BPF_SEQ_PRINTF(seq, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -272,7 +280,7 @@ enum libbpf_tristate { */ #define BPF_SNPRINTF(out, out_size, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -287,7 +295,7 @@ enum libbpf_tristate { #ifdef BPF_NO_GLOBAL_DATA #define BPF_PRINTK_FMT_MOD #else -#define BPF_PRINTK_FMT_MOD static const +#define BPF_PRINTK_FMT_MOD static const ___bpf_fmt_sec #endif #define __bpf_printk(fmt, ...) \ @@ -303,7 +311,7 @@ enum libbpf_tristate { */ #define __bpf_vprintk(fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -317,7 +325,7 @@ enum libbpf_tristate { #define bpf_stream_printk(stream_id, fmt, args...) \ ({ \ - static const char ___fmt[] = fmt; \ + static const char ___fmt[] ___bpf_fmt_sec = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ -- knotmirror 0.0.1 From 27e24617a83b53e0f008837f7a72ba944a078149 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:13 +0000 Subject: [PATCH 14/15] selftests/bpf: Add test for global data in arena Load the program that reads and writes .data, .bss and .rodata from the object with .arena.data section. Check that initial values set through the skeleton reach the program and that the values written by the program are seen through the skeleton. Also check: - alignment of a section. - pointers to data and to a const string that are stored in data and in read-only data, next to a pointer to a function that callx uses. - bpf_strncmp() with a string literal, bpf_printk() and BPF_SNPRINTF(). - an object that has an arena map and __arena variables. - the load fails when a pointer in data points to a section that is not in arena or to a variable of the kernel. - the load fails when there are pointers in data and the arena is pinned or reused. - the open fails when there are __arena variables and no arena map. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-15-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile.skel | 3 +- .../selftests/bpf/prog_tests/data_in_arena.c | 191 ++++++++++++++++++ .../selftests/bpf/progs/data_in_arena.c | 107 ++++++++++ .../selftests/bpf/progs/data_in_arena_decl.c | 37 ++++ .../bpf/progs/data_in_arena_extern.c | 20 ++ .../selftests/bpf/progs/data_in_arena_fail.c | 20 ++ .../selftests/bpf/progs/data_in_arena_nomap.c | 20 ++ 7 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/data_in_arena.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_decl.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_extern.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_fail.c create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_nomap.c diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 2e22bb901bf3..765666a64b89 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -20,7 +20,8 @@ ifneq ($(BPF_CC),) BPF_SRCS := $(notdir $(wildcard progs/*.c)) BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) -SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c +SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c \ + data_in_arena_extern.c data_in_arena_nomap.c LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ linked_vars.skel.h linked_maps.skel.h linked_arena.skel.h \ diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c new file mode 100644 index 000000000000..86dc6798bbde --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "data_in_arena.skel.h" +#include "data_in_arena_decl.skel.h" +#include "data_in_arena_fail.skel.h" + +static int run_prog(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &topts), "test_run")) + return -1; + return topts.retval; +} + +static void run(struct data_in_arena *skel, int counter) +{ + int i; + + ASSERT_EQ(run_prog(skel->progs.use_data), counter + 7 + 1 + 2, "retval"); + ASSERT_EQ(skel->bss->sum, counter + 7 + 1 + 2, "sum"); + ASSERT_EQ(skel->data->counter, counter + 1, "counter"); + ASSERT_EQ(skel->data->pair[1], 5, "pair[1]"); + for (i = 0; i < 4; i++) + ASSERT_EQ(skel->bss->table[i], 10 * (i + 1) + i, "table"); +} + +static void test_in_arena(void) +{ + struct bpf_map_info info = {}; + struct data_in_arena *skel; + __u32 len = sizeof(info); + struct bpf_map *arena; + size_t sz; + + skel = data_in_arena__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + arena = bpf_object__find_map_by_name(skel->obj, "arena"); + if (!ASSERT_OK_PTR(arena, "arena")) + goto out; + ASSERT_EQ(bpf_map__type(arena), BPF_MAP_TYPE_ARENA, "arena type"); + ASSERT_EQ(bpf_map__max_entries(arena), 1, "arena pages"); + ASSERT_OK(bpf_map__set_max_entries(arena, 8), "arena resize"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.data), "data autocreate"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.bss), "bss autocreate"); + ASSERT_FALSE(bpf_map__autocreate(skel->maps.rodata), "rodata autocreate"); + ASSERT_EQ(bpf_map__set_autocreate(skel->maps.data, true), -EOPNOTSUPP, "set_autocreate"); + ASSERT_EQ(bpf_map__set_value_size(skel->maps.bss, 4096), -EOPNOTSUPP, "set_value_size"); + ASSERT_EQ(bpf_map__initial_value(skel->maps.data, &sz), skel->data, "initial_value"); + ASSERT_EQ(sz, sizeof(*skel->data), "initial_value size"); + + /* initial values are set the usual way */ + skel->data->counter = 100; + + if (!ASSERT_OK(data_in_arena__load(skel), "load")) + goto out; + /* there are no maps behind the sections */ + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.data), &info, &len), "data map"); + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &info, &len), "bss map"); + ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.rodata), &info, &len), + "rodata map"); + run(skel, 100); + + /* pointers to data next to pointers to functions */ + ASSERT_EQ(run_prog(skel->progs.use_ops), 42 + 1 + 'e', "use_ops"); + ASSERT_EQ(skel->data->counter, 102, "counter"); + + /* alignment of sections and pointers to data in data */ + ASSERT_EQ((unsigned long)&skel->bss->aligned64 % 64, 0, "alignment"); + ASSERT_EQ(run_prog(skel->progs.use_ptrs), 0, "use_ptrs"); + ASSERT_EQ(skel->data->x, 43, "x"); + ASSERT_EQ(skel->bss->aligned64.v[7], 7, "aligned64"); + ASSERT_EQ(skel->data->px, &skel->data->x, "px"); + + /* format strings of bpf_printk() and BPF_SNPRINTF() */ + ASSERT_EQ(run_prog(skel->progs.use_printk), sizeof("43-7"), "use_printk"); + ASSERT_STREQ(skel->bss->out, "43-7", "out"); +out: + data_in_arena__destroy(skel); +} + +/* The object has an arena map and __arena variables */ +static void test_declared_arena(void) +{ + struct data_in_arena_decl *skel; + struct bpf_map *map; + int arenas = 0; + + skel = data_in_arena_decl__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + bpf_object__for_each_map(map, skel->obj) + arenas += bpf_map__type(map) == BPF_MAP_TYPE_ARENA; + ASSERT_EQ(arenas, 1, "no second arena"); + skel->data->counter = 6; + if (!ASSERT_OK(data_in_arena_decl__load(skel), "load")) + goto out; + ASSERT_EQ(run_prog(skel->progs.use_data), 6 + 7 + 11, "retval"); + ASSERT_EQ(run_prog(skel->progs.use_data), 7 + 7 + 12, "retval"); + ASSERT_EQ(skel->bss->sum, 7 + 7 + 12, "sum"); + ASSERT_EQ(skel->data->counter, 8, "counter"); +out: + data_in_arena_decl__destroy(skel); +} + +/* A pointer in data that can't be made an address of arena fails the load */ +static void test_ptr_to_map(void) +{ + struct data_in_arena_fail *skel; + + skel = data_in_arena_fail__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + ASSERT_ERR(data_in_arena_fail__load(skel), "load"); + data_in_arena_fail__destroy(skel); +} + +/* So does a pointer to a variable of the kernel. There is no skeleton: the open fails. */ +static void test_ptr_to_extern(void) +{ + struct bpf_object *obj; + + obj = bpf_object__open_file("./data_in_arena_extern.bpf.o", NULL); + if (!ASSERT_ERR_PTR(obj, "open")) + bpf_object__close(obj); +} + +/* __arena variables and no arena map. There is no skeleton: the open fails. */ +static void test_arena_var_no_map(void) +{ + struct bpf_object *obj; + + obj = bpf_object__open_file("./data_in_arena_nomap.bpf.o", NULL); + if (!ASSERT_ERR_PTR(obj, "open")) + bpf_object__close(obj); +} + +/* The address of an arena that libbpf doesn't create is not known. No pointers in data then. */ +static void test_not_my_arena(bool pin) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); + struct data_in_arena *skel; + struct bpf_map *arena; + int fd = -1; + + skel = data_in_arena__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + arena = bpf_object__find_map_by_name(skel->obj, "arena"); + if (!ASSERT_OK_PTR(arena, "arena")) + goto out; + if (pin) { + ASSERT_OK(bpf_map__set_pin_path(arena, "/sys/fs/bpf/data_in_arena"), "pin_path"); + } else { + fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, 1, &opts); + if (!ASSERT_GE(fd, 0, "map_create")) + goto out; + ASSERT_OK(bpf_map__reuse_fd(arena, fd), "reuse_fd"); + } + ASSERT_EQ(data_in_arena__load(skel), -ENOTSUP, "load"); +out: + if (fd >= 0) + close(fd); + data_in_arena__destroy(skel); +} + +void test_data_in_arena(void) +{ +#if !defined(__x86_64__) && !defined(__aarch64__) + /* other JITs don't take BPF_F_ARENA_SCALAR */ + test__skip(); + return; +#endif + if (test__start_subtest("arena")) + test_in_arena(); + if (test__start_subtest("declared_arena")) + test_declared_arena(); + if (test__start_subtest("ptr_to_map")) + test_ptr_to_map(); + if (test__start_subtest("ptr_to_extern")) + test_ptr_to_extern(); + if (test__start_subtest("arena_var_no_map")) + test_arena_var_no_map(); + if (test__start_subtest("pinned_arena")) + test_not_my_arena(true); + if (test__start_subtest("reused_arena")) + test_not_my_arena(false); +} diff --git a/tools/testing/selftests/bpf/progs/data_in_arena.c b/tools/testing/selftests/bpf/progs/data_in_arena.c new file mode 100644 index 000000000000..ce3838327d6f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +long pair[2] = { 1, 2 }; +int x = 42; +long sum; +long table[4]; +struct { + long v[8]; +} aligned64 __attribute__((aligned(64))); +const volatile int ro = 7; +const volatile long ro_table[4] = { 10, 20, 30, 40 }; + +/* const strings stay in a map for helpers and kfuncs, a copy of them is in arena */ +const char hello[] SEC(".rodata.str.hello") = "hello"; + +/* pointers to data that are stored in data */ +int *px = &x; +const char *str = hello; +int *const volatile cpx SEC(".data.rel.ro") = &x; + +SEC("syscall") +int use_data(void *ctx) +{ + int i; + + for (i = 0; i < 4; i++) + table[i] = ro_table[i] + i; + sum = counter + ro + pair[0] + pair[1]; + counter++; + __sync_fetch_and_add(&pair[1], 3); + return sum; +} + +typedef int (*op_fn)(int); + +static __noinline int add1(int v) +{ + return v + 1; +} + +/* + * Pointers to functions and to data in read-only data of a program with callx. + * Volatile, so that the compiler doesn't replace the pointers with what + * they point to. + */ +static const volatile struct { + op_fn fn; + int *data; + const char *name; +} ops SEC(".data.rel.ro") = { add1, &x, hello }; + +SEC("syscall") +int use_ops(void *ctx) +{ + /* a program has the arena when its code refers to it */ + counter++; + return ops.fn(*ops.data) + ops.name[1]; +} + +SEC("syscall") +int use_ptrs(void *ctx) +{ + unsigned long addr = (unsigned long)&aligned64; + char local[4] = "abc"; + + /* hide the address from the compiler, it knows that '& 63' is 0 */ + asm volatile ("" : "+r"(addr)); + if (addr & 63) + return 1; + aligned64.v[7] = 7; + if (*px != 42) + return 2; + *px = 43; + if (x != 43 || *cpx != 43) + return 3; + if (str[0] != 'h' || str[4] != 'o' || str[5]) + return 4; + /* the literal is in a map */ + if (bpf_strncmp(local, sizeof(local), "abc")) + return 5; + return 0; +} + +char out[16]; + +/* format strings are in a map */ +SEC("syscall") +int use_printk(void *ctx) +{ + char buf[sizeof(out)]; + int i, n; + + bpf_printk("counter %d", counter); + n = BPF_SNPRINTF(buf, sizeof(buf), "%d-%d", x, ro); + for (i = 0; i < sizeof(out); i++) + out[i] = buf[i]; + return n; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c new file mode 100644 index 000000000000..01bc4fea8f0c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include "bpf_experimental.h" +#include + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 4); +} arena SEC(".maps"); + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +long sum; +const volatile int ro = 7; + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) +int __arena avar = 11; +#else +int avar = 11; +#endif + +SEC("syscall") +int use_data(void *ctx) +{ + sum = counter + ro + avar; + counter++; + avar++; + return sum; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c new file mode 100644 index 000000000000..dc1ad5ca3bdd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +extern const int bpf_prog_active __ksym; + +/* the variable of the kernel is not in arena */ +const void *kp = &bpf_prog_active; + +SEC("syscall") +int ptr_to_extern(void *ctx) +{ + return *(int *)kp; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c new file mode 100644 index 000000000000..ad8afe9afc96 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int x = 42; +/* the table is read-only data with pointers. It's not in arena. */ +int *const volatile tbl[1] SEC(".data.rel.ro") = { &x }; +int *const volatile *pp = tbl; + +SEC("syscall") +int ptr_to_map(void *ctx) +{ + return **pp; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c new file mode 100644 index 000000000000..da27a8f10d6f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_arena_common.h" + +/* global data of the object is in arena */ +char data_in_arena SEC(".arena.data"); + +int counter = 5; +/* needs an arena map that is declared. The one that libbpf creates won't do. */ +int __arena avar = 11; + +SEC("syscall") +int arena_var(void *ctx) +{ + return avar + counter; +} + +char _license[] SEC("license") = "GPL"; -- knotmirror 0.0.1 From e86cb7315451e166f1392032d392f1736fa6853e Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 2 Oct 2026 12:47:14 +0000 Subject: [PATCH 15/15] selftests/bpf: Add test for global data of a program in Rust data_in_arena_rust.rs shows why libbpf keeps .data, .bss and .rodata of a program in Rust in arena. A reference in Rust is an address that can be stored in data. The list in the test has a node in each of the three sections. IN_BSS.next is stored by the program, IN_DATA.next is a relocation in .data against .rodata. sum() follows them and reads all nodes with the same insns: 5: w0 = *(u32 *)(r1 + 0x8) 8: r1 = *(u64 *)(r1 + 0x0) The same program without .arena.data section, when the sections are array maps: libbpf: elf: skipping relo section(9) .rel.data for section(8) .data ... 16: (79) r1 = *(u64 *)(r1 +0) ; frame1: R1=scalar() ... 13: (61) r0 = *(u32 *)(r1 +8) R1 invalid mem access 'scalar' The program is built by upstream rustc for its bpfel-unknown-none target. Nothing is done to the output: rustc emits LLVM bitcode and clang makes the object of it for the cpu version of the test_progs flavor. rustc has no prebuilt core for the target, so core is built from the source that comes with rustc, like the kernel does it. That takes 5 seconds. The subtest is skipped when: - there is no rustc or it is older than 1.87. core is built as edition 2024, which it is since then. - rustc comes without the source of core (rustup component add rust-src). - clang is older than 23 or older than LLVM of rustc. It can't read the bitcode then. There are no kfuncs in the program. rustc doesn't emit debug info for extern functions, so there would be no BTF for them. panic=abort is a stop gap. Nothing in the program panics, so the panic handler is not loaded. The series from Yonghong are about to add support for panic=unwind. Tested with rustc 1.95.0-nightly (LLVM 22.1) and clang 24. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261002124714.180012-16-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 12 +++- .../testing/selftests/bpf/Makefile.buildvars | 25 +++++++ tools/testing/selftests/bpf/Makefile.skel | 15 ++++ .../selftests/bpf/prog_tests/data_in_arena.c | 25 +++++++ .../selftests/bpf/progs/data_in_arena_rust.rs | 70 +++++++++++++++++++ 5 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_rust.rs diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index afa589a27b15..a22be7efd1fa 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -422,6 +422,16 @@ $(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) endif +# #![no_std] looks for compiler_builtins too. Nothing of it is used. +ifneq ($(RUST_CORE),) +$(RUST_CORE): $(RUST_CORE_SRC) + $(call msg,RUSTC,,$@) + $(Q)mkdir -p $(@D) + +$(Q)$(RUSTC_BPF) -A warnings --edition 2024 --crate-name core --out-dir $(@D) $< + +$(Q)echo '#![feature(compiler_builtins)] #![compiler_builtins] #![no_std]' | \ + $(RUSTC_BPF) -A warnings --crate-name compiler_builtins --out-dir $(@D) - +endif + # Generated test list headers define gen_tests_hdr @@ -457,7 +467,7 @@ RUNNER_PREREQS := $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(BPFTOOL) \ $(VERIFY_SIG_HDR) $(PRIVATE_KEY) $(VERIFICATION_CERT) \ $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) \ prog_tests/tests.h map_tests/tests.h \ - $(RUNNER_OBJS) + $(RUNNER_OBJS) $(RUST_CORE) # Runtime fixtures for each test_progs flavor. RUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ diff --git a/tools/testing/selftests/bpf/Makefile.buildvars b/tools/testing/selftests/bpf/Makefile.buildvars index d2a0c0031b87..ff3476bc40a4 100644 --- a/tools/testing/selftests/bpf/Makefile.buildvars +++ b/tools/testing/selftests/bpf/Makefile.buildvars @@ -109,6 +109,31 @@ HOST_INCLUDE_DIR := $(INCLUDE_DIR) endif RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids +# Programs in Rust are built by upstream rustc. It has no prebuilt core for +# the bpf target, so it has to come with the source of core: +# rustup component add rust-src +# core is built as edition 2024, which it is since rustc 1.87. +# rustc emits LLVM bitcode and clang makes the object of it, so clang has to be +# 23 or newer and not older than LLVM of rustc. +# Otherwise RUST_CORE is empty and the tests are skipped. +RUSTC ?= rustc +RUST_CORE_SRC := $(wildcard $(shell $(RUSTC) --print sysroot 2>/dev/null)$\ + /lib/rustlib/src/rust/library/core/src/lib.rs) +ifneq ($(RUST_CORE_SRC),) +ifeq ($(shell { clang=$$(echo __clang_major__ | $(CLANG) -E -P -x c -) && \ + llvm=$$($(srctree)/scripts/rustc-llvm-version.sh $(RUSTC)) && \ + [ $$($(srctree)/scripts/rustc-version.sh $(RUSTC)) -ge 108700 ] && \ + [ $$clang -ge 23 ] && [ $$clang -ge $$((llvm / 10000)) ]; } \ + 2>/dev/null && echo y),y) +RUST_CORE := $(BUILD_DIR)/rust/libcore.rlib +endif +endif +# RUSTC_BOOTSTRAP=1 is to build core with a stable rustc, like the kernel does. +# panic=abort is a stop gap until panic=unwind is supported. +RUSTC_BPF = RUSTC_BOOTSTRAP=1 $(RUSTC) -O -C panic=abort --crate-type rlib \ + --target $(if $(IS_LITTLE_ENDIAN),bpfel,bpfeb)-unknown-none \ + -L $(dir $(RUST_CORE)) + DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool ifneq ($(CROSS_COMPILE),) CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 765666a64b89..06e297a17156 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -138,4 +138,19 @@ $(LINKED_SKELS_H): $(RDIR)/%.skel.h: $$(addprefix $(RDIR)/,$$($$*.skel.h-deps)) $(BPFTOOL) $(GEN_SKEL) | $(RDIR) $(Q)$(cmd_bpf_link_skel) +# Programs in Rust, see Makefile.buildvars. No skeletons: the objects may be absent. +ifneq ($(RUST_CORE),) +ifeq ($(BPF_CC),$(CLANG)) +RUST_OBJS := $(patsubst progs/%.rs,$(RDIR)/%.bpf.o,$(wildcard progs/*.rs)) +BPF_OBJS += $(RUST_OBJS) + +$(RUST_OBJS): $(RDIR)/%.bpf.o: progs/%.rs $(RUST_CORE) | $(RDIR) + $(call msg,RUSTC,$(BINARY),$@) + +$(Q)$(RUSTC_BPF) --edition 2021 -C debuginfo=2 --emit=llvm-bc \ + $(patsubst -mcpu=%,-C target-cpu=%,$(filter -mcpu=%,$(BPF_CC_FLAGS))) \ + -o $(dir $(RUST_CORE))$(BINARY)-$*.bc $< && \ + $(BPF_CC) $(BPF_CC_FLAGS) -c $(dir $(RUST_CORE))$(BINARY)-$*.bc -o $@ $(call skip_on_fail,BPF) +endif +endif + endif # BPF_CC diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c index 86dc6798bbde..cb1023507c01 100644 --- a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c +++ b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c @@ -167,6 +167,29 @@ out: data_in_arena__destroy(skel); } +/* The object is there if rustc and clang can build it, see Makefile.buildvars */ +static void test_rust(void) +{ + const char *file = "./data_in_arena_rust.bpf.o"; + struct bpf_object *obj; + + if (access(file, R_OK)) { + test__skip(); + return; + } + obj = bpf_object__open_file(file, NULL); + if (!ASSERT_OK_PTR(obj, "open")) + return; + if (!ASSERT_OK(bpf_object__load(obj), "load")) + goto out; + /* libbpf goes on without BTF when the kernel doesn't take it */ + ASSERT_GE(bpf_object__btf_fd(obj), 0, "btf_fd"); + ASSERT_EQ(run_prog(bpf_object__find_program_by_name(obj, "list_in_data")), + 100 + 20 + 3, "retval"); +out: + bpf_object__close(obj); +} + void test_data_in_arena(void) { #if !defined(__x86_64__) && !defined(__aarch64__) @@ -188,4 +211,6 @@ void test_data_in_arena(void) test_not_my_arena(true); if (test__start_subtest("reused_arena")) test_not_my_arena(false); + if (test__start_subtest("rust")) + test_rust(); } diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs new file mode 100644 index 000000000000..b69128487a0f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Why .data, .bss and .rodata of a program in Rust are in arena. +// +// A reference in Rust is an address. It doesn't say what it points to and it +// can be stored in data. The list below has a node in each of the sections. +// The nodes are linked by references that are in the data: +// IN_BSS.next is stored by the program, +// IN_DATA.next is a relocation in .data against .rodata. +// sum() loads the references back and reads the three nodes with the same insn. +// +// When the sections are array maps libbpf skips the relocation in .data, and +// what sum() loads from a node is a number that can't be dereferenced: +// R1 invalid mem access 'scalar' +// In arena the address of a node is a number to begin with. + +#![no_std] +#![no_main] + +// Tell libbpf to keep .data, .bss and .rodata in arena. +#[used] +#[link_section = ".arena.data"] +static DATA_IN_ARENA: u8 = 0; + +#[used] +#[link_section = "license"] +static LICENSE: [u8; 4] = *b"GPL\0"; + +// panic=abort is a stop gap until panic=unwind is supported. +// Nothing here panics, so the handler is not a part of the program. +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + loop {} +} + +pub struct Node { + val: u32, + next: Option<&'static Node>, +} + +// no_mangle makes them visible outside, so LLVM can't fold the list into a constant. +#[no_mangle] +static IN_RODATA: Node = Node { val: 3, next: None }; +#[no_mangle] +static mut IN_DATA: Node = Node { val: 20, next: Some(&IN_RODATA) }; +#[no_mangle] +static mut IN_BSS: Node = Node { val: 0, next: None }; + +#[inline(never)] +fn sum(mut node: Option<&Node>) -> u32 { + let mut sum = 0; + // The verifier wants a bound. + for _ in 0..8 { + let Some(n) = node else { break }; + sum += n.val; + node = n.next; + } + sum +} + +#[no_mangle] +#[link_section = "syscall"] +pub extern "C" fn list_in_data(_ctx: *mut u8) -> u32 { + unsafe { + let head = &mut *&raw mut IN_BSS; + head.val = 100; + head.next = Some(&*&raw const IN_DATA); + sum(Some(head)) + } +} -- knotmirror 0.0.1