Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

bpf, arm64: Inline bpf_get_current_task/_btf() helpers

On ARM64, the pointer to task_struct is always available in the sp_el0
register and therefore the calls to bpf_get_current_task() and
bpf_get_current_task_btf() can be inlined into a single MRS instruction.

Here is the difference before and after this change:

Before:

; struct task_struct *task = bpf_get_current_task_btf();
54: mov x10, #0xffffffffffff7978 // #-34440
58: movk x10, #0x802b, lsl #16
5c: movk x10, #0x8000, lsl #32
60: blr x10 --------------> 0xffff8000802b7978 <+0>: mrs x0, sp_el0
64: add x7, x0, #0x0 <-------------- 0xffff8000802b797c <+4>: ret

After:

; struct task_struct *task = bpf_get_current_task_btf();
54: mrs x7, sp_el0

This shows around 1% performance improvement in artificial microbenchmark.

Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Xu Kuohai <xukuohai@huawei.com>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20240619131334.4297-1-puranjay@kernel.org

authored by

Puranjay Mohan and committed by
Andrii Nakryiko
2bb138cb 2807db78

+9
+9
arch/arm64/net/bpf_jit_comp.c
··· 1244 1244 break; 1245 1245 } 1246 1246 1247 + /* Implement helper call to bpf_get_current_task/_btf() inline */ 1248 + if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task || 1249 + insn->imm == BPF_FUNC_get_current_task_btf)) { 1250 + emit(A64_MRS_SP_EL0(r0), ctx); 1251 + break; 1252 + } 1253 + 1247 1254 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1248 1255 &func_addr, &func_addr_fixed); 1249 1256 if (ret < 0) ··· 2587 2580 { 2588 2581 switch (imm) { 2589 2582 case BPF_FUNC_get_smp_processor_id: 2583 + case BPF_FUNC_get_current_task: 2584 + case BPF_FUNC_get_current_task_btf: 2590 2585 return true; 2591 2586 default: 2592 2587 return false;