tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_core_read.h>
6#include "../../../include/linux/filter.h"
7#include "bpf_misc.h"
8#include <stdbool.h>
9#include "bpf_kfuncs.h"
10
11SEC("raw_tp")
12__arch_x86_64
13__log_level(4) __msg("stack depth 8")
14__xlated("4: r5 = 5")
15__xlated("5: w0 = ")
16__xlated("6: r0 = &(void __percpu *)(r0)")
17__xlated("7: r0 = *(u32 *)(r0 +0)")
18__xlated("8: exit")
19__success
20__naked void simple(void)
21{
22 asm volatile (
23 "r1 = 1;"
24 "r2 = 2;"
25 "r3 = 3;"
26 "r4 = 4;"
27 "r5 = 5;"
28 "*(u64 *)(r10 - 16) = r1;"
29 "*(u64 *)(r10 - 24) = r2;"
30 "*(u64 *)(r10 - 32) = r3;"
31 "*(u64 *)(r10 - 40) = r4;"
32 "*(u64 *)(r10 - 48) = r5;"
33 "call %[bpf_get_smp_processor_id];"
34 "r5 = *(u64 *)(r10 - 48);"
35 "r4 = *(u64 *)(r10 - 40);"
36 "r3 = *(u64 *)(r10 - 32);"
37 "r2 = *(u64 *)(r10 - 24);"
38 "r1 = *(u64 *)(r10 - 16);"
39 "exit;"
40 :
41 : __imm(bpf_get_smp_processor_id)
42 : __clobber_all);
43}
44
45/* The logic for detecting and verifying bpf_fastcall pattern is the same for
46 * any arch, however x86 differs from arm64 or riscv64 in a way
47 * bpf_get_smp_processor_id is rewritten:
48 * - on x86 it is done by verifier
49 * - on arm64 and riscv64 it is done by jit
50 *
51 * Which leads to different xlated patterns for different archs:
52 * - on x86 the call is expanded as 3 instructions
53 * - on arm64 and riscv64 the call remains as is
54 * (but spills/fills are still removed)
55 *
56 * It is really desirable to check instruction indexes in the xlated
57 * patterns, so add this canary test to check that function rewrite by
58 * jit is correctly processed by bpf_fastcall logic, keep the rest of the
59 * tests as x86.
60 */
61SEC("raw_tp")
62__arch_arm64
63__arch_riscv64
64__xlated("0: r1 = 1")
65__xlated("1: call bpf_get_smp_processor_id")
66__xlated("2: exit")
67__success
68__naked void canary_arm64_riscv64(void)
69{
70 asm volatile (
71 "r1 = 1;"
72 "*(u64 *)(r10 - 16) = r1;"
73 "call %[bpf_get_smp_processor_id];"
74 "r1 = *(u64 *)(r10 - 16);"
75 "exit;"
76 :
77 : __imm(bpf_get_smp_processor_id)
78 : __clobber_all);
79}
80
81SEC("raw_tp")
82__arch_x86_64
83__xlated("1: r0 = &(void __percpu *)(r0)")
84__xlated("...")
85__xlated("3: exit")
86__success
87__naked void canary_zero_spills(void)
88{
89 asm volatile (
90 "call %[bpf_get_smp_processor_id];"
91 "exit;"
92 :
93 : __imm(bpf_get_smp_processor_id)
94 : __clobber_all);
95}
96
97SEC("raw_tp")
98__arch_x86_64
99__log_level(4) __msg("stack depth 16")
100__xlated("1: *(u64 *)(r10 -16) = r1")
101__xlated("...")
102__xlated("3: r0 = &(void __percpu *)(r0)")
103__xlated("...")
104__xlated("5: r2 = *(u64 *)(r10 -16)")
105__success
106__naked void wrong_reg_in_pattern1(void)
107{
108 asm volatile (
109 "r1 = 1;"
110 "*(u64 *)(r10 - 16) = r1;"
111 "call %[bpf_get_smp_processor_id];"
112 "r2 = *(u64 *)(r10 - 16);"
113 "exit;"
114 :
115 : __imm(bpf_get_smp_processor_id)
116 : __clobber_all);
117}
118
119SEC("raw_tp")
120__arch_x86_64
121__xlated("1: *(u64 *)(r10 -16) = r6")
122__xlated("...")
123__xlated("3: r0 = &(void __percpu *)(r0)")
124__xlated("...")
125__xlated("5: r6 = *(u64 *)(r10 -16)")
126__success
127__naked void wrong_reg_in_pattern2(void)
128{
129 asm volatile (
130 "r6 = 1;"
131 "*(u64 *)(r10 - 16) = r6;"
132 "call %[bpf_get_smp_processor_id];"
133 "r6 = *(u64 *)(r10 - 16);"
134 "exit;"
135 :
136 : __imm(bpf_get_smp_processor_id)
137 : __clobber_all);
138}
139
140SEC("raw_tp")
141__arch_x86_64
142__xlated("1: *(u64 *)(r10 -16) = r0")
143__xlated("...")
144__xlated("3: r0 = &(void __percpu *)(r0)")
145__xlated("...")
146__xlated("5: r0 = *(u64 *)(r10 -16)")
147__success
148__naked void wrong_reg_in_pattern3(void)
149{
150 asm volatile (
151 "r0 = 1;"
152 "*(u64 *)(r10 - 16) = r0;"
153 "call %[bpf_get_smp_processor_id];"
154 "r0 = *(u64 *)(r10 - 16);"
155 "exit;"
156 :
157 : __imm(bpf_get_smp_processor_id)
158 : __clobber_all);
159}
160
161SEC("raw_tp")
162__arch_x86_64
163__xlated("2: *(u64 *)(r2 -16) = r1")
164__xlated("...")
165__xlated("4: r0 = &(void __percpu *)(r0)")
166__xlated("...")
167__xlated("6: r1 = *(u64 *)(r10 -16)")
168__success
169__naked void wrong_base_in_pattern(void)
170{
171 asm volatile (
172 "r1 = 1;"
173 "r2 = r10;"
174 "*(u64 *)(r2 - 16) = r1;"
175 "call %[bpf_get_smp_processor_id];"
176 "r1 = *(u64 *)(r10 - 16);"
177 "exit;"
178 :
179 : __imm(bpf_get_smp_processor_id)
180 : __clobber_all);
181}
182
183SEC("raw_tp")
184__arch_x86_64
185__xlated("1: *(u64 *)(r10 -16) = r1")
186__xlated("...")
187__xlated("3: r0 = &(void __percpu *)(r0)")
188__xlated("...")
189__xlated("5: r2 = 1")
190__success
191__naked void wrong_insn_in_pattern(void)
192{
193 asm volatile (
194 "r1 = 1;"
195 "*(u64 *)(r10 - 16) = r1;"
196 "call %[bpf_get_smp_processor_id];"
197 "r2 = 1;"
198 "r1 = *(u64 *)(r10 - 16);"
199 "exit;"
200 :
201 : __imm(bpf_get_smp_processor_id)
202 : __clobber_all);
203}
204
205SEC("raw_tp")
206__arch_x86_64
207__xlated("2: *(u64 *)(r10 -16) = r1")
208__xlated("...")
209__xlated("4: r0 = &(void __percpu *)(r0)")
210__xlated("...")
211__xlated("6: r1 = *(u64 *)(r10 -8)")
212__success
213__naked void wrong_off_in_pattern1(void)
214{
215 asm volatile (
216 "r1 = 1;"
217 "*(u64 *)(r10 - 8) = r1;"
218 "*(u64 *)(r10 - 16) = r1;"
219 "call %[bpf_get_smp_processor_id];"
220 "r1 = *(u64 *)(r10 - 8);"
221 "exit;"
222 :
223 : __imm(bpf_get_smp_processor_id)
224 : __clobber_all);
225}
226
227SEC("raw_tp")
228__arch_x86_64
229__xlated("1: *(u32 *)(r10 -4) = r1")
230__xlated("...")
231__xlated("3: r0 = &(void __percpu *)(r0)")
232__xlated("...")
233__xlated("5: r1 = *(u32 *)(r10 -4)")
234__success
235__naked void wrong_off_in_pattern2(void)
236{
237 asm volatile (
238 "r1 = 1;"
239 "*(u32 *)(r10 - 4) = r1;"
240 "call %[bpf_get_smp_processor_id];"
241 "r1 = *(u32 *)(r10 - 4);"
242 "exit;"
243 :
244 : __imm(bpf_get_smp_processor_id)
245 : __clobber_all);
246}
247
248SEC("raw_tp")
249__arch_x86_64
250__xlated("1: *(u32 *)(r10 -16) = r1")
251__xlated("...")
252__xlated("3: r0 = &(void __percpu *)(r0)")
253__xlated("...")
254__xlated("5: r1 = *(u32 *)(r10 -16)")
255__success
256__naked void wrong_size_in_pattern(void)
257{
258 asm volatile (
259 "r1 = 1;"
260 "*(u32 *)(r10 - 16) = r1;"
261 "call %[bpf_get_smp_processor_id];"
262 "r1 = *(u32 *)(r10 - 16);"
263 "exit;"
264 :
265 : __imm(bpf_get_smp_processor_id)
266 : __clobber_all);
267}
268
269SEC("raw_tp")
270__arch_x86_64
271__xlated("2: *(u32 *)(r10 -8) = r1")
272__xlated("...")
273__xlated("4: r0 = &(void __percpu *)(r0)")
274__xlated("...")
275__xlated("6: r1 = *(u32 *)(r10 -8)")
276__success
277__naked void partial_pattern(void)
278{
279 asm volatile (
280 "r1 = 1;"
281 "r2 = 2;"
282 "*(u32 *)(r10 - 8) = r1;"
283 "*(u64 *)(r10 - 16) = r2;"
284 "call %[bpf_get_smp_processor_id];"
285 "r2 = *(u64 *)(r10 - 16);"
286 "r1 = *(u32 *)(r10 - 8);"
287 "exit;"
288 :
289 : __imm(bpf_get_smp_processor_id)
290 : __clobber_all);
291}
292
293SEC("raw_tp")
294__arch_x86_64
295__xlated("0: r1 = 1")
296__xlated("1: r2 = 2")
297/* not patched, spills for -8, -16 not removed */
298__xlated("2: *(u64 *)(r10 -8) = r1")
299__xlated("3: *(u64 *)(r10 -16) = r2")
300__xlated("...")
301__xlated("5: r0 = &(void __percpu *)(r0)")
302__xlated("...")
303__xlated("7: r2 = *(u64 *)(r10 -16)")
304__xlated("8: r1 = *(u64 *)(r10 -8)")
305/* patched, spills for -24, -32 removed */
306__xlated("...")
307__xlated("10: r0 = &(void __percpu *)(r0)")
308__xlated("...")
309__xlated("12: exit")
310__success
311__naked void min_stack_offset(void)
312{
313 asm volatile (
314 "r1 = 1;"
315 "r2 = 2;"
316 /* this call won't be patched */
317 "*(u64 *)(r10 - 8) = r1;"
318 "*(u64 *)(r10 - 16) = r2;"
319 "call %[bpf_get_smp_processor_id];"
320 "r2 = *(u64 *)(r10 - 16);"
321 "r1 = *(u64 *)(r10 - 8);"
322 /* this call would be patched */
323 "*(u64 *)(r10 - 24) = r1;"
324 "*(u64 *)(r10 - 32) = r2;"
325 "call %[bpf_get_smp_processor_id];"
326 "r2 = *(u64 *)(r10 - 32);"
327 "r1 = *(u64 *)(r10 - 24);"
328 "exit;"
329 :
330 : __imm(bpf_get_smp_processor_id)
331 : __clobber_all);
332}
333
334SEC("raw_tp")
335__arch_x86_64
336__xlated("1: *(u64 *)(r10 -8) = r1")
337__xlated("...")
338__xlated("3: r0 = &(void __percpu *)(r0)")
339__xlated("...")
340__xlated("5: r1 = *(u64 *)(r10 -8)")
341__success
342__naked void bad_fixed_read(void)
343{
344 asm volatile (
345 "r1 = 1;"
346 "*(u64 *)(r10 - 8) = r1;"
347 "call %[bpf_get_smp_processor_id];"
348 "r1 = *(u64 *)(r10 - 8);"
349 "r1 = r10;"
350 "r1 += -8;"
351 "r1 = *(u64 *)(r1 - 0);"
352 "exit;"
353 :
354 : __imm(bpf_get_smp_processor_id)
355 : __clobber_all);
356}
357
358SEC("raw_tp")
359__arch_x86_64
360__xlated("1: *(u64 *)(r10 -8) = r1")
361__xlated("...")
362__xlated("3: r0 = &(void __percpu *)(r0)")
363__xlated("...")
364__xlated("5: r1 = *(u64 *)(r10 -8)")
365__success
366__naked void bad_fixed_write(void)
367{
368 asm volatile (
369 "r1 = 1;"
370 "*(u64 *)(r10 - 8) = r1;"
371 "call %[bpf_get_smp_processor_id];"
372 "r1 = *(u64 *)(r10 - 8);"
373 "r1 = r10;"
374 "r1 += -8;"
375 "*(u64 *)(r1 - 0) = r1;"
376 "exit;"
377 :
378 : __imm(bpf_get_smp_processor_id)
379 : __clobber_all);
380}
381
382SEC("raw_tp")
383__arch_x86_64
384__xlated("6: *(u64 *)(r10 -16) = r1")
385__xlated("...")
386__xlated("8: r0 = &(void __percpu *)(r0)")
387__xlated("...")
388__xlated("10: r1 = *(u64 *)(r10 -16)")
389__success
390__naked void bad_varying_read(void)
391{
392 asm volatile (
393 "r6 = *(u64 *)(r1 + 0);" /* random scalar value */
394 "r6 &= 0x7;" /* r6 range [0..7] */
395 "r6 += 0x2;" /* r6 range [2..9] */
396 "r7 = 0;"
397 "r7 -= r6;" /* r7 range [-9..-2] */
398 "r1 = 1;"
399 "*(u64 *)(r10 - 16) = r1;"
400 "call %[bpf_get_smp_processor_id];"
401 "r1 = *(u64 *)(r10 - 16);"
402 "r1 = r10;"
403 "r1 += r7;"
404 "r1 = *(u8 *)(r1 - 0);" /* touches slot [-16..-9] where spills are stored */
405 "exit;"
406 :
407 : __imm(bpf_get_smp_processor_id)
408 : __clobber_all);
409}
410
411SEC("raw_tp")
412__arch_x86_64
413__xlated("6: *(u64 *)(r10 -16) = r1")
414__xlated("...")
415__xlated("8: r0 = &(void __percpu *)(r0)")
416__xlated("...")
417__xlated("10: r1 = *(u64 *)(r10 -16)")
418__success
419__naked void bad_varying_write(void)
420{
421 asm volatile (
422 "r6 = *(u64 *)(r1 + 0);" /* random scalar value */
423 "r6 &= 0x7;" /* r6 range [0..7] */
424 "r6 += 0x2;" /* r6 range [2..9] */
425 "r7 = 0;"
426 "r7 -= r6;" /* r7 range [-9..-2] */
427 "r1 = 1;"
428 "*(u64 *)(r10 - 16) = r1;"
429 "call %[bpf_get_smp_processor_id];"
430 "r1 = *(u64 *)(r10 - 16);"
431 "r1 = r10;"
432 "r1 += r7;"
433 "*(u8 *)(r1 - 0) = r7;" /* touches slot [-16..-9] where spills are stored */
434 "exit;"
435 :
436 : __imm(bpf_get_smp_processor_id)
437 : __clobber_all);
438}
439
440SEC("raw_tp")
441__arch_x86_64
442__xlated("1: *(u64 *)(r10 -8) = r1")
443__xlated("...")
444__xlated("3: r0 = &(void __percpu *)(r0)")
445__xlated("...")
446__xlated("5: r1 = *(u64 *)(r10 -8)")
447__success
448__naked void bad_write_in_subprog(void)
449{
450 asm volatile (
451 "r1 = 1;"
452 "*(u64 *)(r10 - 8) = r1;"
453 "call %[bpf_get_smp_processor_id];"
454 "r1 = *(u64 *)(r10 - 8);"
455 "r1 = r10;"
456 "r1 += -8;"
457 "call bad_write_in_subprog_aux;"
458 "exit;"
459 :
460 : __imm(bpf_get_smp_processor_id)
461 : __clobber_all);
462}
463
464__used
465__naked static void bad_write_in_subprog_aux(void)
466{
467 asm volatile (
468 "r0 = 1;"
469 "*(u64 *)(r1 - 0) = r0;" /* invalidates bpf_fastcall contract for caller: */
470 "exit;" /* caller stack at -8 used outside of the pattern */
471 ::: __clobber_all);
472}
473
474SEC("raw_tp")
475__arch_x86_64
476__xlated("1: *(u64 *)(r10 -8) = r1")
477__xlated("...")
478__xlated("3: r0 = &(void __percpu *)(r0)")
479__xlated("...")
480__xlated("5: r1 = *(u64 *)(r10 -8)")
481__success
482__naked void bad_helper_write(void)
483{
484 asm volatile (
485 "r1 = 1;"
486 /* bpf_fastcall pattern with stack offset -8 */
487 "*(u64 *)(r10 - 8) = r1;"
488 "call %[bpf_get_smp_processor_id];"
489 "r1 = *(u64 *)(r10 - 8);"
490 "r1 = r10;"
491 "r1 += -8;"
492 "r2 = 1;"
493 "r3 = 42;"
494 /* read dst is fp[-8], thus bpf_fastcall rewrite not applied */
495 "call %[bpf_probe_read_kernel];"
496 "exit;"
497 :
498 : __imm(bpf_get_smp_processor_id),
499 __imm(bpf_probe_read_kernel)
500 : __clobber_all);
501}
502
503SEC("raw_tp")
504__arch_x86_64
505/* main, not patched */
506__xlated("1: *(u64 *)(r10 -8) = r1")
507__xlated("...")
508__xlated("3: r0 = &(void __percpu *)(r0)")
509__xlated("...")
510__xlated("5: r1 = *(u64 *)(r10 -8)")
511__xlated("...")
512__xlated("9: call pc+1")
513__xlated("...")
514__xlated("10: exit")
515/* subprogram, patched */
516__xlated("11: r1 = 1")
517__xlated("...")
518__xlated("13: r0 = &(void __percpu *)(r0)")
519__xlated("...")
520__xlated("15: exit")
521__success
522__naked void invalidate_one_subprog(void)
523{
524 asm volatile (
525 "r1 = 1;"
526 "*(u64 *)(r10 - 8) = r1;"
527 "call %[bpf_get_smp_processor_id];"
528 "r1 = *(u64 *)(r10 - 8);"
529 "r1 = r10;"
530 "r1 += -8;"
531 "r1 = *(u64 *)(r1 - 0);"
532 "call invalidate_one_subprog_aux;"
533 "exit;"
534 :
535 : __imm(bpf_get_smp_processor_id)
536 : __clobber_all);
537}
538
539__used
540__naked static void invalidate_one_subprog_aux(void)
541{
542 asm volatile (
543 "r1 = 1;"
544 "*(u64 *)(r10 - 8) = r1;"
545 "call %[bpf_get_smp_processor_id];"
546 "r1 = *(u64 *)(r10 - 8);"
547 "exit;"
548 :
549 : __imm(bpf_get_smp_processor_id)
550 : __clobber_all);
551}
552
553SEC("raw_tp")
554__arch_x86_64
555/* main */
556__xlated("0: r1 = 1")
557__xlated("...")
558__xlated("2: r0 = &(void __percpu *)(r0)")
559__xlated("...")
560__xlated("4: call pc+1")
561__xlated("5: exit")
562/* subprogram */
563__xlated("6: r1 = 1")
564__xlated("...")
565__xlated("8: r0 = &(void __percpu *)(r0)")
566__xlated("...")
567__xlated("10: *(u64 *)(r10 -16) = r1")
568__xlated("11: exit")
569__success
570__naked void subprogs_use_independent_offsets(void)
571{
572 asm volatile (
573 "r1 = 1;"
574 "*(u64 *)(r10 - 16) = r1;"
575 "call %[bpf_get_smp_processor_id];"
576 "r1 = *(u64 *)(r10 - 16);"
577 "call subprogs_use_independent_offsets_aux;"
578 "exit;"
579 :
580 : __imm(bpf_get_smp_processor_id)
581 : __clobber_all);
582}
583
584__used
585__naked static void subprogs_use_independent_offsets_aux(void)
586{
587 asm volatile (
588 "r1 = 1;"
589 "*(u64 *)(r10 - 24) = r1;"
590 "call %[bpf_get_smp_processor_id];"
591 "r1 = *(u64 *)(r10 - 24);"
592 "*(u64 *)(r10 - 16) = r1;"
593 "exit;"
594 :
595 : __imm(bpf_get_smp_processor_id)
596 : __clobber_all);
597}
598
599SEC("raw_tp")
600__arch_x86_64
601__log_level(4) __msg("stack depth 8")
602__xlated("2: r0 = &(void __percpu *)(r0)")
603__success
604__naked void helper_call_does_not_prevent_bpf_fastcall(void)
605{
606 asm volatile (
607 "r1 = 1;"
608 "*(u64 *)(r10 - 8) = r1;"
609 "call %[bpf_get_smp_processor_id];"
610 "r1 = *(u64 *)(r10 - 8);"
611 "*(u64 *)(r10 - 8) = r1;"
612 "call %[bpf_get_prandom_u32];"
613 "r1 = *(u64 *)(r10 - 8);"
614 "exit;"
615 :
616 : __imm(bpf_get_smp_processor_id),
617 __imm(bpf_get_prandom_u32)
618 : __clobber_all);
619}
620
621SEC("raw_tp")
622__arch_x86_64
623__log_level(4) __msg("stack depth 16")
624/* may_goto counter at -16 */
625__xlated("0: *(u64 *)(r10 -16) =")
626__xlated("1: r1 = 1")
627__xlated("...")
628__xlated("3: r0 = &(void __percpu *)(r0)")
629__xlated("...")
630/* may_goto expansion starts */
631__xlated("5: r11 = *(u64 *)(r10 -16)")
632__xlated("6: if r11 == 0x0 goto pc+3")
633__xlated("7: r11 -= 1")
634__xlated("8: *(u64 *)(r10 -16) = r11")
635/* may_goto expansion ends */
636__xlated("9: *(u64 *)(r10 -8) = r1")
637__xlated("10: exit")
638__success
639__naked void may_goto_interaction(void)
640{
641 asm volatile (
642 "r1 = 1;"
643 "*(u64 *)(r10 - 16) = r1;"
644 "call %[bpf_get_smp_processor_id];"
645 "r1 = *(u64 *)(r10 - 16);"
646 ".8byte %[may_goto];"
647 /* just touch some stack at -8 */
648 "*(u64 *)(r10 - 8) = r1;"
649 "exit;"
650 :
651 : __imm(bpf_get_smp_processor_id),
652 __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, +1 /* offset */, 0))
653 : __clobber_all);
654}
655
656__used
657__naked static void dummy_loop_callback(void)
658{
659 asm volatile (
660 "r0 = 0;"
661 "exit;"
662 ::: __clobber_all);
663}
664
665SEC("raw_tp")
666__arch_x86_64
667__log_level(4) __msg("stack depth 32+0")
668__xlated("2: r1 = 1")
669__xlated("3: w0 =")
670__xlated("4: r0 = &(void __percpu *)(r0)")
671__xlated("5: r0 = *(u32 *)(r0 +0)")
672/* bpf_loop params setup */
673__xlated("6: r2 =")
674__xlated("7: r3 = 0")
675__xlated("8: r4 = 0")
676__xlated("...")
677/* ... part of the inlined bpf_loop */
678__xlated("12: *(u64 *)(r10 -32) = r6")
679__xlated("13: *(u64 *)(r10 -24) = r7")
680__xlated("14: *(u64 *)(r10 -16) = r8")
681__xlated("...")
682__xlated("21: call pc+8") /* dummy_loop_callback */
683/* ... last insns of the bpf_loop_interaction1 */
684__xlated("...")
685__xlated("28: r0 = 0")
686__xlated("29: exit")
687/* dummy_loop_callback */
688__xlated("30: r0 = 0")
689__xlated("31: exit")
690__success
691__naked int bpf_loop_interaction1(void)
692{
693 asm volatile (
694 "r1 = 1;"
695 /* bpf_fastcall stack region at -16, but could be removed */
696 "*(u64 *)(r10 - 16) = r1;"
697 "call %[bpf_get_smp_processor_id];"
698 "r1 = *(u64 *)(r10 - 16);"
699 "r2 = %[dummy_loop_callback];"
700 "r3 = 0;"
701 "r4 = 0;"
702 "call %[bpf_loop];"
703 "r0 = 0;"
704 "exit;"
705 :
706 : __imm_ptr(dummy_loop_callback),
707 __imm(bpf_get_smp_processor_id),
708 __imm(bpf_loop)
709 : __clobber_common
710 );
711}
712
713SEC("raw_tp")
714__arch_x86_64
715__log_level(4) __msg("stack depth 40+0")
716/* call bpf_get_smp_processor_id */
717__xlated("2: r1 = 42")
718__xlated("3: w0 =")
719__xlated("4: r0 = &(void __percpu *)(r0)")
720__xlated("5: r0 = *(u32 *)(r0 +0)")
721/* call bpf_get_prandom_u32 */
722__xlated("6: *(u64 *)(r10 -16) = r1")
723__xlated("7: call")
724__xlated("8: r1 = *(u64 *)(r10 -16)")
725__xlated("...")
726/* ... part of the inlined bpf_loop */
727__xlated("15: *(u64 *)(r10 -40) = r6")
728__xlated("16: *(u64 *)(r10 -32) = r7")
729__xlated("17: *(u64 *)(r10 -24) = r8")
730__success
731__naked int bpf_loop_interaction2(void)
732{
733 asm volatile (
734 "r1 = 42;"
735 /* bpf_fastcall stack region at -16, cannot be removed */
736 "*(u64 *)(r10 - 16) = r1;"
737 "call %[bpf_get_smp_processor_id];"
738 "r1 = *(u64 *)(r10 - 16);"
739 "*(u64 *)(r10 - 16) = r1;"
740 "call %[bpf_get_prandom_u32];"
741 "r1 = *(u64 *)(r10 - 16);"
742 "r2 = %[dummy_loop_callback];"
743 "r3 = 0;"
744 "r4 = 0;"
745 "call %[bpf_loop];"
746 "r0 = 0;"
747 "exit;"
748 :
749 : __imm_ptr(dummy_loop_callback),
750 __imm(bpf_get_smp_processor_id),
751 __imm(bpf_get_prandom_u32),
752 __imm(bpf_loop)
753 : __clobber_common
754 );
755}
756
757SEC("raw_tp")
758__arch_x86_64
759__log_level(4)
760__msg("stack depth 512+0")
761/* just to print xlated version when debugging */
762__xlated("r0 = &(void __percpu *)(r0)")
763__success
764/* cumulative_stack_depth() stack usage is MAX_BPF_STACK,
765 * called subprogram uses an additional slot for bpf_fastcall spill/fill,
766 * since bpf_fastcall spill/fill could be removed the program still fits
767 * in MAX_BPF_STACK and should be accepted.
768 */
769__naked int cumulative_stack_depth(void)
770{
771 asm volatile(
772 "r1 = 42;"
773 "*(u64 *)(r10 - %[max_bpf_stack]) = r1;"
774 "call cumulative_stack_depth_subprog;"
775 "exit;"
776 :
777 : __imm_const(max_bpf_stack, MAX_BPF_STACK)
778 : __clobber_all
779 );
780}
781
782__used
783__naked static void cumulative_stack_depth_subprog(void)
784{
785 asm volatile (
786 "*(u64 *)(r10 - 8) = r1;"
787 "call %[bpf_get_smp_processor_id];"
788 "r1 = *(u64 *)(r10 - 8);"
789 "exit;"
790 :: __imm(bpf_get_smp_processor_id) : __clobber_all);
791}
792
793SEC("raw_tp")
794__arch_x86_64
795__log_level(4)
796__msg("stack depth 512")
797__xlated("0: r1 = 42")
798__xlated("1: *(u64 *)(r10 -512) = r1")
799__xlated("2: w0 = ")
800__xlated("3: r0 = &(void __percpu *)(r0)")
801__xlated("4: r0 = *(u32 *)(r0 +0)")
802__xlated("5: exit")
803__success
804__naked int bpf_fastcall_max_stack_ok(void)
805{
806 asm volatile(
807 "r1 = 42;"
808 "*(u64 *)(r10 - %[max_bpf_stack]) = r1;"
809 "*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;"
810 "call %[bpf_get_smp_processor_id];"
811 "r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);"
812 "exit;"
813 :
814 : __imm_const(max_bpf_stack, MAX_BPF_STACK),
815 __imm_const(max_bpf_stack_8, MAX_BPF_STACK + 8),
816 __imm(bpf_get_smp_processor_id)
817 : __clobber_all
818 );
819}
820
821SEC("raw_tp")
822__arch_x86_64
823__log_level(4)
824__msg("stack depth 520")
825__failure
826__naked int bpf_fastcall_max_stack_fail(void)
827{
828 asm volatile(
829 "r1 = 42;"
830 "*(u64 *)(r10 - %[max_bpf_stack]) = r1;"
831 "*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;"
832 "call %[bpf_get_smp_processor_id];"
833 "r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);"
834 /* call to prandom blocks bpf_fastcall rewrite */
835 "*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;"
836 "call %[bpf_get_prandom_u32];"
837 "r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);"
838 "exit;"
839 :
840 : __imm_const(max_bpf_stack, MAX_BPF_STACK),
841 __imm_const(max_bpf_stack_8, MAX_BPF_STACK + 8),
842 __imm(bpf_get_smp_processor_id),
843 __imm(bpf_get_prandom_u32)
844 : __clobber_all
845 );
846}
847
848SEC("cgroup/getsockname_unix")
849__xlated("0: r2 = 1")
850/* bpf_cast_to_kern_ctx is replaced by a single assignment */
851__xlated("1: r0 = r1")
852__xlated("2: r0 = r2")
853__xlated("3: exit")
854__success
855__naked void kfunc_bpf_cast_to_kern_ctx(void)
856{
857 asm volatile (
858 "r2 = 1;"
859 "*(u64 *)(r10 - 32) = r2;"
860 "call %[bpf_cast_to_kern_ctx];"
861 "r2 = *(u64 *)(r10 - 32);"
862 "r0 = r2;"
863 "exit;"
864 :
865 : __imm(bpf_cast_to_kern_ctx)
866 : __clobber_all);
867}
868
869SEC("raw_tp")
870__xlated("3: r3 = 1")
871/* bpf_rdonly_cast is replaced by a single assignment */
872__xlated("4: r0 = r1")
873__xlated("5: r0 = r3")
874void kfunc_bpf_rdonly_cast(void)
875{
876 asm volatile (
877 "r2 = %[btf_id];"
878 "r3 = 1;"
879 "*(u64 *)(r10 - 32) = r3;"
880 "call %[bpf_rdonly_cast];"
881 "r3 = *(u64 *)(r10 - 32);"
882 "r0 = r3;"
883 :
884 : __imm(bpf_rdonly_cast),
885 [btf_id]"r"(bpf_core_type_id_kernel(union bpf_attr))
886 : __clobber_common);
887}
888
889/* BTF FUNC records are not generated for kfuncs referenced
890 * from inline assembly. These records are necessary for
891 * libbpf to link the program. The function below is a hack
892 * to ensure that BTF FUNC records are generated.
893 */
894void kfunc_root(void)
895{
896 bpf_cast_to_kern_ctx(0);
897 bpf_rdonly_cast(0, 0);
898}
899
900char _license[] SEC("license") = "GPL";