Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2
3/*
4 * Copyright (C) 2023 Google LLC.
5 */
6
7#ifndef __UNROLL_H
8#define __UNROLL_H
9
10#include <linux/args.h>
11
12#ifdef CONFIG_CC_IS_CLANG
13#define __pick_unrolled(x, y) _Pragma(#x)
14#elif CONFIG_GCC_VERSION >= 80000
15#define __pick_unrolled(x, y) _Pragma(#y)
16#else
17#define __pick_unrolled(x, y) /* not supported */
18#endif
19
20/**
21 * unrolled - loop attributes to ask the compiler to unroll it
22 *
23 * Usage:
24 *
25 * #define BATCH 8
26 *
27 * unrolled_count(BATCH)
28 * for (u32 i = 0; i < BATCH; i++)
29 * // loop body without cross-iteration dependencies
30 *
31 * This is only a hint and the compiler is free to disable unrolling if it
32 * thinks the count is suboptimal and may hurt performance and/or hugely
33 * increase object code size.
34 * Not having any cross-iteration dependencies (i.e. when iter x + 1 depends
35 * on what iter x will do with variables) is not a strict requirement, but
36 * provides best performance and object code size.
37 * Available only on Clang and GCC 8.x onwards.
38 */
39
40/* Ask the compiler to pick an optimal unroll count, Clang only */
41#define unrolled \
42 __pick_unrolled(clang loop unroll(enable), /* nothing */)
43
44/* Unroll each @n iterations of the loop */
45#define unrolled_count(n) \
46 __pick_unrolled(clang loop unroll_count(n), GCC unroll n)
47
48/* Unroll the whole loop */
49#define unrolled_full \
50 __pick_unrolled(clang loop unroll(full), GCC unroll 65534)
51
52/* Never unroll the loop */
53#define unrolled_none \
54 __pick_unrolled(clang loop unroll(disable), GCC unroll 1)
55
56#define UNROLL(N, MACRO, args...) CONCATENATE(__UNROLL_, N)(MACRO, args)
57
58#define __UNROLL_0(MACRO, args...)
59#define __UNROLL_1(MACRO, args...) __UNROLL_0(MACRO, args) MACRO(0, args)
60#define __UNROLL_2(MACRO, args...) __UNROLL_1(MACRO, args) MACRO(1, args)
61#define __UNROLL_3(MACRO, args...) __UNROLL_2(MACRO, args) MACRO(2, args)
62#define __UNROLL_4(MACRO, args...) __UNROLL_3(MACRO, args) MACRO(3, args)
63#define __UNROLL_5(MACRO, args...) __UNROLL_4(MACRO, args) MACRO(4, args)
64#define __UNROLL_6(MACRO, args...) __UNROLL_5(MACRO, args) MACRO(5, args)
65#define __UNROLL_7(MACRO, args...) __UNROLL_6(MACRO, args) MACRO(6, args)
66#define __UNROLL_8(MACRO, args...) __UNROLL_7(MACRO, args) MACRO(7, args)
67#define __UNROLL_9(MACRO, args...) __UNROLL_8(MACRO, args) MACRO(8, args)
68#define __UNROLL_10(MACRO, args...) __UNROLL_9(MACRO, args) MACRO(9, args)
69#define __UNROLL_11(MACRO, args...) __UNROLL_10(MACRO, args) MACRO(10, args)
70#define __UNROLL_12(MACRO, args...) __UNROLL_11(MACRO, args) MACRO(11, args)
71#define __UNROLL_13(MACRO, args...) __UNROLL_12(MACRO, args) MACRO(12, args)
72#define __UNROLL_14(MACRO, args...) __UNROLL_13(MACRO, args) MACRO(13, args)
73#define __UNROLL_15(MACRO, args...) __UNROLL_14(MACRO, args) MACRO(14, args)
74#define __UNROLL_16(MACRO, args...) __UNROLL_15(MACRO, args) MACRO(15, args)
75#define __UNROLL_17(MACRO, args...) __UNROLL_16(MACRO, args) MACRO(16, args)
76#define __UNROLL_18(MACRO, args...) __UNROLL_17(MACRO, args) MACRO(17, args)
77#define __UNROLL_19(MACRO, args...) __UNROLL_18(MACRO, args) MACRO(18, args)
78#define __UNROLL_20(MACRO, args...) __UNROLL_19(MACRO, args) MACRO(19, args)
79
80#endif /* __UNROLL_H */