tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

work around gcc bugs with 'asm goto' with outputs

We've had issues with gcc and 'asm goto' before, and we created a
'asm_volatile_goto()' macro for that in the past: see commits
3f0116c3238a ("compiler/gcc4: Add quirk for 'asm goto' miscompilation
bug") and a9f180345f53 ("compiler/gcc4: Make quirk for
asm_volatile_goto() unconditional").

Then, much later, we ended up removing the workaround in commit
43c249ea0b1e ("compiler-gcc.h: remove ancient workaround for gcc PR
58670") because we no longer supported building the kernel with the
affected gcc versions, but we left the macro uses around.

Now, Sean Christopherson reports a new version of a very similar
problem, which is fixed by re-applying that ancient workaround. But the
problem in question is limited to only the 'asm goto with outputs'
cases, so instead of re-introducing the old workaround as-is, let's
rename and limit the workaround to just that much less common case.

It looks like there are at least two separate issues that all hit in
this area:

(a) some versions of gcc don't mark the asm goto as 'volatile' when it
has outputs:

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98619
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110420

which is easy to work around by just adding the 'volatile' by hand.

(b) Internal compiler errors:

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110422

which are worked around by adding the extra empty 'asm' as a
barrier, as in the original workaround.

but the problem Sean sees may be a third thing since it involves bad
code generation (not an ICE) even with the manually added 'volatile'.

but the same old workaround works for this case, even if this feels a
bit like voodoo programming and may only be hiding the issue.

Reported-and-tested-by: Sean Christopherson <seanjc@google.com>
Link: https://lore.kernel.org/all/20240208220604.140859-1-seanjc@google.com/
Cc: Nick Desaulniers <ndesaulniers@google.com>
Cc: Uros Bizjak <ubizjak@gmail.com>
Cc: Jakub Jelinek <jakub@redhat.com>
Cc: Andrew Pinski <quic_apinski@quicinc.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Linus Torvalds 4356e9f8 9ed18b0b

+96 -77
+2 -2
arch/arc/include/asm/jump_label.h
··· 31 31 static __always_inline bool arch_static_branch(struct static_key *key, 32 32 bool branch) 33 33 { 34 - asm_volatile_goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)" \n" 34 + asm goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)" \n" 35 35 "1: \n" 36 36 "nop \n" 37 37 ".pushsection __jump_table, \"aw\" \n" ··· 47 47 static __always_inline bool arch_static_branch_jump(struct static_key *key, 48 48 bool branch) 49 49 { 50 - asm_volatile_goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)" \n" 50 + asm goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)" \n" 51 51 "1: \n" 52 52 "b %l[l_yes] \n" 53 53 ".pushsection __jump_table, \"aw\" \n"
+2 -2
arch/arm/include/asm/jump_label.h
··· 11 11 12 12 static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 13 13 { 14 - asm_volatile_goto("1:\n\t" 14 + asm goto("1:\n\t" 15 15 WASM(nop) "\n\t" 16 16 ".pushsection __jump_table, \"aw\"\n\t" 17 17 ".word 1b, %l[l_yes], %c0\n\t" ··· 25 25 26 26 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 27 27 { 28 - asm_volatile_goto("1:\n\t" 28 + asm goto("1:\n\t" 29 29 WASM(b) " %l[l_yes]\n\t" 30 30 ".pushsection __jump_table, \"aw\"\n\t" 31 31 ".word 1b, %l[l_yes], %c0\n\t"
+2 -2
arch/arm64/include/asm/alternative-macros.h
··· 229 229 if (!cpucap_is_possible(cpucap)) 230 230 return false; 231 231 232 - asm_volatile_goto( 232 + asm goto( 233 233 ALTERNATIVE_CB("b %l[l_no]", %[cpucap], alt_cb_patch_nops) 234 234 : 235 235 : [cpucap] "i" (cpucap) ··· 247 247 if (!cpucap_is_possible(cpucap)) 248 248 return false; 249 249 250 - asm_volatile_goto( 250 + asm goto( 251 251 ALTERNATIVE("nop", "b %l[l_yes]", %[cpucap]) 252 252 : 253 253 : [cpucap] "i" (cpucap)
+2 -2
arch/arm64/include/asm/jump_label.h
··· 18 18 static __always_inline bool arch_static_branch(struct static_key * const key, 19 19 const bool branch) 20 20 { 21 - asm_volatile_goto( 21 + asm goto( 22 22 "1: nop \n\t" 23 23 " .pushsection __jump_table, \"aw\" \n\t" 24 24 " .align 3 \n\t" ··· 35 35 static __always_inline bool arch_static_branch_jump(struct static_key * const key, 36 36 const bool branch) 37 37 { 38 - asm_volatile_goto( 38 + asm goto( 39 39 "1: b %l[l_yes] \n\t" 40 40 " .pushsection __jump_table, \"aw\" \n\t" 41 41 " .align 3 \n\t"
+2 -2
arch/csky/include/asm/jump_label.h
··· 12 12 static __always_inline bool arch_static_branch(struct static_key *key, 13 13 bool branch) 14 14 { 15 - asm_volatile_goto( 15 + asm goto( 16 16 "1: nop32 \n" 17 17 " .pushsection __jump_table, \"aw\" \n" 18 18 " .align 2 \n" ··· 29 29 static __always_inline bool arch_static_branch_jump(struct static_key *key, 30 30 bool branch) 31 31 { 32 - asm_volatile_goto( 32 + asm goto( 33 33 "1: bsr32 %l[label] \n" 34 34 " .pushsection __jump_table, \"aw\" \n" 35 35 " .align 2 \n"
+2 -2
arch/loongarch/include/asm/jump_label.h
··· 22 22 23 23 static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) 24 24 { 25 - asm_volatile_goto( 25 + asm goto( 26 26 "1: nop \n\t" 27 27 JUMP_TABLE_ENTRY 28 28 : : "i"(&((char *)key)[branch]) : : l_yes); ··· 35 35 36 36 static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) 37 37 { 38 - asm_volatile_goto( 38 + asm goto( 39 39 "1: b %l[l_yes] \n\t" 40 40 JUMP_TABLE_ENTRY 41 41 : : "i"(&((char *)key)[branch]) : : l_yes);
+2 -2
arch/mips/include/asm/jump_label.h
··· 39 39 40 40 static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 41 41 { 42 - asm_volatile_goto("1:\t" B_INSN " 2f\n\t" 42 + asm goto("1:\t" B_INSN " 2f\n\t" 43 43 "2:\t.insn\n\t" 44 44 ".pushsection __jump_table, \"aw\"\n\t" 45 45 WORD_INSN " 1b, %l[l_yes], %0\n\t" ··· 53 53 54 54 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 55 55 { 56 - asm_volatile_goto("1:\t" J_INSN " %l[l_yes]\n\t" 56 + asm goto("1:\t" J_INSN " %l[l_yes]\n\t" 57 57 ".pushsection __jump_table, \"aw\"\n\t" 58 58 WORD_INSN " 1b, %l[l_yes], %0\n\t" 59 59 ".popsection\n\t"
+2 -2
arch/parisc/include/asm/jump_label.h
··· 12 12 13 13 static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 14 14 { 15 - asm_volatile_goto("1:\n\t" 15 + asm goto("1:\n\t" 16 16 "nop\n\t" 17 17 ".pushsection __jump_table, \"aw\"\n\t" 18 18 ".align %1\n\t" ··· 29 29 30 30 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 31 31 { 32 - asm_volatile_goto("1:\n\t" 32 + asm goto("1:\n\t" 33 33 "b,n %l[l_yes]\n\t" 34 34 ".pushsection __jump_table, \"aw\"\n\t" 35 35 ".align %1\n\t"
+2 -2
arch/powerpc/include/asm/jump_label.h
··· 17 17 18 18 static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 19 19 { 20 - asm_volatile_goto("1:\n\t" 20 + asm goto("1:\n\t" 21 21 "nop # arch_static_branch\n\t" 22 22 ".pushsection __jump_table, \"aw\"\n\t" 23 23 ".long 1b - ., %l[l_yes] - .\n\t" ··· 32 32 33 33 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 34 34 { 35 - asm_volatile_goto("1:\n\t" 35 + asm goto("1:\n\t" 36 36 "b %l[l_yes] # arch_static_branch_jump\n\t" 37 37 ".pushsection __jump_table, \"aw\"\n\t" 38 38 ".long 1b - ., %l[l_yes] - .\n\t"
+6 -6
arch/powerpc/include/asm/uaccess.h
··· 74 74 /* -mprefixed can generate offsets beyond range, fall back hack */ 75 75 #ifdef CONFIG_PPC_KERNEL_PREFIXED 76 76 #define __put_user_asm_goto(x, addr, label, op) \ 77 - asm_volatile_goto( \ 77 + asm goto( \ 78 78 "1: " op " %0,0(%1) # put_user\n" \ 79 79 EX_TABLE(1b, %l2) \ 80 80 : \ ··· 83 83 : label) 84 84 #else 85 85 #define __put_user_asm_goto(x, addr, label, op) \ 86 - asm_volatile_goto( \ 86 + asm goto( \ 87 87 "1: " op "%U1%X1 %0,%1 # put_user\n" \ 88 88 EX_TABLE(1b, %l2) \ 89 89 : \ ··· 97 97 __put_user_asm_goto(x, ptr, label, "std") 98 98 #else /* __powerpc64__ */ 99 99 #define __put_user_asm2_goto(x, addr, label) \ 100 - asm_volatile_goto( \ 100 + asm goto( \ 101 101 "1: stw%X1 %0, %1\n" \ 102 102 "2: stw%X1 %L0, %L1\n" \ 103 103 EX_TABLE(1b, %l2) \ ··· 146 146 /* -mprefixed can generate offsets beyond range, fall back hack */ 147 147 #ifdef CONFIG_PPC_KERNEL_PREFIXED 148 148 #define __get_user_asm_goto(x, addr, label, op) \ 149 - asm_volatile_goto( \ 149 + asm_goto_output( \ 150 150 "1: "op" %0,0(%1) # get_user\n" \ 151 151 EX_TABLE(1b, %l2) \ 152 152 : "=r" (x) \ ··· 155 155 : label) 156 156 #else 157 157 #define __get_user_asm_goto(x, addr, label, op) \ 158 - asm_volatile_goto( \ 158 + asm_goto_output( \ 159 159 "1: "op"%U1%X1 %0, %1 # get_user\n" \ 160 160 EX_TABLE(1b, %l2) \ 161 161 : "=r" (x) \ ··· 169 169 __get_user_asm_goto(x, addr, label, "ld") 170 170 #else /* __powerpc64__ */ 171 171 #define __get_user_asm2_goto(x, addr, label) \ 172 - asm_volatile_goto( \ 172 + asm_goto_output( \ 173 173 "1: lwz%X1 %0, %1\n" \ 174 174 "2: lwz%X1 %L0, %L1\n" \ 175 175 EX_TABLE(1b, %l2) \
+1 -1
arch/powerpc/kernel/irq_64.c
··· 230 230 * This allows interrupts to be unmasked without hard disabling, and 231 231 * also without new hard interrupts coming in ahead of pending ones. 232 232 */ 233 - asm_volatile_goto( 233 + asm goto( 234 234 "1: \n" 235 235 " lbz 9,%0(13) \n" 236 236 " cmpwi 9,0 \n"
+2 -2
arch/riscv/include/asm/arch_hweight.h
··· 20 20 static __always_inline unsigned int __arch_hweight32(unsigned int w) 21 21 { 22 22 #ifdef CONFIG_RISCV_ISA_ZBB 23 - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 23 + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 24 24 RISCV_ISA_EXT_ZBB, 1) 25 25 : : : : legacy); 26 26 ··· 51 51 static __always_inline unsigned long __arch_hweight64(__u64 w) 52 52 { 53 53 # ifdef CONFIG_RISCV_ISA_ZBB 54 - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 54 + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 55 55 RISCV_ISA_EXT_ZBB, 1) 56 56 : : : : legacy); 57 57
+4 -4
arch/riscv/include/asm/bitops.h
··· 39 39 { 40 40 int num; 41 41 42 - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 42 + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 43 43 RISCV_ISA_EXT_ZBB, 1) 44 44 : : : : legacy); 45 45 ··· 95 95 { 96 96 int num; 97 97 98 - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 98 + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 99 99 RISCV_ISA_EXT_ZBB, 1) 100 100 : : : : legacy); 101 101 ··· 154 154 if (!x) 155 155 return 0; 156 156 157 - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 157 + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 158 158 RISCV_ISA_EXT_ZBB, 1) 159 159 : : : : legacy); 160 160 ··· 209 209 if (!x) 210 210 return 0; 211 211 212 - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 212 + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 213 213 RISCV_ISA_EXT_ZBB, 1) 214 214 : : : : legacy); 215 215
+1 -1
arch/riscv/include/asm/checksum.h
··· 53 53 IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { 54 54 unsigned long fold_temp; 55 55 56 - asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 56 + asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 57 57 RISCV_ISA_EXT_ZBB, 1) 58 58 : 59 59 :
+2 -2
arch/riscv/include/asm/cpufeature.h
··· 80 80 "ext must be < RISCV_ISA_EXT_MAX"); 81 81 82 82 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { 83 - asm_volatile_goto( 83 + asm goto( 84 84 ALTERNATIVE("j %l[l_no]", "nop", 0, %[ext], 1) 85 85 : 86 86 : [ext] "i" (ext) ··· 103 103 "ext must be < RISCV_ISA_EXT_MAX"); 104 104 105 105 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { 106 - asm_volatile_goto( 106 + asm goto( 107 107 ALTERNATIVE("nop", "j %l[l_yes]", 0, %[ext], 1) 108 108 : 109 109 : [ext] "i" (ext)
+2 -2
arch/riscv/include/asm/jump_label.h
··· 17 17 static __always_inline bool arch_static_branch(struct static_key * const key, 18 18 const bool branch) 19 19 { 20 - asm_volatile_goto( 20 + asm goto( 21 21 " .align 2 \n\t" 22 22 " .option push \n\t" 23 23 " .option norelax \n\t" ··· 39 39 static __always_inline bool arch_static_branch_jump(struct static_key * const key, 40 40 const bool branch) 41 41 { 42 - asm_volatile_goto( 42 + asm goto( 43 43 " .align 2 \n\t" 44 44 " .option push \n\t" 45 45 " .option norelax \n\t"
+5 -5
arch/riscv/lib/csum.c
··· 53 53 * support, so nop when Zbb is available and jump when Zbb is 54 54 * not available. 55 55 */ 56 - asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 56 + asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 57 57 RISCV_ISA_EXT_ZBB, 1) 58 58 : 59 59 : ··· 170 170 * support, so nop when Zbb is available and jump when Zbb is 171 171 * not available. 172 172 */ 173 - asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 173 + asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 174 174 RISCV_ISA_EXT_ZBB, 1) 175 175 : 176 176 : ··· 178 178 : no_zbb); 179 179 180 180 #ifdef CONFIG_32BIT 181 - asm_volatile_goto(".option push \n\ 181 + asm_goto_output(".option push \n\ 182 182 .option arch,+zbb \n\ 183 183 rori %[fold_temp], %[csum], 16 \n\ 184 184 andi %[offset], %[offset], 1 \n\ ··· 193 193 194 194 return (unsigned short)csum; 195 195 #else /* !CONFIG_32BIT */ 196 - asm_volatile_goto(".option push \n\ 196 + asm_goto_output(".option push \n\ 197 197 .option arch,+zbb \n\ 198 198 rori %[fold_temp], %[csum], 32 \n\ 199 199 add %[csum], %[fold_temp], %[csum] \n\ ··· 257 257 * support, so nop when Zbb is available and jump when Zbb is 258 258 * not available. 259 259 */ 260 - asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 260 + asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, 261 261 RISCV_ISA_EXT_ZBB, 1) 262 262 : 263 263 :
+2 -2
arch/s390/include/asm/jump_label.h
··· 25 25 */ 26 26 static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 27 27 { 28 - asm_volatile_goto("0: brcl 0,%l[label]\n" 28 + asm goto("0: brcl 0,%l[label]\n" 29 29 ".pushsection __jump_table,\"aw\"\n" 30 30 ".balign 8\n" 31 31 ".long 0b-.,%l[label]-.\n" ··· 39 39 40 40 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 41 41 { 42 - asm_volatile_goto("0: brcl 15,%l[label]\n" 42 + asm goto("0: brcl 15,%l[label]\n" 43 43 ".pushsection __jump_table,\"aw\"\n" 44 44 ".balign 8\n" 45 45 ".long 0b-.,%l[label]-.\n"
+2 -2
arch/sparc/include/asm/jump_label.h
··· 10 10 11 11 static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 12 12 { 13 - asm_volatile_goto("1:\n\t" 13 + asm goto("1:\n\t" 14 14 "nop\n\t" 15 15 "nop\n\t" 16 16 ".pushsection __jump_table, \"aw\"\n\t" ··· 26 26 27 27 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 28 28 { 29 - asm_volatile_goto("1:\n\t" 29 + asm goto("1:\n\t" 30 30 "b %l[l_yes]\n\t" 31 31 "nop\n\t" 32 32 ".pushsection __jump_table, \"aw\"\n\t"
+1 -1
arch/um/include/asm/cpufeature.h
··· 75 75 */ 76 76 static __always_inline bool _static_cpu_has(u16 bit) 77 77 { 78 - asm_volatile_goto("1: jmp 6f\n" 78 + asm goto("1: jmp 6f\n" 79 79 "2:\n" 80 80 ".skip -(((5f-4f) - (2b-1b)) > 0) * " 81 81 "((5f-4f) - (2b-1b)),0x90\n"
+1 -1
arch/x86/include/asm/cpufeature.h
··· 168 168 */ 169 169 static __always_inline bool _static_cpu_has(u16 bit) 170 170 { 171 - asm_volatile_goto( 171 + asm goto( 172 172 ALTERNATIVE_TERNARY("jmp 6f", %P[feature], "", "jmp %l[t_no]") 173 173 ".pushsection .altinstr_aux,\"ax\"\n" 174 174 "6:\n"
+3 -3
arch/x86/include/asm/jump_label.h
··· 24 24 25 25 static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 26 26 { 27 - asm_volatile_goto("1:" 27 + asm goto("1:" 28 28 "jmp %l[l_yes] # objtool NOPs this \n\t" 29 29 JUMP_TABLE_ENTRY 30 30 : : "i" (key), "i" (2 | branch) : : l_yes); ··· 38 38 39 39 static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) 40 40 { 41 - asm_volatile_goto("1:" 41 + asm goto("1:" 42 42 ".byte " __stringify(BYTES_NOP5) "\n\t" 43 43 JUMP_TABLE_ENTRY 44 44 : : "i" (key), "i" (branch) : : l_yes); ··· 52 52 53 53 static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) 54 54 { 55 - asm_volatile_goto("1:" 55 + asm goto("1:" 56 56 "jmp %l[l_yes]\n\t" 57 57 JUMP_TABLE_ENTRY 58 58 : : "i" (key), "i" (branch) : : l_yes);
+1 -1
arch/x86/include/asm/rmwcc.h
··· 13 13 #define __GEN_RMWcc(fullop, _var, cc, clobbers, ...) \ 14 14 ({ \ 15 15 bool c = false; \ 16 - asm_volatile_goto (fullop "; j" #cc " %l[cc_label]" \ 16 + asm goto (fullop "; j" #cc " %l[cc_label]" \ 17 17 : : [var] "m" (_var), ## __VA_ARGS__ \ 18 18 : clobbers : cc_label); \ 19 19 if (0) { \
+1 -1
arch/x86/include/asm/special_insns.h
··· 205 205 #ifdef CONFIG_X86_USER_SHADOW_STACK 206 206 static inline int write_user_shstk_64(u64 __user *addr, u64 val) 207 207 { 208 - asm_volatile_goto("1: wrussq %[val], (%[addr])\n" 208 + asm goto("1: wrussq %[val], (%[addr])\n" 209 209 _ASM_EXTABLE(1b, %l[fail]) 210 210 :: [addr] "r" (addr), [val] "r" (val) 211 211 :: fail);
+5 -5
arch/x86/include/asm/uaccess.h
··· 133 133 134 134 #ifdef CONFIG_X86_32 135 135 #define __put_user_goto_u64(x, addr, label) \ 136 - asm_volatile_goto("\n" \ 136 + asm goto("\n" \ 137 137 "1: movl %%eax,0(%1)\n" \ 138 138 "2: movl %%edx,4(%1)\n" \ 139 139 _ASM_EXTABLE_UA(1b, %l2) \ ··· 295 295 } while (0) 296 296 297 297 #define __get_user_asm(x, addr, itype, ltype, label) \ 298 - asm_volatile_goto("\n" \ 298 + asm_goto_output("\n" \ 299 299 "1: mov"itype" %[umem],%[output]\n" \ 300 300 _ASM_EXTABLE_UA(1b, %l2) \ 301 301 : [output] ltype(x) \ ··· 375 375 __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ 376 376 __typeof__(*(_ptr)) __old = *_old; \ 377 377 __typeof__(*(_ptr)) __new = (_new); \ 378 - asm_volatile_goto("\n" \ 378 + asm_goto_output("\n" \ 379 379 "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\ 380 380 _ASM_EXTABLE_UA(1b, %l[label]) \ 381 381 : CC_OUT(z) (success), \ ··· 394 394 __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ 395 395 __typeof__(*(_ptr)) __old = *_old; \ 396 396 __typeof__(*(_ptr)) __new = (_new); \ 397 - asm_volatile_goto("\n" \ 397 + asm_goto_output("\n" \ 398 398 "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \ 399 399 _ASM_EXTABLE_UA(1b, %l[label]) \ 400 400 : CC_OUT(z) (success), \ ··· 477 477 * aliasing issues. 478 478 */ 479 479 #define __put_user_goto(x, addr, itype, ltype, label) \ 480 - asm_volatile_goto("\n" \ 480 + asm goto("\n" \ 481 481 "1: mov"itype" %0,%1\n" \ 482 482 _ASM_EXTABLE_UA(1b, %l2) \ 483 483 : : ltype(x), "m" (__m(addr)) \
+3 -3
arch/x86/kvm/svm/svm_ops.h
··· 8 8 9 9 #define svm_asm(insn, clobber...) \ 10 10 do { \ 11 - asm_volatile_goto("1: " __stringify(insn) "\n\t" \ 11 + asm goto("1: " __stringify(insn) "\n\t" \ 12 12 _ASM_EXTABLE(1b, %l[fault]) \ 13 13 ::: clobber : fault); \ 14 14 return; \ ··· 18 18 19 19 #define svm_asm1(insn, op1, clobber...) \ 20 20 do { \ 21 - asm_volatile_goto("1: " __stringify(insn) " %0\n\t" \ 21 + asm goto("1: " __stringify(insn) " %0\n\t" \ 22 22 _ASM_EXTABLE(1b, %l[fault]) \ 23 23 :: op1 : clobber : fault); \ 24 24 return; \ ··· 28 28 29 29 #define svm_asm2(insn, op1, op2, clobber...) \ 30 30 do { \ 31 - asm_volatile_goto("1: " __stringify(insn) " %1, %0\n\t" \ 31 + asm goto("1: " __stringify(insn) " %1, %0\n\t" \ 32 32 _ASM_EXTABLE(1b, %l[fault]) \ 33 33 :: op1, op2 : clobber : fault); \ 34 34 return; \
+2 -2
arch/x86/kvm/vmx/vmx.c
··· 738 738 */ 739 739 static int kvm_cpu_vmxoff(void) 740 740 { 741 - asm_volatile_goto("1: vmxoff\n\t" 741 + asm goto("1: vmxoff\n\t" 742 742 _ASM_EXTABLE(1b, %l[fault]) 743 743 ::: "cc", "memory" : fault); 744 744 ··· 2784 2784 2785 2785 cr4_set_bits(X86_CR4_VMXE); 2786 2786 2787 - asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t" 2787 + asm goto("1: vmxon %[vmxon_pointer]\n\t" 2788 2788 _ASM_EXTABLE(1b, %l[fault]) 2789 2789 : : [vmxon_pointer] "m"(vmxon_pointer) 2790 2790 : : fault);
+3 -3
arch/x86/kvm/vmx/vmx_ops.h
··· 94 94 95 95 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 96 96 97 - asm_volatile_goto("1: vmread %[field], %[output]\n\t" 97 + asm_goto_output("1: vmread %[field], %[output]\n\t" 98 98 "jna %l[do_fail]\n\t" 99 99 100 100 _ASM_EXTABLE(1b, %l[do_exception]) ··· 188 188 189 189 #define vmx_asm1(insn, op1, error_args...) \ 190 190 do { \ 191 - asm_volatile_goto("1: " __stringify(insn) " %0\n\t" \ 191 + asm goto("1: " __stringify(insn) " %0\n\t" \ 192 192 ".byte 0x2e\n\t" /* branch not taken hint */ \ 193 193 "jna %l[error]\n\t" \ 194 194 _ASM_EXTABLE(1b, %l[fault]) \ ··· 205 205 206 206 #define vmx_asm2(insn, op1, op2, error_args...) \ 207 207 do { \ 208 - asm_volatile_goto("1: " __stringify(insn) " %1, %0\n\t" \ 208 + asm goto("1: " __stringify(insn) " %1, %0\n\t" \ 209 209 ".byte 0x2e\n\t" /* branch not taken hint */ \ 210 210 "jna %l[error]\n\t" \ 211 211 _ASM_EXTABLE(1b, %l[fault]) \
+2 -2
arch/xtensa/include/asm/jump_label.h
··· 13 13 static __always_inline bool arch_static_branch(struct static_key *key, 14 14 bool branch) 15 15 { 16 - asm_volatile_goto("1:\n\t" 16 + asm goto("1:\n\t" 17 17 "_nop\n\t" 18 18 ".pushsection __jump_table, \"aw\"\n\t" 19 19 ".word 1b, %l[l_yes], %c0\n\t" ··· 38 38 * make it reachable and wrap both into a no-transform block 39 39 * to avoid any assembler interference with this. 40 40 */ 41 - asm_volatile_goto("1:\n\t" 41 + asm goto("1:\n\t" 42 42 ".begin no-transform\n\t" 43 43 "_j %l[l_yes]\n\t" 44 44 "2:\n\t"
+19
include/linux/compiler-gcc.h
··· 64 64 __builtin_unreachable(); \ 65 65 } while (0) 66 66 67 + /* 68 + * GCC 'asm goto' with outputs miscompiles certain code sequences: 69 + * 70 + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110420 71 + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110422 72 + * 73 + * Work it around via the same compiler barrier quirk that we used 74 + * to use for the old 'asm goto' workaround. 75 + * 76 + * Also, always mark such 'asm goto' statements as volatile: all 77 + * asm goto statements are supposed to be volatile as per the 78 + * documentation, but some versions of gcc didn't actually do 79 + * that for asms with outputs: 80 + * 81 + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98619 82 + */ 83 + #define asm_goto_output(x...) \ 84 + do { asm volatile goto(x); asm (""); } while (0) 85 + 67 86 #if defined(CONFIG_ARCH_USE_BUILTIN_BSWAP) 68 87 #define __HAVE_BUILTIN_BSWAP32__ 69 88 #define __HAVE_BUILTIN_BSWAP64__
+2 -2
include/linux/compiler_types.h
··· 362 362 #define __member_size(p) __builtin_object_size(p, 1) 363 363 #endif 364 364 365 - #ifndef asm_volatile_goto 366 - #define asm_volatile_goto(x...) asm goto(x) 365 + #ifndef asm_goto_output 366 + #define asm_goto_output(x...) asm goto(x) 367 367 #endif 368 368 369 369 #ifdef CONFIG_CC_HAS_ASM_INLINE
+1 -1
net/netfilter/nft_set_pipapo_avx2.c
··· 57 57 58 58 /* Jump to label if @reg is zero */ 59 59 #define NFT_PIPAPO_AVX2_NOMATCH_GOTO(reg, label) \ 60 - asm_volatile_goto("vptest %%ymm" #reg ", %%ymm" #reg ";" \ 60 + asm goto("vptest %%ymm" #reg ", %%ymm" #reg ";" \ 61 61 "je %l[" #label "]" : : : : label) 62 62 63 63 /* Store 256 bits from YMM register into memory. Contrary to bucket load
+4 -4
samples/bpf/asm_goto_workaround.h
··· 4 4 #define __ASM_GOTO_WORKAROUND_H 5 5 6 6 /* 7 - * This will bring in asm_volatile_goto and asm_inline macro definitions 7 + * This will bring in asm_goto_output and asm_inline macro definitions 8 8 * if enabled by compiler and config options. 9 9 */ 10 10 #include <linux/types.h> 11 11 12 - #ifdef asm_volatile_goto 13 - #undef asm_volatile_goto 14 - #define asm_volatile_goto(x...) asm volatile("invalid use of asm_volatile_goto") 12 + #ifdef asm_goto_output 13 + #undef asm_goto_output 14 + #define asm_goto_output(x...) asm volatile("invalid use of asm_goto_output") 15 15 #endif 16 16 17 17 /*
+1 -1
tools/arch/x86/include/asm/rmwcc.h
··· 4 4 5 5 #define __GEN_RMWcc(fullop, var, cc, ...) \ 6 6 do { \ 7 - asm_volatile_goto (fullop "; j" cc " %l[cc_label]" \ 7 + asm goto (fullop "; j" cc " %l[cc_label]" \ 8 8 : : "m" (var), ## __VA_ARGS__ \ 9 9 : "memory" : cc_label); \ 10 10 return 0; \
+2 -2
tools/include/linux/compiler_types.h
··· 36 36 #include <linux/compiler-gcc.h> 37 37 #endif 38 38 39 - #ifndef asm_volatile_goto 40 - #define asm_volatile_goto(x...) asm goto(x) 39 + #ifndef asm_goto_output 40 + #define asm_goto_output(x...) asm goto(x) 41 41 #endif 42 42 43 43 #endif /* __LINUX_COMPILER_TYPES_H */