tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

x86/alternatives: Add alt_instr.flags

Add a struct alt_instr.flags field which will contain different flags
controlling alternatives patching behavior.

The initial idea was to be able to specify it as a separate macro
parameter but that would mean touching all possible invocations of the
alternatives macros and thus a lot of churn.

What is more, as PeterZ suggested, being able to say ALT_NOT(feature) is
very readable and explains exactly what is meant.

So make the feature field a u32 where the patching flags are the upper
u16 part of the dword quantity while the lower u16 word is the feature.

The highest feature number currently is 0x26a (i.e., word 19) so there
is plenty of space. If that becomes insufficient, the field can be
extended to u64 which will then make struct alt_instr of the nice size
of 16 bytes (14 bytes currently).

There should be no functional changes resulting from this.

Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/Y6RCoJEtxxZWwotd@zn.tnic

Borislav Petkov (AMD) 5d1dd961 1b929c02

+84 -66
+75 -55
arch/x86/include/asm/alternative.h
··· 6 6 #include <linux/stringify.h> 7 7 #include <asm/asm.h> 8 8 9 - #define ALTINSTR_FLAG_INV (1 << 15) 10 - #define ALT_NOT(feat) ((feat) | ALTINSTR_FLAG_INV) 9 + #define ALT_FLAGS_SHIFT 16 10 + 11 + #define ALT_FLAG_NOT BIT(0) 12 + #define ALT_NOT(feature) ((ALT_FLAG_NOT << ALT_FLAGS_SHIFT) | (feature)) 11 13 12 14 #ifndef __ASSEMBLY__ 13 15 ··· 61 59 ".long 999b - .\n\t" \ 62 60 ".popsection\n\t" 63 61 62 + /* 63 + * The patching flags are part of the upper bits of the @ft_flags parameter when 64 + * specifying them. The split is currently like this: 65 + * 66 + * [31... flags ...16][15... CPUID feature bit ...0] 67 + * 68 + * but since this is all hidden in the macros argument being split, those fields can be 69 + * extended in the future to fit in a u64 or however the need arises. 70 + */ 64 71 struct alt_instr { 65 72 s32 instr_offset; /* original instruction */ 66 73 s32 repl_offset; /* offset to replacement instruction */ 67 - u16 cpuid; /* cpuid bit set for replacement */ 74 + 75 + union { 76 + struct { 77 + u32 cpuid: 16; /* CPUID bit set for replacement */ 78 + u32 flags: 16; /* patching control flags */ 79 + }; 80 + u32 ft_flags; 81 + }; 82 + 68 83 u8 instrlen; /* length of original instruction */ 69 84 u8 replacementlen; /* length of new instruction */ 70 85 } __packed; ··· 201 182 " - (" alt_slen ")), 0x90\n" \ 202 183 alt_end_marker ":\n" 203 184 204 - #define ALTINSTR_ENTRY(feature, num) \ 185 + #define ALTINSTR_ENTRY(ft_flags, num) \ 205 186 " .long 661b - .\n" /* label */ \ 206 187 " .long " b_replacement(num)"f - .\n" /* new instruction */ \ 207 - " .word " __stringify(feature) "\n" /* feature bit */ \ 188 + " .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \ 208 189 " .byte " alt_total_slen "\n" /* source len */ \ 209 190 " .byte " alt_rlen(num) "\n" /* replacement len */ 210 191 ··· 213 194 b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n" 214 195 215 196 /* alternative assembly primitive: */ 216 - #define ALTERNATIVE(oldinstr, newinstr, feature) \ 197 + #define ALTERNATIVE(oldinstr, newinstr, ft_flags) \ 217 198 OLDINSTR(oldinstr, 1) \ 218 199 ".pushsection .altinstructions,\"a\"\n" \ 219 - ALTINSTR_ENTRY(feature, 1) \ 200 + ALTINSTR_ENTRY(ft_flags, 1) \ 220 201 ".popsection\n" \ 221 202 ".pushsection .altinstr_replacement, \"ax\"\n" \ 222 203 ALTINSTR_REPLACEMENT(newinstr, 1) \ 223 204 ".popsection\n" 224 205 225 - #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ 206 + #define ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \ 226 207 OLDINSTR_2(oldinstr, 1, 2) \ 227 208 ".pushsection .altinstructions,\"a\"\n" \ 228 - ALTINSTR_ENTRY(feature1, 1) \ 229 - ALTINSTR_ENTRY(feature2, 2) \ 209 + ALTINSTR_ENTRY(ft_flags1, 1) \ 210 + ALTINSTR_ENTRY(ft_flags2, 2) \ 230 211 ".popsection\n" \ 231 212 ".pushsection .altinstr_replacement, \"ax\"\n" \ 232 213 ALTINSTR_REPLACEMENT(newinstr1, 1) \ ··· 234 215 ".popsection\n" 235 216 236 217 /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */ 237 - #define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \ 218 + #define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ 238 219 ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \ 239 - newinstr_yes, feature) 220 + newinstr_yes, ft_flags) 240 221 241 - #define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \ 242 - OLDINSTR_3(oldinsn, 1, 2, 3) \ 243 - ".pushsection .altinstructions,\"a\"\n" \ 244 - ALTINSTR_ENTRY(feat1, 1) \ 245 - ALTINSTR_ENTRY(feat2, 2) \ 246 - ALTINSTR_ENTRY(feat3, 3) \ 247 - ".popsection\n" \ 248 - ".pushsection .altinstr_replacement, \"ax\"\n" \ 249 - ALTINSTR_REPLACEMENT(newinsn1, 1) \ 250 - ALTINSTR_REPLACEMENT(newinsn2, 2) \ 251 - ALTINSTR_REPLACEMENT(newinsn3, 3) \ 222 + #define ALTERNATIVE_3(oldinsn, newinsn1, ft_flags1, newinsn2, ft_flags2, \ 223 + newinsn3, ft_flags3) \ 224 + OLDINSTR_3(oldinsn, 1, 2, 3) \ 225 + ".pushsection .altinstructions,\"a\"\n" \ 226 + ALTINSTR_ENTRY(ft_flags1, 1) \ 227 + ALTINSTR_ENTRY(ft_flags2, 2) \ 228 + ALTINSTR_ENTRY(ft_flags3, 3) \ 229 + ".popsection\n" \ 230 + ".pushsection .altinstr_replacement, \"ax\"\n" \ 231 + ALTINSTR_REPLACEMENT(newinsn1, 1) \ 232 + ALTINSTR_REPLACEMENT(newinsn2, 2) \ 233 + ALTINSTR_REPLACEMENT(newinsn3, 3) \ 252 234 ".popsection\n" 253 235 254 236 /* ··· 264 244 * For non barrier like inlines please define new variants 265 245 * without volatile and memory clobber. 266 246 */ 267 - #define alternative(oldinstr, newinstr, feature) \ 268 - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") 247 + #define alternative(oldinstr, newinstr, ft_flags) \ 248 + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) : : : "memory") 269 249 270 - #define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \ 271 - asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory") 250 + #define alternative_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \ 251 + asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) ::: "memory") 272 252 273 - #define alternative_ternary(oldinstr, feature, newinstr_yes, newinstr_no) \ 274 - asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) ::: "memory") 253 + #define alternative_ternary(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ 254 + asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) ::: "memory") 275 255 276 256 /* 277 257 * Alternative inline assembly with input. ··· 281 261 * Argument numbers start with 1. 282 262 * Leaving an unused argument 0 to keep API compatibility. 283 263 */ 284 - #define alternative_input(oldinstr, newinstr, feature, input...) \ 285 - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 264 + #define alternative_input(oldinstr, newinstr, ft_flags, input...) \ 265 + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \ 286 266 : : "i" (0), ## input) 287 267 288 268 /* ··· 293 273 * Otherwise, if CPU has feature1, newinstr1 is used. 294 274 * Otherwise, oldinstr is used. 295 275 */ 296 - #define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \ 297 - feature2, input...) \ 298 - asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \ 299 - newinstr2, feature2) \ 276 + #define alternative_input_2(oldinstr, newinstr1, ft_flags1, newinstr2, \ 277 + ft_flags2, input...) \ 278 + asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, \ 279 + newinstr2, ft_flags2) \ 300 280 : : "i" (0), ## input) 301 281 302 282 /* Like alternative_input, but with a single output argument */ 303 - #define alternative_io(oldinstr, newinstr, feature, output, input...) \ 304 - asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 283 + #define alternative_io(oldinstr, newinstr, ft_flags, output, input...) \ 284 + asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \ 305 285 : output : "i" (0), ## input) 306 286 307 287 /* Like alternative_io, but for replacing a direct call with another one. */ 308 - #define alternative_call(oldfunc, newfunc, feature, output, input...) \ 309 - asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ 288 + #define alternative_call(oldfunc, newfunc, ft_flags, output, input...) \ 289 + asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", ft_flags) \ 310 290 : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) 311 291 312 292 /* ··· 315 295 * Otherwise, if CPU has feature1, function1 is used. 316 296 * Otherwise, old function is used. 317 297 */ 318 - #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \ 298 + #define alternative_call_2(oldfunc, newfunc1, ft_flags1, newfunc2, ft_flags2, \ 319 299 output, input...) \ 320 - asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\ 321 - "call %P[new2]", feature2) \ 300 + asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", ft_flags1,\ 301 + "call %P[new2]", ft_flags2) \ 322 302 : output, ASM_CALL_CONSTRAINT \ 323 303 : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ 324 304 [new2] "i" (newfunc2), ## input) ··· 367 347 * enough information for the alternatives patching code to patch an 368 348 * instruction. See apply_alternatives(). 369 349 */ 370 - .macro altinstruction_entry orig alt feature orig_len alt_len 350 + .macro altinstr_entry orig alt ft_flags orig_len alt_len 371 351 .long \orig - . 372 352 .long \alt - . 373 - .word \feature 353 + .4byte \ft_flags 374 354 .byte \orig_len 375 355 .byte \alt_len 376 356 .endm ··· 381 361 * @newinstr. ".skip" directive takes care of proper instruction padding 382 362 * in case @newinstr is longer than @oldinstr. 383 363 */ 384 - .macro ALTERNATIVE oldinstr, newinstr, feature 364 + .macro ALTERNATIVE oldinstr, newinstr, ft_flags 385 365 140: 386 366 \oldinstr 387 367 141: ··· 389 369 142: 390 370 391 371 .pushsection .altinstructions,"a" 392 - altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f 372 + altinstr_entry 140b,143f,\ft_flags,142b-140b,144f-143f 393 373 .popsection 394 374 395 375 .pushsection .altinstr_replacement,"ax" ··· 419 399 * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has 420 400 * @feature2, it replaces @oldinstr with @feature2. 421 401 */ 422 - .macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2 402 + .macro ALTERNATIVE_2 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2 423 403 140: 424 404 \oldinstr 425 405 141: ··· 428 408 142: 429 409 430 410 .pushsection .altinstructions,"a" 431 - altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f 432 - altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f 411 + altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f 412 + altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f 433 413 .popsection 434 414 435 415 .pushsection .altinstr_replacement,"ax" ··· 441 421 .popsection 442 422 .endm 443 423 444 - .macro ALTERNATIVE_3 oldinstr, newinstr1, feature1, newinstr2, feature2, newinstr3, feature3 424 + .macro ALTERNATIVE_3 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2, newinstr3, ft_flags3 445 425 140: 446 426 \oldinstr 447 427 141: ··· 450 430 142: 451 431 452 432 .pushsection .altinstructions,"a" 453 - altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f 454 - altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f 455 - altinstruction_entry 140b,145f,\feature3,142b-140b,146f-145f 433 + altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f 434 + altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f 435 + altinstr_entry 140b,145f,\ft_flags3,142b-140b,146f-145f 456 436 .popsection 457 437 458 438 .pushsection .altinstr_replacement,"ax" ··· 467 447 .endm 468 448 469 449 /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */ 470 - #define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \ 450 + #define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \ 471 451 ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \ 472 - newinstr_yes, feature 452 + newinstr_yes, ft_flags 473 453 474 454 #endif /* __ASSEMBLY__ */ 475 455
+6 -8
arch/x86/kernel/alternative.c
··· 282 282 */ 283 283 for (a = start; a < end; a++) { 284 284 int insn_buff_sz = 0; 285 - /* Mask away "NOT" flag bit for feature to test. */ 286 - u16 feature = a->cpuid & ~ALTINSTR_FLAG_INV; 287 285 288 286 instr = (u8 *)&a->instr_offset + a->instr_offset; 289 287 replacement = (u8 *)&a->repl_offset + a->repl_offset; 290 288 BUG_ON(a->instrlen > sizeof(insn_buff)); 291 - BUG_ON(feature >= (NCAPINTS + NBUGINTS) * 32); 289 + BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); 292 290 293 291 /* 294 292 * Patch if either: 295 293 * - feature is present 296 - * - feature not present but ALTINSTR_FLAG_INV is set to mean, 294 + * - feature not present but ALT_FLAG_NOT is set to mean, 297 295 * patch if feature is *NOT* present. 298 296 */ 299 - if (!boot_cpu_has(feature) == !(a->cpuid & ALTINSTR_FLAG_INV)) 297 + if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) 300 298 goto next; 301 299 302 300 DPRINTK("feat: %s%d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d)", 303 - (a->cpuid & ALTINSTR_FLAG_INV) ? "!" : "", 304 - feature >> 5, 305 - feature & 0x1f, 301 + (a->flags & ALT_FLAG_NOT) ? "!" : "", 302 + a->cpuid >> 5, 303 + a->cpuid & 0x1f, 306 304 instr, instr, a->instrlen, 307 305 replacement, a->replacementlen); 308 306
+3 -3
tools/objtool/arch/x86/include/arch/special.h
··· 11 11 #define JUMP_NEW_OFFSET 4 12 12 #define JUMP_KEY_OFFSET 8 13 13 14 - #define ALT_ENTRY_SIZE 12 14 + #define ALT_ENTRY_SIZE 14 15 15 #define ALT_ORIG_OFFSET 0 16 16 #define ALT_NEW_OFFSET 4 17 17 #define ALT_FEATURE_OFFSET 8 18 - #define ALT_ORIG_LEN_OFFSET 10 19 - #define ALT_NEW_LEN_OFFSET 11 18 + #define ALT_ORIG_LEN_OFFSET 12 19 + #define ALT_NEW_LEN_OFFSET 13 20 20 21 21 #endif /* _X86_ARCH_SPECIAL_H */