Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

x86/percpu: Clean up percpu_xchg_op()

The core percpu macros already have a switch on the data size, so the switch
in the x86 code is redundant and produces more dead code.

Also use appropriate types for the width of the instructions. This avoids
errors when compiling with Clang.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Signed-off-by: Nick Desaulniers <ndesaulniers@google.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Nick Desaulniers <ndesaulniers@google.com>
Tested-by: Sedat Dilek <sedat.dilek@gmail.com>
Reviewed-by: Nick Desaulniers <ndesaulniers@google.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Dennis Zhou <dennis@kernel.org>
Link: https://lkml.kernel.org/r/20200720204925.3654302-8-ndesaulniers@google.com

authored by

Brian Gerst and committed by
Thomas Gleixner
73ca542f bbff583b

+18 -43
+18 -43
arch/x86/include/asm/percpu.h
··· 215 215 * expensive due to the implied lock prefix. The processor cannot prefetch 216 216 * cachelines if xchg is used. 217 217 */ 218 - #define percpu_xchg_op(qual, var, nval) \ 218 + #define percpu_xchg_op(size, qual, _var, _nval) \ 219 219 ({ \ 220 - typeof(var) pxo_ret__; \ 221 - typeof(var) pxo_new__ = (nval); \ 222 - switch (sizeof(var)) { \ 223 - case 1: \ 224 - asm qual ("\n\tmov "__percpu_arg(1)",%%al" \ 225 - "\n1:\tcmpxchgb %2, "__percpu_arg(1) \ 226 - "\n\tjnz 1b" \ 227 - : "=&a" (pxo_ret__), "+m" (var) \ 228 - : "q" (pxo_new__) \ 229 - : "memory"); \ 230 - break; \ 231 - case 2: \ 232 - asm qual ("\n\tmov "__percpu_arg(1)",%%ax" \ 233 - "\n1:\tcmpxchgw %2, "__percpu_arg(1) \ 234 - "\n\tjnz 1b" \ 235 - : "=&a" (pxo_ret__), "+m" (var) \ 236 - : "r" (pxo_new__) \ 237 - : "memory"); \ 238 - break; \ 239 - case 4: \ 240 - asm qual ("\n\tmov "__percpu_arg(1)",%%eax" \ 241 - "\n1:\tcmpxchgl %2, "__percpu_arg(1) \ 242 - "\n\tjnz 1b" \ 243 - : "=&a" (pxo_ret__), "+m" (var) \ 244 - : "r" (pxo_new__) \ 245 - : "memory"); \ 246 - break; \ 247 - case 8: \ 248 - asm qual ("\n\tmov "__percpu_arg(1)",%%rax" \ 249 - "\n1:\tcmpxchgq %2, "__percpu_arg(1) \ 250 - "\n\tjnz 1b" \ 251 - : "=&a" (pxo_ret__), "+m" (var) \ 252 - : "r" (pxo_new__) \ 253 - : "memory"); \ 254 - break; \ 255 - default: __bad_percpu_size(); \ 256 - } \ 257 - pxo_ret__; \ 220 + __pcpu_type_##size pxo_old__; \ 221 + __pcpu_type_##size pxo_new__ = __pcpu_cast_##size(_nval); \ 222 + asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), \ 223 + "%[oval]") \ 224 + "\n1:\t" \ 225 + __pcpu_op2_##size("cmpxchg", "%[nval]", \ 226 + __percpu_arg([var])) \ 227 + "\n\tjnz 1b" \ 228 + : [oval] "=&a" (pxo_old__), \ 229 + [var] "+m" (_var) \ 230 + : [nval] __pcpu_reg_##size(, pxo_new__) \ 231 + : "memory"); \ 232 + (typeof(_var))(unsigned long) pxo_old__; \ 258 233 }) 259 234 260 235 /* ··· 329 354 #define this_cpu_or_1(pcp, val) percpu_to_op(1, volatile, "or", (pcp), val) 330 355 #define this_cpu_or_2(pcp, val) percpu_to_op(2, volatile, "or", (pcp), val) 331 356 #define this_cpu_or_4(pcp, val) percpu_to_op(4, volatile, "or", (pcp), val) 332 - #define this_cpu_xchg_1(pcp, nval) percpu_xchg_op(volatile, pcp, nval) 333 - #define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(volatile, pcp, nval) 334 - #define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(volatile, pcp, nval) 357 + #define this_cpu_xchg_1(pcp, nval) percpu_xchg_op(1, volatile, pcp, nval) 358 + #define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(2, volatile, pcp, nval) 359 + #define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(4, volatile, pcp, nval) 335 360 336 361 #define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(1, , pcp, val) 337 362 #define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(2, , pcp, val) ··· 384 409 #define this_cpu_and_8(pcp, val) percpu_to_op(8, volatile, "and", (pcp), val) 385 410 #define this_cpu_or_8(pcp, val) percpu_to_op(8, volatile, "or", (pcp), val) 386 411 #define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val) 387 - #define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(volatile, pcp, nval) 412 + #define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(8, volatile, pcp, nval) 388 413 #define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(volatile, pcp, oval, nval) 389 414 390 415 /*