Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

x86,vdso: Use LSL unconditionally for vgetcpu

LSL is faster than RDTSCP and works everywhere; there's no need to
switch between them depending on CPU.

Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Andi Kleen <andi@firstfloor.org>
Link: http://lkml.kernel.org/r/72f73d5ec4514e02bba345b9759177ef03742efb.1414706021.git.luto@amacapital.net
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

authored by

Andy Lutomirski and committed by
Thomas Gleixner
e76b027e a92f101b

+19 -43
+19
arch/x86/include/asm/vgtod.h
··· 70 70 ++s->seq; 71 71 } 72 72 73 + #ifdef CONFIG_X86_64 74 + 75 + #define VGETCPU_CPU_MASK 0xfff 76 + 77 + static inline unsigned int __getcpu(void) 78 + { 79 + unsigned int p; 80 + 81 + /* 82 + * Load per CPU data from GDT. LSL is faster than RDTSCP and 83 + * works on all CPUs. 84 + */ 85 + asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG)); 86 + 87 + return p; 88 + } 89 + 90 + #endif /* CONFIG_X86_64 */ 91 + 73 92 #endif /* _ASM_X86_VGTOD_H */
-29
arch/x86/include/asm/vsyscall.h
··· 4 4 #include <linux/seqlock.h> 5 5 #include <uapi/asm/vsyscall.h> 6 6 7 - #define VGETCPU_RDTSCP 1 8 - #define VGETCPU_LSL 2 9 - 10 - /* kernel space (writeable) */ 11 - extern int vgetcpu_mode; 12 - extern struct timezone sys_tz; 13 - 14 - #include <asm/vvar.h> 15 - 16 7 extern void map_vsyscall(void); 17 8 18 9 /* ··· 11 20 * Returns true if handled. 12 21 */ 13 22 extern bool emulate_vsyscall(struct pt_regs *regs, unsigned long address); 14 - 15 - #ifdef CONFIG_X86_64 16 - 17 - #define VGETCPU_CPU_MASK 0xfff 18 - 19 - static inline unsigned int __getcpu(void) 20 - { 21 - unsigned int p; 22 - 23 - if (VVAR(vgetcpu_mode) == VGETCPU_RDTSCP) { 24 - /* Load per CPU data from RDTSCP */ 25 - native_read_tscp(&p); 26 - } else { 27 - /* Load per CPU data from GDT */ 28 - asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG)); 29 - } 30 - 31 - return p; 32 - } 33 - #endif /* CONFIG_X86_64 */ 34 23 35 24 #endif /* _ASM_X86_VSYSCALL_H */
-1
arch/x86/include/asm/vvar.h
··· 44 44 45 45 /* DECLARE_VVAR(offset, type, name) */ 46 46 47 - DECLARE_VVAR(16, int, vgetcpu_mode) 48 47 DECLARE_VVAR(128, struct vsyscall_gtod_data, vsyscall_gtod_data) 49 48 50 49 #undef DECLARE_VVAR
-10
arch/x86/kernel/cpu/common.c
··· 956 956 } 957 957 958 958 #ifdef CONFIG_X86_64 959 - static void vgetcpu_set_mode(void) 960 - { 961 - if (cpu_has(&boot_cpu_data, X86_FEATURE_RDTSCP)) 962 - vgetcpu_mode = VGETCPU_RDTSCP; 963 - else 964 - vgetcpu_mode = VGETCPU_LSL; 965 - } 966 - 967 959 #ifdef CONFIG_IA32_EMULATION 968 960 /* May not be __init: called during resume */ 969 961 static void syscall32_cpu_init(void) ··· 998 1006 #ifdef CONFIG_X86_32 999 1007 sysenter_setup(); 1000 1008 enable_sep_cpu(); 1001 - #else 1002 - vgetcpu_set_mode(); 1003 1009 #endif 1004 1010 cpu_detect_tlb(&boot_cpu_data); 1005 1011 }
-2
arch/x86/kernel/vsyscall_64.c
··· 52 52 #define CREATE_TRACE_POINTS 53 53 #include "vsyscall_trace.h" 54 54 55 - DEFINE_VVAR(int, vgetcpu_mode); 56 - 57 55 static enum { EMULATE, NATIVE, NONE } vsyscall_mode = EMULATE; 58 56 59 57 static int __init vsyscall_setup(char *str)
-1
arch/x86/vdso/vgetcpu.c
··· 8 8 #include <linux/kernel.h> 9 9 #include <linux/getcpu.h> 10 10 #include <linux/time.h> 11 - #include <asm/vsyscall.h> 12 11 #include <asm/vgtod.h> 13 12 14 13 notrace long