tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
x86/mm: Reduce untagged_addr() overhead for systems without LAM
Use alternatives to reduce untagged_addr() overhead.
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: https://lore.kernel.org/all/20230312112612.31869-8-kirill.shutemov%40linux.intel.com
authored by
Kirill A. Shutemov
and committed by
Dave Hansen
e0bddc19
74c228d2
+37
-12
+7
-1
arch/x86/include/asm/disabled-features.h
+7
-1
arch/x86/include/asm/disabled-features.h
···
75
75
# define DISABLE_CALL_DEPTH_TRACKING (1 << (X86_FEATURE_CALL_DEPTH & 31))
76
76
#endif
77
77
78
+
#ifdef CONFIG_ADDRESS_MASKING
79
+
# define DISABLE_LAM 0
80
+
#else
81
+
# define DISABLE_LAM (1 << (X86_FEATURE_LAM & 31))
82
+
#endif
83
+
78
84
#ifdef CONFIG_INTEL_IOMMU_SVM
79
85
# define DISABLE_ENQCMD 0
80
86
#else
···
121
115
#define DISABLED_MASK10 0
122
116
#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \
123
117
DISABLE_CALL_DEPTH_TRACKING)
124
-
#define DISABLED_MASK12 0
118
+
#define DISABLED_MASK12 (DISABLE_LAM)
125
119
#define DISABLED_MASK13 0
126
120
#define DISABLED_MASK14 0
127
121
#define DISABLED_MASK15 0
+30
-11
arch/x86/include/asm/uaccess.h
+30
-11
arch/x86/include/asm/uaccess.h
···
9
9
#include <linux/kasan-checks.h>
10
10
#include <linux/mm_types.h>
11
11
#include <linux/string.h>
12
+
#include <linux/mmap_lock.h>
12
13
#include <asm/asm.h>
13
14
#include <asm/page.h>
14
15
#include <asm/smap.h>
···
31
30
* Magic with the 'sign' allows to untag userspace pointer without any branches
32
31
* while leaving kernel addresses intact.
33
32
*/
34
-
static inline unsigned long __untagged_addr(unsigned long addr,
35
-
unsigned long mask)
33
+
static inline unsigned long __untagged_addr(unsigned long addr)
36
34
{
37
-
long sign = addr >> 63;
35
+
long sign;
38
36
39
-
addr &= mask | sign;
37
+
/*
38
+
* Refer tlbstate_untag_mask directly to avoid RIP-relative relocation
39
+
* in alternative instructions. The relocation gets wrong when gets
40
+
* copied to the target place.
41
+
*/
42
+
asm (ALTERNATIVE("",
43
+
"sar $63, %[sign]\n\t" /* user_ptr ? 0 : -1UL */
44
+
"or %%gs:tlbstate_untag_mask, %[sign]\n\t"
45
+
"and %[sign], %[addr]\n\t", X86_FEATURE_LAM)
46
+
: [addr] "+r" (addr), [sign] "=r" (sign)
47
+
: "m" (tlbstate_untag_mask), "[sign]" (addr));
48
+
40
49
return addr;
41
50
}
42
51
43
52
#define untagged_addr(addr) ({ \
44
-
u64 __addr = (__force u64)(addr); \
45
-
__addr = __untagged_addr(__addr, current_untag_mask()); \
46
-
(__force __typeof__(addr))__addr; \
53
+
unsigned long __addr = (__force unsigned long)(addr); \
54
+
(__force __typeof__(addr))__untagged_addr(__addr); \
47
55
})
48
56
57
+
static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
58
+
unsigned long addr)
59
+
{
60
+
long sign = addr >> 63;
61
+
62
+
mmap_assert_locked(mm);
63
+
addr &= (mm)->context.untag_mask | sign;
64
+
65
+
return addr;
66
+
}
67
+
49
68
#define untagged_addr_remote(mm, addr) ({ \
50
-
u64 __addr = (__force u64)(addr); \
51
-
mmap_assert_locked(mm); \
52
-
__addr = __untagged_addr(__addr, (mm)->context.untag_mask); \
53
-
(__force __typeof__(addr))__addr; \
69
+
unsigned long __addr = (__force unsigned long)(addr); \
70
+
(__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \
54
71
})
55
72
56
73
#else