tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

x86, smap: Add STAC and CLAC instructions to control user space access

When Supervisor Mode Access Prevention (SMAP) is enabled, access to
userspace from the kernel is controlled by the AC flag. To make the
performance of manipulating that flag acceptable, there are two new
instructions, STAC and CLAC, to set and clear it.

This patch adds those instructions, via alternative(), when the SMAP
feature is enabled. It also adds X86_EFLAGS_AC unconditionally to the
SYSCALL entry mask; there is simply no reason to make that one
conditional.

Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Link: http://lkml.kernel.org/r/1348256595-29119-9-git-send-email-hpa@linux.intel.com

H. Peter Anvin 63bcff2a a052858f

+106 -32
+6
arch/x86/ia32/ia32entry.S
··· 14 14 #include <asm/segment.h> 15 15 #include <asm/irqflags.h> 16 16 #include <asm/asm.h> 17 + #include <asm/smap.h> 17 18 #include <linux/linkage.h> 18 19 #include <linux/err.h> 19 20 ··· 147 146 SAVE_ARGS 0,1,0 148 147 /* no need to do an access_ok check here because rbp has been 149 148 32bit zero extended */ 149 + ASM_STAC 150 150 1: movl (%rbp),%ebp 151 151 _ASM_EXTABLE(1b,ia32_badarg) 152 + ASM_CLAC 152 153 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 153 154 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 154 155 CFI_REMEMBER_STATE ··· 304 301 /* no need to do an access_ok check here because r8 has been 305 302 32bit zero extended */ 306 303 /* hardware stack frame is complete now */ 304 + ASM_STAC 307 305 1: movl (%r8),%r9d 308 306 _ASM_EXTABLE(1b,ia32_badarg) 307 + ASM_CLAC 309 308 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 310 309 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 311 310 CFI_REMEMBER_STATE ··· 370 365 END(ia32_cstar_target) 371 366 372 367 ia32_badarg: 368 + ASM_CLAC 373 369 movq $-EFAULT,%rax 374 370 jmp ia32_sysret 375 371 CFI_ENDPROC
+6 -4
arch/x86/include/asm/fpu-internal.h
··· 126 126 127 127 /* See comment in fxsave() below. */ 128 128 #ifdef CONFIG_AS_FXSAVEQ 129 - asm volatile("1: fxsaveq %[fx]\n\t" 130 - "2:\n" 129 + asm volatile(ASM_STAC "\n" 130 + "1: fxsaveq %[fx]\n\t" 131 + "2: " ASM_CLAC "\n" 131 132 ".section .fixup,\"ax\"\n" 132 133 "3: movl $-1,%[err]\n" 133 134 " jmp 2b\n" ··· 137 136 : [err] "=r" (err), [fx] "=m" (*fx) 138 137 : "0" (0)); 139 138 #else 140 - asm volatile("1: rex64/fxsave (%[fx])\n\t" 141 - "2:\n" 139 + asm volatile(ASM_STAC "\n" 140 + "1: rex64/fxsave (%[fx])\n\t" 141 + "2: " ASM_CLAC "\n" 142 142 ".section .fixup,\"ax\"\n" 143 143 "3: movl $-1,%[err]\n" 144 144 " jmp 2b\n"
+13 -6
arch/x86/include/asm/futex.h
··· 9 9 #include <asm/asm.h> 10 10 #include <asm/errno.h> 11 11 #include <asm/processor.h> 12 + #include <asm/smap.h> 12 13 13 14 #define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \ 14 - asm volatile("1:\t" insn "\n" \ 15 - "2:\t.section .fixup,\"ax\"\n" \ 15 + asm volatile("\t" ASM_STAC "\n" \ 16 + "1:\t" insn "\n" \ 17 + "2:\t" ASM_CLAC "\n" \ 18 + "\t.section .fixup,\"ax\"\n" \ 16 19 "3:\tmov\t%3, %1\n" \ 17 20 "\tjmp\t2b\n" \ 18 21 "\t.previous\n" \ ··· 24 21 : "i" (-EFAULT), "0" (oparg), "1" (0)) 25 22 26 23 #define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \ 27 - asm volatile("1:\tmovl %2, %0\n" \ 24 + asm volatile("\t" ASM_STAC "\n" \ 25 + "1:\tmovl %2, %0\n" \ 28 26 "\tmovl\t%0, %3\n" \ 29 27 "\t" insn "\n" \ 30 28 "2:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ 31 29 "\tjnz\t1b\n" \ 32 - "3:\t.section .fixup,\"ax\"\n" \ 30 + "3:\t" ASM_CLAC "\n" \ 31 + "\t.section .fixup,\"ax\"\n" \ 33 32 "4:\tmov\t%5, %1\n" \ 34 33 "\tjmp\t3b\n" \ 35 34 "\t.previous\n" \ ··· 127 122 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) 128 123 return -EFAULT; 129 124 130 - asm volatile("1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" 131 - "2:\t.section .fixup, \"ax\"\n" 125 + asm volatile("\t" ASM_STAC "\n" 126 + "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" 127 + "2:\t" ASM_CLAC "\n" 128 + "\t.section .fixup, \"ax\"\n" 132 129 "3:\tmov %3, %0\n" 133 130 "\tjmp 2b\n" 134 131 "\t.previous\n"
+2 -2
arch/x86/include/asm/smap.h
··· 58 58 59 59 #ifdef CONFIG_X86_SMAP 60 60 61 - static inline void clac(void) 61 + static __always_inline void clac(void) 62 62 { 63 63 /* Note: a barrier is implicit in alternative() */ 64 64 alternative(ASM_NOP3, __stringify(__ASM_CLAC), X86_FEATURE_SMAP); 65 65 } 66 66 67 - static inline void stac(void) 67 + static __always_inline void stac(void) 68 68 { 69 69 /* Note: a barrier is implicit in alternative() */ 70 70 alternative(ASM_NOP3, __stringify(__ASM_STAC), X86_FEATURE_SMAP);
+19 -12
arch/x86/include/asm/uaccess.h
··· 9 9 #include <linux/string.h> 10 10 #include <asm/asm.h> 11 11 #include <asm/page.h> 12 + #include <asm/smap.h> 12 13 13 14 #define VERIFY_READ 0 14 15 #define VERIFY_WRITE 1 ··· 193 192 194 193 #ifdef CONFIG_X86_32 195 194 #define __put_user_asm_u64(x, addr, err, errret) \ 196 - asm volatile("1: movl %%eax,0(%2)\n" \ 195 + asm volatile(ASM_STAC "\n" \ 196 + "1: movl %%eax,0(%2)\n" \ 197 197 "2: movl %%edx,4(%2)\n" \ 198 - "3:\n" \ 198 + "3: " ASM_CLAC "\n" \ 199 199 ".section .fixup,\"ax\"\n" \ 200 200 "4: movl %3,%0\n" \ 201 201 " jmp 3b\n" \ ··· 207 205 : "A" (x), "r" (addr), "i" (errret), "0" (err)) 208 206 209 207 #define __put_user_asm_ex_u64(x, addr) \ 210 - asm volatile("1: movl %%eax,0(%1)\n" \ 208 + asm volatile(ASM_STAC "\n" \ 209 + "1: movl %%eax,0(%1)\n" \ 211 210 "2: movl %%edx,4(%1)\n" \ 212 - "3:\n" \ 211 + "3: " ASM_CLAC "\n" \ 213 212 _ASM_EXTABLE_EX(1b, 2b) \ 214 213 _ASM_EXTABLE_EX(2b, 3b) \ 215 214 : : "A" (x), "r" (addr)) ··· 382 379 } while (0) 383 380 384 381 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ 385 - asm volatile("1: mov"itype" %2,%"rtype"1\n" \ 386 - "2:\n" \ 382 + asm volatile(ASM_STAC "\n" \ 383 + "1: mov"itype" %2,%"rtype"1\n" \ 384 + "2: " ASM_CLAC "\n" \ 387 385 ".section .fixup,\"ax\"\n" \ 388 386 "3: mov %3,%0\n" \ 389 387 " xor"itype" %"rtype"1,%"rtype"1\n" \ ··· 416 412 } while (0) 417 413 418 414 #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \ 419 - asm volatile("1: mov"itype" %1,%"rtype"0\n" \ 420 - "2:\n" \ 415 + asm volatile(ASM_STAC "\n" \ 416 + "1: mov"itype" %1,%"rtype"0\n" \ 417 + "2: " ASM_CLAC "\n" \ 421 418 _ASM_EXTABLE_EX(1b, 2b) \ 422 419 : ltype(x) : "m" (__m(addr))) 423 420 ··· 448 443 * aliasing issues. 449 444 */ 450 445 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ 451 - asm volatile("1: mov"itype" %"rtype"1,%2\n" \ 452 - "2:\n" \ 446 + asm volatile(ASM_STAC "\n" \ 447 + "1: mov"itype" %"rtype"1,%2\n" \ 448 + "2: " ASM_CLAC "\n" \ 453 449 ".section .fixup,\"ax\"\n" \ 454 450 "3: mov %3,%0\n" \ 455 451 " jmp 2b\n" \ ··· 460 454 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) 461 455 462 456 #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \ 463 - asm volatile("1: mov"itype" %"rtype"0,%1\n" \ 464 - "2:\n" \ 457 + asm volatile(ASM_STAC "\n" \ 458 + "1: mov"itype" %"rtype"0,%1\n" \ 459 + "2: " ASM_CLAC "\n" \ 465 460 _ASM_EXTABLE_EX(1b, 2b) \ 466 461 : : ltype(x), "m" (__m(addr))) 467 462
+6 -4
arch/x86/include/asm/xsave.h
··· 74 74 if (unlikely(err)) 75 75 return -EFAULT; 76 76 77 - __asm__ __volatile__("1: .byte " REX_PREFIX "0x0f,0xae,0x27\n" 78 - "2:\n" 77 + __asm__ __volatile__(ASM_STAC "\n" 78 + "1: .byte " REX_PREFIX "0x0f,0xae,0x27\n" 79 + "2: " ASM_CLAC "\n" 79 80 ".section .fixup,\"ax\"\n" 80 81 "3: movl $-1,%[err]\n" 81 82 " jmp 2b\n" ··· 98 97 u32 lmask = mask; 99 98 u32 hmask = mask >> 32; 100 99 101 - __asm__ __volatile__("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n" 102 - "2:\n" 100 + __asm__ __volatile__(ASM_STAC "\n" 101 + "1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n" 102 + "2: " ASM_CLAC "\n" 103 103 ".section .fixup,\"ax\"\n" 104 104 "3: movl $-1,%[err]\n" 105 105 " jmp 2b\n"
+2 -1
arch/x86/kernel/cpu/common.c
··· 1113 1113 1114 1114 /* Flags to clear on syscall */ 1115 1115 wrmsrl(MSR_SYSCALL_MASK, 1116 - X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL); 1116 + X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF| 1117 + X86_EFLAGS_IOPL|X86_EFLAGS_AC); 1117 1118 } 1118 1119 1119 1120 unsigned long kernel_eflags;
+10 -1
arch/x86/kernel/entry_64.S
··· 56 56 #include <asm/ftrace.h> 57 57 #include <asm/percpu.h> 58 58 #include <asm/asm.h> 59 + #include <asm/smap.h> 59 60 #include <linux/err.h> 60 61 61 62 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ ··· 466 465 * System call entry. Up to 6 arguments in registers are supported. 467 466 * 468 467 * SYSCALL does not save anything on the stack and does not change the 469 - * stack pointer. 468 + * stack pointer. However, it does mask the flags register for us, so 469 + * CLD and CLAC are not needed. 470 470 */ 471 471 472 472 /* ··· 886 884 */ 887 885 .p2align CONFIG_X86_L1_CACHE_SHIFT 888 886 common_interrupt: 887 + ASM_CLAC 889 888 XCPT_FRAME 890 889 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */ 891 890 interrupt do_IRQ ··· 1026 1023 */ 1027 1024 .macro apicinterrupt num sym do_sym 1028 1025 ENTRY(\sym) 1026 + ASM_CLAC 1029 1027 INTR_FRAME 1030 1028 pushq_cfi $~(\num) 1031 1029 .Lcommon_\sym: ··· 1081 1077 */ 1082 1078 .macro zeroentry sym do_sym 1083 1079 ENTRY(\sym) 1080 + ASM_CLAC 1084 1081 INTR_FRAME 1085 1082 PARAVIRT_ADJUST_EXCEPTION_FRAME 1086 1083 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ ··· 1099 1094 1100 1095 .macro paranoidzeroentry sym do_sym 1101 1096 ENTRY(\sym) 1097 + ASM_CLAC 1102 1098 INTR_FRAME 1103 1099 PARAVIRT_ADJUST_EXCEPTION_FRAME 1104 1100 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ ··· 1118 1112 #define INIT_TSS_IST(x) PER_CPU_VAR(init_tss) + (TSS_ist + ((x) - 1) * 8) 1119 1113 .macro paranoidzeroentry_ist sym do_sym ist 1120 1114 ENTRY(\sym) 1115 + ASM_CLAC 1121 1116 INTR_FRAME 1122 1117 PARAVIRT_ADJUST_EXCEPTION_FRAME 1123 1118 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ ··· 1138 1131 1139 1132 .macro errorentry sym do_sym 1140 1133 ENTRY(\sym) 1134 + ASM_CLAC 1141 1135 XCPT_FRAME 1142 1136 PARAVIRT_ADJUST_EXCEPTION_FRAME 1143 1137 subq $ORIG_RAX-R15, %rsp ··· 1157 1149 /* error code is on the stack already */ 1158 1150 .macro paranoiderrorentry sym do_sym 1159 1151 ENTRY(\sym) 1152 + ASM_CLAC 1160 1153 XCPT_FRAME 1161 1154 PARAVIRT_ADJUST_EXCEPTION_FRAME 1162 1155 subq $ORIG_RAX-R15, %rsp
+7
arch/x86/lib/copy_user_64.S
··· 17 17 #include <asm/cpufeature.h> 18 18 #include <asm/alternative-asm.h> 19 19 #include <asm/asm.h> 20 + #include <asm/smap.h> 20 21 21 22 /* 22 23 * By placing feature2 after feature1 in altinstructions section, we logically ··· 131 130 */ 132 131 ENTRY(copy_user_generic_unrolled) 133 132 CFI_STARTPROC 133 + ASM_STAC 134 134 cmpl $8,%edx 135 135 jb 20f /* less then 8 bytes, go to byte copy loop */ 136 136 ALIGN_DESTINATION ··· 179 177 decl %ecx 180 178 jnz 21b 181 179 23: xor %eax,%eax 180 + ASM_CLAC 182 181 ret 183 182 184 183 .section .fixup,"ax" ··· 235 232 */ 236 233 ENTRY(copy_user_generic_string) 237 234 CFI_STARTPROC 235 + ASM_STAC 238 236 andl %edx,%edx 239 237 jz 4f 240 238 cmpl $8,%edx ··· 250 246 3: rep 251 247 movsb 252 248 4: xorl %eax,%eax 249 + ASM_CLAC 253 250 ret 254 251 255 252 .section .fixup,"ax" ··· 278 273 */ 279 274 ENTRY(copy_user_enhanced_fast_string) 280 275 CFI_STARTPROC 276 + ASM_STAC 281 277 andl %edx,%edx 282 278 jz 2f 283 279 movl %edx,%ecx 284 280 1: rep 285 281 movsb 286 282 2: xorl %eax,%eax 283 + ASM_CLAC 287 284 ret 288 285 289 286 .section .fixup,"ax"
+3
arch/x86/lib/copy_user_nocache_64.S
··· 15 15 #include <asm/asm-offsets.h> 16 16 #include <asm/thread_info.h> 17 17 #include <asm/asm.h> 18 + #include <asm/smap.h> 18 19 19 20 .macro ALIGN_DESTINATION 20 21 #ifdef FIX_ALIGNMENT ··· 49 48 */ 50 49 ENTRY(__copy_user_nocache) 51 50 CFI_STARTPROC 51 + ASM_STAC 52 52 cmpl $8,%edx 53 53 jb 20f /* less then 8 bytes, go to byte copy loop */ 54 54 ALIGN_DESTINATION ··· 97 95 decl %ecx 98 96 jnz 21b 99 97 23: xorl %eax,%eax 98 + ASM_CLAC 100 99 sfence 101 100 ret 102 101
+10
arch/x86/lib/getuser.S
··· 33 33 #include <asm/asm-offsets.h> 34 34 #include <asm/thread_info.h> 35 35 #include <asm/asm.h> 36 + #include <asm/smap.h> 36 37 37 38 .text 38 39 ENTRY(__get_user_1) ··· 41 40 GET_THREAD_INFO(%_ASM_DX) 42 41 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX 43 42 jae bad_get_user 43 + ASM_STAC 44 44 1: movzb (%_ASM_AX),%edx 45 45 xor %eax,%eax 46 + ASM_CLAC 46 47 ret 47 48 CFI_ENDPROC 48 49 ENDPROC(__get_user_1) ··· 56 53 GET_THREAD_INFO(%_ASM_DX) 57 54 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX 58 55 jae bad_get_user 56 + ASM_STAC 59 57 2: movzwl -1(%_ASM_AX),%edx 60 58 xor %eax,%eax 59 + ASM_CLAC 61 60 ret 62 61 CFI_ENDPROC 63 62 ENDPROC(__get_user_2) ··· 71 66 GET_THREAD_INFO(%_ASM_DX) 72 67 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX 73 68 jae bad_get_user 69 + ASM_STAC 74 70 3: mov -3(%_ASM_AX),%edx 75 71 xor %eax,%eax 72 + ASM_CLAC 76 73 ret 77 74 CFI_ENDPROC 78 75 ENDPROC(__get_user_4) ··· 87 80 GET_THREAD_INFO(%_ASM_DX) 88 81 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX 89 82 jae bad_get_user 83 + ASM_STAC 90 84 4: movq -7(%_ASM_AX),%_ASM_DX 91 85 xor %eax,%eax 86 + ASM_CLAC 92 87 ret 93 88 CFI_ENDPROC 94 89 ENDPROC(__get_user_8) ··· 100 91 CFI_STARTPROC 101 92 xor %edx,%edx 102 93 mov $(-EFAULT),%_ASM_AX 94 + ASM_CLAC 103 95 ret 104 96 CFI_ENDPROC 105 97 END(bad_get_user)
+7 -1
arch/x86/lib/putuser.S
··· 15 15 #include <asm/thread_info.h> 16 16 #include <asm/errno.h> 17 17 #include <asm/asm.h> 18 + #include <asm/smap.h> 18 19 19 20 20 21 /* ··· 32 31 33 32 #define ENTER CFI_STARTPROC ; \ 34 33 GET_THREAD_INFO(%_ASM_BX) 35 - #define EXIT ret ; \ 34 + #define EXIT ASM_CLAC ; \ 35 + ret ; \ 36 36 CFI_ENDPROC 37 37 38 38 .text ··· 41 39 ENTER 42 40 cmp TI_addr_limit(%_ASM_BX),%_ASM_CX 43 41 jae bad_put_user 42 + ASM_STAC 44 43 1: movb %al,(%_ASM_CX) 45 44 xor %eax,%eax 46 45 EXIT ··· 53 50 sub $1,%_ASM_BX 54 51 cmp %_ASM_BX,%_ASM_CX 55 52 jae bad_put_user 53 + ASM_STAC 56 54 2: movw %ax,(%_ASM_CX) 57 55 xor %eax,%eax 58 56 EXIT ··· 65 61 sub $3,%_ASM_BX 66 62 cmp %_ASM_BX,%_ASM_CX 67 63 jae bad_put_user 64 + ASM_STAC 68 65 3: movl %eax,(%_ASM_CX) 69 66 xor %eax,%eax 70 67 EXIT ··· 77 72 sub $7,%_ASM_BX 78 73 cmp %_ASM_BX,%_ASM_CX 79 74 jae bad_put_user 75 + ASM_STAC 80 76 4: mov %_ASM_AX,(%_ASM_CX) 81 77 #ifdef CONFIG_X86_32 82 78 5: movl %edx,4(%_ASM_CX)
+12 -1
arch/x86/lib/usercopy_32.c
··· 42 42 int __d0; \ 43 43 might_fault(); \ 44 44 __asm__ __volatile__( \ 45 + ASM_STAC "\n" \ 45 46 "0: rep; stosl\n" \ 46 47 " movl %2,%0\n" \ 47 48 "1: rep; stosb\n" \ 48 - "2:\n" \ 49 + "2: " ASM_CLAC "\n" \ 49 50 ".section .fixup,\"ax\"\n" \ 50 51 "3: lea 0(%2,%0,4),%0\n" \ 51 52 " jmp 2b\n" \ ··· 627 626 return n; 628 627 } 629 628 #endif 629 + stac(); 630 630 if (movsl_is_ok(to, from, n)) 631 631 __copy_user(to, from, n); 632 632 else 633 633 n = __copy_user_intel(to, from, n); 634 + clac(); 634 635 return n; 635 636 } 636 637 EXPORT_SYMBOL(__copy_to_user_ll); ··· 640 637 unsigned long __copy_from_user_ll(void *to, const void __user *from, 641 638 unsigned long n) 642 639 { 640 + stac(); 643 641 if (movsl_is_ok(to, from, n)) 644 642 __copy_user_zeroing(to, from, n); 645 643 else 646 644 n = __copy_user_zeroing_intel(to, from, n); 645 + clac(); 647 646 return n; 648 647 } 649 648 EXPORT_SYMBOL(__copy_from_user_ll); ··· 653 648 unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from, 654 649 unsigned long n) 655 650 { 651 + stac(); 656 652 if (movsl_is_ok(to, from, n)) 657 653 __copy_user(to, from, n); 658 654 else 659 655 n = __copy_user_intel((void __user *)to, 660 656 (const void *)from, n); 657 + clac(); 661 658 return n; 662 659 } 663 660 EXPORT_SYMBOL(__copy_from_user_ll_nozero); ··· 667 660 unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from, 668 661 unsigned long n) 669 662 { 663 + stac(); 670 664 #ifdef CONFIG_X86_INTEL_USERCOPY 671 665 if (n > 64 && cpu_has_xmm2) 672 666 n = __copy_user_zeroing_intel_nocache(to, from, n); ··· 676 668 #else 677 669 __copy_user_zeroing(to, from, n); 678 670 #endif 671 + clac(); 679 672 return n; 680 673 } 681 674 EXPORT_SYMBOL(__copy_from_user_ll_nocache); ··· 684 675 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, 685 676 unsigned long n) 686 677 { 678 + stac(); 687 679 #ifdef CONFIG_X86_INTEL_USERCOPY 688 680 if (n > 64 && cpu_has_xmm2) 689 681 n = __copy_user_intel_nocache(to, from, n); ··· 693 683 #else 694 684 __copy_user(to, from, n); 695 685 #endif 686 + clac(); 696 687 return n; 697 688 } 698 689 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
+3
arch/x86/lib/usercopy_64.c
··· 18 18 might_fault(); 19 19 /* no memory constraint because it doesn't change any memory gcc knows 20 20 about */ 21 + stac(); 21 22 asm volatile( 22 23 " testq %[size8],%[size8]\n" 23 24 " jz 4f\n" ··· 41 40 : [size8] "=&c"(size), [dst] "=&D" (__d0) 42 41 : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr), 43 42 [zero] "r" (0UL), [eight] "r" (8UL)); 43 + clac(); 44 44 return size; 45 45 } 46 46 EXPORT_SYMBOL(__clear_user); ··· 84 82 for (c = 0, zero_len = len; zerorest && zero_len; --zero_len) 85 83 if (__put_user_nocheck(c, to++, sizeof(char))) 86 84 break; 85 + clac(); 87 86 return len; 88 87 }