tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

selftests/rseq: x86-32: use %gs segment selector for accessing rseq thread area

Rather than use rseq_get_abi() and pass its result through a register to
the inline assembler, directly access the per-thread rseq area through a
memory reference combining the %gs segment selector, the constant offset
of the field in struct rseq, and the rseq_offset value (in a register).

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20220124171253.22072-16-mathieu.desnoyers@efficios.com

authored by

Mathieu Desnoyers and committed by
Peter Zijlstra 127b6429 4e15bb76

+34 -32
+34 -32
tools/testing/selftests/rseq/rseq-x86.h
··· 633 633 634 634 #elif defined(__i386__) 635 635 636 + #define RSEQ_ASM_TP_SEGMENT %%gs 637 + 636 638 #define rseq_smp_mb() \ 637 639 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 638 640 #define rseq_smp_rmb() \ ··· 734 732 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 735 733 #endif 736 734 /* Start rseq by storing table entry pointer into rseq_cs. */ 737 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 738 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 735 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 736 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 739 737 RSEQ_INJECT_ASM(3) 740 738 "cmpl %[v], %[expect]\n\t" 741 739 "jnz %l[cmpfail]\n\t" 742 740 RSEQ_INJECT_ASM(4) 743 741 #ifdef RSEQ_COMPARE_TWICE 744 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 742 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 745 743 "cmpl %[v], %[expect]\n\t" 746 744 "jnz %l[error2]\n\t" 747 745 #endif ··· 752 750 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 753 751 : /* gcc asm goto does not allow outputs */ 754 752 : [cpu_id] "r" (cpu), 755 - [rseq_abi] "r" (rseq_get_abi()), 753 + [rseq_offset] "r" (rseq_offset), 756 754 [v] "m" (*v), 757 755 [expect] "r" (expect), 758 756 [newv] "r" (newv) ··· 800 798 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 801 799 #endif 802 800 /* Start rseq by storing table entry pointer into rseq_cs. */ 803 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 804 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 801 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 802 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 805 803 RSEQ_INJECT_ASM(3) 806 804 "movl %[v], %%ebx\n\t" 807 805 "cmpl %%ebx, %[expectnot]\n\t" 808 806 "je %l[cmpfail]\n\t" 809 807 RSEQ_INJECT_ASM(4) 810 808 #ifdef RSEQ_COMPARE_TWICE 811 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 809 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 812 810 "movl %[v], %%ebx\n\t" 813 811 "cmpl %%ebx, %[expectnot]\n\t" 814 812 "je %l[error2]\n\t" ··· 823 821 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 824 822 : /* gcc asm goto does not allow outputs */ 825 823 : [cpu_id] "r" (cpu), 826 - [rseq_abi] "r" (rseq_get_abi()), 824 + [rseq_offset] "r" (rseq_offset), 827 825 /* final store input */ 828 826 [v] "m" (*v), 829 827 [expectnot] "r" (expectnot), ··· 866 864 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 867 865 #endif 868 866 /* Start rseq by storing table entry pointer into rseq_cs. */ 869 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 870 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 867 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 868 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 871 869 RSEQ_INJECT_ASM(3) 872 870 #ifdef RSEQ_COMPARE_TWICE 873 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 871 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 874 872 #endif 875 873 /* final store */ 876 874 "addl %[count], %[v]\n\t" ··· 879 877 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 880 878 : /* gcc asm goto does not allow outputs */ 881 879 : [cpu_id] "r" (cpu), 882 - [rseq_abi] "r" (rseq_get_abi()), 880 + [rseq_offset] "r" (rseq_offset), 883 881 /* final store input */ 884 882 [v] "m" (*v), 885 883 [count] "ir" (count) ··· 918 916 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 919 917 #endif 920 918 /* Start rseq by storing table entry pointer into rseq_cs. */ 921 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 922 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 919 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 920 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 923 921 RSEQ_INJECT_ASM(3) 924 922 "cmpl %[v], %[expect]\n\t" 925 923 "jnz %l[cmpfail]\n\t" 926 924 RSEQ_INJECT_ASM(4) 927 925 #ifdef RSEQ_COMPARE_TWICE 928 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 926 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 929 927 "cmpl %[v], %[expect]\n\t" 930 928 "jnz %l[error2]\n\t" 931 929 #endif ··· 940 938 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 941 939 : /* gcc asm goto does not allow outputs */ 942 940 : [cpu_id] "r" (cpu), 943 - [rseq_abi] "r" (rseq_get_abi()), 941 + [rseq_offset] "r" (rseq_offset), 944 942 /* try store input */ 945 943 [v2] "m" (*v2), 946 944 [newv2] "m" (newv2), ··· 989 987 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 990 988 #endif 991 989 /* Start rseq by storing table entry pointer into rseq_cs. */ 992 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 993 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 990 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 991 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 994 992 RSEQ_INJECT_ASM(3) 995 993 "movl %[expect], %%eax\n\t" 996 994 "cmpl %[v], %%eax\n\t" 997 995 "jnz %l[cmpfail]\n\t" 998 996 RSEQ_INJECT_ASM(4) 999 997 #ifdef RSEQ_COMPARE_TWICE 1000 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 998 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 1001 999 "movl %[expect], %%eax\n\t" 1002 1000 "cmpl %[v], %%eax\n\t" 1003 1001 "jnz %l[error2]\n\t" ··· 1013 1011 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 1014 1012 : /* gcc asm goto does not allow outputs */ 1015 1013 : [cpu_id] "r" (cpu), 1016 - [rseq_abi] "r" (rseq_get_abi()), 1014 + [rseq_offset] "r" (rseq_offset), 1017 1015 /* try store input */ 1018 1016 [v2] "m" (*v2), 1019 1017 [newv2] "r" (newv2), ··· 1064 1062 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 1065 1063 #endif 1066 1064 /* Start rseq by storing table entry pointer into rseq_cs. */ 1067 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 1068 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 1065 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 1066 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 1069 1067 RSEQ_INJECT_ASM(3) 1070 1068 "cmpl %[v], %[expect]\n\t" 1071 1069 "jnz %l[cmpfail]\n\t" ··· 1074 1072 "jnz %l[cmpfail]\n\t" 1075 1073 RSEQ_INJECT_ASM(5) 1076 1074 #ifdef RSEQ_COMPARE_TWICE 1077 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) 1075 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 1078 1076 "cmpl %[v], %[expect]\n\t" 1079 1077 "jnz %l[error2]\n\t" 1080 1078 "cmpl %[expect2], %[v2]\n\t" ··· 1088 1086 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 1089 1087 : /* gcc asm goto does not allow outputs */ 1090 1088 : [cpu_id] "r" (cpu), 1091 - [rseq_abi] "r" (rseq_get_abi()), 1089 + [rseq_offset] "r" (rseq_offset), 1092 1090 /* cmp2 input */ 1093 1091 [v2] "m" (*v2), 1094 1092 [expect2] "r" (expect2), ··· 1146 1144 "movl %[dst], %[rseq_scratch1]\n\t" 1147 1145 "movl %[len], %[rseq_scratch2]\n\t" 1148 1146 /* Start rseq by storing table entry pointer into rseq_cs. */ 1149 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 1150 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 1147 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 1148 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 1151 1149 RSEQ_INJECT_ASM(3) 1152 1150 "movl %[expect], %%eax\n\t" 1153 1151 "cmpl %%eax, %[v]\n\t" 1154 1152 "jnz 5f\n\t" 1155 1153 RSEQ_INJECT_ASM(4) 1156 1154 #ifdef RSEQ_COMPARE_TWICE 1157 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) 1155 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) 1158 1156 "movl %[expect], %%eax\n\t" 1159 1157 "cmpl %%eax, %[v]\n\t" 1160 1158 "jnz 7f\n\t" ··· 1204 1202 #endif 1205 1203 : /* gcc asm goto does not allow outputs */ 1206 1204 : [cpu_id] "r" (cpu), 1207 - [rseq_abi] "r" (rseq_get_abi()), 1205 + [rseq_offset] "r" (rseq_offset), 1208 1206 /* final store input */ 1209 1207 [v] "m" (*v), 1210 1208 [expect] "m" (expect), ··· 1263 1261 "movl %[dst], %[rseq_scratch1]\n\t" 1264 1262 "movl %[len], %[rseq_scratch2]\n\t" 1265 1263 /* Start rseq by storing table entry pointer into rseq_cs. */ 1266 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 1267 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) 1264 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 1265 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) 1268 1266 RSEQ_INJECT_ASM(3) 1269 1267 "movl %[expect], %%eax\n\t" 1270 1268 "cmpl %%eax, %[v]\n\t" 1271 1269 "jnz 5f\n\t" 1272 1270 RSEQ_INJECT_ASM(4) 1273 1271 #ifdef RSEQ_COMPARE_TWICE 1274 - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) 1272 + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) 1275 1273 "movl %[expect], %%eax\n\t" 1276 1274 "cmpl %%eax, %[v]\n\t" 1277 1275 "jnz 7f\n\t" ··· 1322 1320 #endif 1323 1321 : /* gcc asm goto does not allow outputs */ 1324 1322 : [cpu_id] "r" (cpu), 1325 - [rseq_abi] "r" (rseq_get_abi()), 1323 + [rseq_offset] "r" (rseq_offset), 1326 1324 /* final store input */ 1327 1325 [v] "m" (*v), 1328 1326 [expect] "m" (expect),