tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

rseq/selftests: Add __rseq_exit_point_array section for debuggers

Knowing all exit points is useful to assist debuggers stepping over the
rseq critical sections without requiring them to disassemble the content
of the critical section to figure out the exit points.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
CC: Thomas Gleixner <tglx@linutronix.de>
CC: Joel Fernandes <joelaf@google.com>
CC: Peter Zijlstra <peterz@infradead.org>
CC: Catalin Marinas <catalin.marinas@arm.com>
CC: Dave Watson <davejwatson@fb.com>
CC: Will Deacon <will.deacon@arm.com>
CC: Shuah Khan <shuah@kernel.org>
CC: Andi Kleen <andi@firstfloor.org>
CC: linux-kselftest@vger.kernel.org
CC: "H . Peter Anvin" <hpa@zytor.com>
CC: Chris Lameter <cl@linux.com>
CC: Russell King <linux@arm.linux.org.uk>
CC: Michael Kerrisk <mtk.manpages@gmail.com>
CC: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com>
CC: Paul Turner <pjt@google.com>
CC: Boqun Feng <boqun.feng@gmail.com>
CC: Josh Triplett <josh@joshtriplett.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ben Maurer <bmaurer@fb.com>
CC: linux-api@vger.kernel.org
CC: Andy Lutomirski <luto@amacapital.net>
CC: Andrew Morton <akpm@linux-foundation.org>
CC: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>

authored by

Mathieu Desnoyers and committed by
Shuah Khan 4fe2088e fe22983d

+372
+52
tools/testing/selftests/rseq/rseq-arm.h
··· 42 42 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ 43 43 (post_commit_ip - start_ip), abort_ip) 44 44 45 + /* 46 + * Exit points of a rseq critical section consist of all instructions outside 47 + * of the critical section where a critical section can either branch to or 48 + * reach through the normal course of its execution. The abort IP and the 49 + * post-commit IP are already part of the __rseq_table section and should not 50 + * be explicitly defined as additional exit points. Knowing all exit points is 51 + * useful to assist debuggers stepping over the critical section. 52 + */ 53 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 54 + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 55 + ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 56 + ".popsection\n\t" 57 + 45 58 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 46 59 RSEQ_INJECT_ASM(1) \ 47 60 "adr r0, " __rseq_str(cs_label) "\n\t" \ ··· 100 87 rseq_workaround_gcc_asm_size_guess(); 101 88 __asm__ __volatile__ goto ( 102 89 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 90 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 91 + #ifdef RSEQ_COMPARE_TWICE 92 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 93 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 94 + #endif 103 95 /* Start rseq by storing table entry pointer into rseq_cs. */ 104 96 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 105 97 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 167 149 rseq_workaround_gcc_asm_size_guess(); 168 150 __asm__ __volatile__ goto ( 169 151 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 152 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 153 + #ifdef RSEQ_COMPARE_TWICE 154 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 155 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 156 + #endif 170 157 /* Start rseq by storing table entry pointer into rseq_cs. */ 171 158 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 172 159 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 238 215 rseq_workaround_gcc_asm_size_guess(); 239 216 __asm__ __volatile__ goto ( 240 217 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 218 + #ifdef RSEQ_COMPARE_TWICE 219 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 220 + #endif 241 221 /* Start rseq by storing table entry pointer into rseq_cs. */ 242 222 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 243 223 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 293 267 rseq_workaround_gcc_asm_size_guess(); 294 268 __asm__ __volatile__ goto ( 295 269 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 270 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 271 + #ifdef RSEQ_COMPARE_TWICE 272 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 273 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 274 + #endif 296 275 /* Start rseq by storing table entry pointer into rseq_cs. */ 297 276 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 298 277 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 368 337 rseq_workaround_gcc_asm_size_guess(); 369 338 __asm__ __volatile__ goto ( 370 339 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 340 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 341 + #ifdef RSEQ_COMPARE_TWICE 342 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 343 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 344 + #endif 371 345 /* Start rseq by storing table entry pointer into rseq_cs. */ 372 346 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 373 347 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 444 408 rseq_workaround_gcc_asm_size_guess(); 445 409 __asm__ __volatile__ goto ( 446 410 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 411 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 412 + #ifdef RSEQ_COMPARE_TWICE 413 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 414 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 415 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 416 + #endif 447 417 /* Start rseq by storing table entry pointer into rseq_cs. */ 448 418 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 449 419 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 528 486 rseq_workaround_gcc_asm_size_guess(); 529 487 __asm__ __volatile__ goto ( 530 488 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 489 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 490 + #ifdef RSEQ_COMPARE_TWICE 491 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 492 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 493 + #endif 531 494 "str %[src], %[rseq_scratch0]\n\t" 532 495 "str %[dst], %[rseq_scratch1]\n\t" 533 496 "str %[len], %[rseq_scratch2]\n\t" ··· 652 605 rseq_workaround_gcc_asm_size_guess(); 653 606 __asm__ __volatile__ goto ( 654 607 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 608 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 609 + #ifdef RSEQ_COMPARE_TWICE 610 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 611 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 612 + #endif 655 613 "str %[src], %[rseq_scratch0]\n\t" 656 614 "str %[dst], %[rseq_scratch1]\n\t" 657 615 "str %[len], %[rseq_scratch2]\n\t"
+52
tools/testing/selftests/rseq/rseq-arm64.h
··· 95 95 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 96 96 (post_commit_ip - start_ip), abort_ip) 97 97 98 + /* 99 + * Exit points of a rseq critical section consist of all instructions outside 100 + * of the critical section where a critical section can either branch to or 101 + * reach through the normal course of its execution. The abort IP and the 102 + * post-commit IP are already part of the __rseq_table section and should not 103 + * be explicitly defined as additional exit points. Knowing all exit points is 104 + * useful to assist debuggers stepping over the critical section. 105 + */ 106 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 107 + " .pushsection __rseq_exit_point_array, \"aw\"\n" \ 108 + " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ 109 + " .popsection\n" 110 + 98 111 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 99 112 RSEQ_INJECT_ASM(1) \ 100 113 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \ ··· 195 182 196 183 __asm__ __volatile__ goto ( 197 184 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 185 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 186 + #ifdef RSEQ_COMPARE_TWICE 187 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 188 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 189 + #endif 198 190 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 199 191 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 200 192 RSEQ_INJECT_ASM(3) ··· 249 231 250 232 __asm__ __volatile__ goto ( 251 233 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 234 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 235 + #ifdef RSEQ_COMPARE_TWICE 236 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 237 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 238 + #endif 252 239 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 253 240 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 254 241 RSEQ_INJECT_ASM(3) ··· 305 282 306 283 __asm__ __volatile__ goto ( 307 284 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 285 + #ifdef RSEQ_COMPARE_TWICE 286 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 287 + #endif 308 288 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 309 289 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 310 290 RSEQ_INJECT_ASM(3) ··· 351 325 352 326 __asm__ __volatile__ goto ( 353 327 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 328 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 329 + #ifdef RSEQ_COMPARE_TWICE 330 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 331 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 332 + #endif 354 333 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 355 334 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 356 335 RSEQ_INJECT_ASM(3) ··· 410 379 411 380 __asm__ __volatile__ goto ( 412 381 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 382 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 383 + #ifdef RSEQ_COMPARE_TWICE 384 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 385 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 386 + #endif 413 387 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 414 388 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 415 389 RSEQ_INJECT_ASM(3) ··· 469 433 470 434 __asm__ __volatile__ goto ( 471 435 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 436 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 437 + #ifdef RSEQ_COMPARE_TWICE 438 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 439 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 440 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) 441 + #endif 472 442 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 473 443 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 474 444 RSEQ_INJECT_ASM(3) ··· 532 490 533 491 __asm__ __volatile__ goto ( 534 492 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 493 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 494 + #ifdef RSEQ_COMPARE_TWICE 495 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 496 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 497 + #endif 535 498 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 536 499 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 537 500 RSEQ_INJECT_ASM(3) ··· 592 545 593 546 __asm__ __volatile__ goto ( 594 547 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 548 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 549 + #ifdef RSEQ_COMPARE_TWICE 550 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 551 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 552 + #endif 595 553 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 596 554 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 597 555 RSEQ_INJECT_ASM(3)
+53
tools/testing/selftests/rseq/rseq-mips.h
··· 68 68 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ 69 69 (post_commit_ip - start_ip), abort_ip) 70 70 71 + /* 72 + * Exit points of a rseq critical section consist of all instructions outside 73 + * of the critical section where a critical section can either branch to or 74 + * reach through the normal course of its execution. The abort IP and the 75 + * post-commit IP are already part of the __rseq_table section and should not 76 + * be explicitly defined as additional exit points. Knowing all exit points is 77 + * useful to assist debuggers stepping over the critical section. 78 + */ 79 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 80 + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 81 + LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 82 + LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \ 83 + ".popsection\n\t" 84 + 71 85 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 72 86 RSEQ_INJECT_ASM(1) \ 73 87 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \ ··· 128 114 rseq_workaround_gcc_asm_size_guess(); 129 115 __asm__ __volatile__ goto ( 130 116 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 117 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 118 + #ifdef RSEQ_COMPARE_TWICE 119 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 120 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 121 + #endif 131 122 /* Start rseq by storing table entry pointer into rseq_cs. */ 132 123 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 133 124 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 193 174 rseq_workaround_gcc_asm_size_guess(); 194 175 __asm__ __volatile__ goto ( 195 176 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 177 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 178 + #ifdef RSEQ_COMPARE_TWICE 179 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 180 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 181 + #endif 196 182 /* Start rseq by storing table entry pointer into rseq_cs. */ 197 183 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 198 184 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 262 238 rseq_workaround_gcc_asm_size_guess(); 263 239 __asm__ __volatile__ goto ( 264 240 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 241 + #ifdef RSEQ_COMPARE_TWICE 242 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 243 + #endif 265 244 /* Start rseq by storing table entry pointer into rseq_cs. */ 266 245 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 267 246 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 317 290 rseq_workaround_gcc_asm_size_guess(); 318 291 __asm__ __volatile__ goto ( 319 292 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 293 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 294 + #ifdef RSEQ_COMPARE_TWICE 295 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 296 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 297 + #endif 320 298 /* Start rseq by storing table entry pointer into rseq_cs. */ 321 299 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 322 300 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 390 358 rseq_workaround_gcc_asm_size_guess(); 391 359 __asm__ __volatile__ goto ( 392 360 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 361 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 362 + #ifdef RSEQ_COMPARE_TWICE 363 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 364 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 365 + #endif 393 366 /* Start rseq by storing table entry pointer into rseq_cs. */ 394 367 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 395 368 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 464 427 rseq_workaround_gcc_asm_size_guess(); 465 428 __asm__ __volatile__ goto ( 466 429 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 430 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 431 + #ifdef RSEQ_COMPARE_TWICE 432 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 433 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 434 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 435 + #endif 467 436 /* Start rseq by storing table entry pointer into rseq_cs. */ 468 437 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 469 438 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 544 501 rseq_workaround_gcc_asm_size_guess(); 545 502 __asm__ __volatile__ goto ( 546 503 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 504 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 505 + #ifdef RSEQ_COMPARE_TWICE 506 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 507 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 508 + #endif 547 509 LONG_S " %[src], %[rseq_scratch0]\n\t" 548 510 LONG_S " %[dst], %[rseq_scratch1]\n\t" 549 511 LONG_S " %[len], %[rseq_scratch2]\n\t" ··· 665 617 rseq_workaround_gcc_asm_size_guess(); 666 618 __asm__ __volatile__ goto ( 667 619 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 620 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 621 + #ifdef RSEQ_COMPARE_TWICE 622 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 623 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 624 + #endif 668 625 LONG_S " %[src], %[rseq_scratch0]\n\t" 669 626 LONG_S " %[dst], %[rseq_scratch1]\n\t" 670 627 LONG_S " %[len], %[rseq_scratch2]\n\t"
+66
tools/testing/selftests/rseq/rseq-ppc.h
··· 63 63 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ 64 64 __rseq_str(label) ":\n\t" 65 65 66 + /* 67 + * Exit points of a rseq critical section consist of all instructions outside 68 + * of the critical section where a critical section can either branch to or 69 + * reach through the normal course of its execution. The abort IP and the 70 + * post-commit IP are already part of the __rseq_table section and should not 71 + * be explicitly defined as additional exit points. Knowing all exit points is 72 + * useful to assist debuggers stepping over the critical section. 73 + */ 74 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 75 + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 76 + ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 77 + ".popsection\n\t" 78 + 66 79 #else /* #ifdef __PPC64__ */ 67 80 68 81 #define STORE_WORD "stw " ··· 91 78 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 92 79 /* 32-bit only supported on BE */ \ 93 80 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 81 + ".popsection\n\t" 82 + 83 + /* 84 + * Exit points of a rseq critical section consist of all instructions outside 85 + * of the critical section where a critical section can either branch to or 86 + * reach through the normal course of its execution. The abort IP and the 87 + * post-commit IP are already part of the __rseq_table section and should not 88 + * be explicitly defined as additional exit points. Knowing all exit points is 89 + * useful to assist debuggers stepping over the critical section. 90 + */ 91 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 92 + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 93 + /* 32-bit only supported on BE */ \ 94 + ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ 94 95 ".popsection\n\t" 95 96 96 97 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ ··· 196 169 197 170 __asm__ __volatile__ goto ( 198 171 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 172 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 173 + #ifdef RSEQ_COMPARE_TWICE 174 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 175 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 176 + #endif 199 177 /* Start rseq by storing table entry pointer into rseq_cs. */ 200 178 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 201 179 /* cmp cpuid */ ··· 256 224 257 225 __asm__ __volatile__ goto ( 258 226 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 227 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 228 + #ifdef RSEQ_COMPARE_TWICE 229 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 230 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 231 + #endif 259 232 /* Start rseq by storing table entry pointer into rseq_cs. */ 260 233 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 261 234 /* cmp cpuid */ ··· 323 286 324 287 __asm__ __volatile__ goto ( 325 288 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 289 + #ifdef RSEQ_COMPARE_TWICE 290 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 291 + #endif 326 292 /* Start rseq by storing table entry pointer into rseq_cs. */ 327 293 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 328 294 /* cmp cpuid */ ··· 377 337 378 338 __asm__ __volatile__ goto ( 379 339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 340 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 341 + #ifdef RSEQ_COMPARE_TWICE 342 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 343 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 344 + #endif 380 345 /* Start rseq by storing table entry pointer into rseq_cs. */ 381 346 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 382 347 /* cmp cpuid */ ··· 445 400 446 401 __asm__ __volatile__ goto ( 447 402 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 403 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 404 + #ifdef RSEQ_COMPARE_TWICE 405 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 406 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 407 + #endif 448 408 /* Start rseq by storing table entry pointer into rseq_cs. */ 449 409 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 450 410 /* cmp cpuid */ ··· 515 465 516 466 __asm__ __volatile__ goto ( 517 467 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 468 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 469 + #ifdef RSEQ_COMPARE_TWICE 470 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 471 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 472 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 473 + #endif 518 474 /* Start rseq by storing table entry pointer into rseq_cs. */ 519 475 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 520 476 /* cmp cpuid */ ··· 588 532 589 533 __asm__ __volatile__ goto ( 590 534 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 535 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 536 + #ifdef RSEQ_COMPARE_TWICE 537 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 538 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 539 + #endif 591 540 /* setup for mempcy */ 592 541 "mr %%r19, %[len]\n\t" 593 542 "mr %%r20, %[src]\n\t" ··· 662 601 663 602 __asm__ __volatile__ goto ( 664 603 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 604 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 605 + #ifdef RSEQ_COMPARE_TWICE 606 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 607 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 608 + #endif 665 609 /* setup for mempcy */ 666 610 "mr %%r19, %[len]\n\t" 667 611 "mr %%r20, %[src]\n\t"
+55
tools/testing/selftests/rseq/rseq-s390.h
··· 44 44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 45 45 ".popsection\n\t" 46 46 47 + /* 48 + * Exit points of a rseq critical section consist of all instructions outside 49 + * of the critical section where a critical section can either branch to or 50 + * reach through the normal course of its execution. The abort IP and the 51 + * post-commit IP are already part of the __rseq_table section and should not 52 + * be explicitly defined as additional exit points. Knowing all exit points is 53 + * useful to assist debuggers stepping over the critical section. 54 + */ 55 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 56 + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 57 + ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 58 + ".popsection\n\t" 59 + 47 60 #elif __s390__ 48 61 49 62 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ ··· 66 53 __rseq_str(label) ":\n\t" \ 67 54 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 68 55 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 56 + ".popsection\n\t" 57 + 58 + /* 59 + * Exit points of a rseq critical section consist of all instructions outside 60 + * of the critical section where a critical section can either branch to or 61 + * reach through the normal course of its execution. The abort IP and the 62 + * post-commit IP are already part of the __rseq_table section and should not 63 + * be explicitly defined as additional exit points. Knowing all exit points is 64 + * useful to assist debuggers stepping over the critical section. 65 + */ 66 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 67 + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 68 + ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ 69 69 ".popsection\n\t" 70 70 71 71 #define LONG_L "l" ··· 128 102 129 103 __asm__ __volatile__ goto ( 130 104 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 105 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 106 + #ifdef RSEQ_COMPARE_TWICE 107 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 108 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 109 + #endif 131 110 /* Start rseq by storing table entry pointer into rseq_cs. */ 132 111 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 133 112 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 191 160 192 161 __asm__ __volatile__ goto ( 193 162 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 163 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 164 + #ifdef RSEQ_COMPARE_TWICE 165 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 166 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 167 + #endif 194 168 /* Start rseq by storing table entry pointer into rseq_cs. */ 195 169 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 196 170 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 256 220 257 221 __asm__ __volatile__ goto ( 258 222 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 223 + #ifdef RSEQ_COMPARE_TWICE 224 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 225 + #endif 259 226 /* Start rseq by storing table entry pointer into rseq_cs. */ 260 227 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 261 228 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 307 268 308 269 __asm__ __volatile__ goto ( 309 270 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 271 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 272 + #ifdef RSEQ_COMPARE_TWICE 273 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 274 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 275 + #endif 310 276 /* Start rseq by storing table entry pointer into rseq_cs. */ 311 277 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 312 278 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 383 339 384 340 __asm__ __volatile__ goto ( 385 341 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 342 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 343 + #ifdef RSEQ_COMPARE_TWICE 344 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 345 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 346 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 347 + #endif 386 348 /* Start rseq by storing table entry pointer into rseq_cs. */ 387 349 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 388 350 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) ··· 457 407 458 408 __asm__ __volatile__ goto ( 459 409 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 410 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 411 + #ifdef RSEQ_COMPARE_TWICE 412 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 413 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 414 + #endif 460 415 LONG_S " %[src], %[rseq_scratch0]\n\t" 461 416 LONG_S " %[dst], %[rseq_scratch1]\n\t" 462 417 LONG_S " %[len], %[rseq_scratch2]\n\t"
+94
tools/testing/selftests/rseq/rseq-x86.h
··· 58 58 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 59 59 (post_commit_ip - start_ip), abort_ip) 60 60 61 + /* 62 + * Exit points of a rseq critical section consist of all instructions outside 63 + * of the critical section where a critical section can either branch to or 64 + * reach through the normal course of its execution. The abort IP and the 65 + * post-commit IP are already part of the __rseq_table section and should not 66 + * be explicitly defined as additional exit points. Knowing all exit points is 67 + * useful to assist debuggers stepping over the critical section. 68 + */ 69 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 70 + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 71 + ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 72 + ".popsection\n\t" 73 + 61 74 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 62 75 RSEQ_INJECT_ASM(1) \ 63 76 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ ··· 106 93 107 94 __asm__ __volatile__ goto ( 108 95 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 96 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 97 + #ifdef RSEQ_COMPARE_TWICE 98 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 99 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 100 + #endif 109 101 /* Start rseq by storing table entry pointer into rseq_cs. */ 110 102 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 111 103 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 167 149 168 150 __asm__ __volatile__ goto ( 169 151 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 152 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 153 + #ifdef RSEQ_COMPARE_TWICE 154 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 155 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 156 + #endif 170 157 /* Start rseq by storing table entry pointer into rseq_cs. */ 171 158 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 172 159 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 230 207 231 208 __asm__ __volatile__ goto ( 232 209 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 210 + #ifdef RSEQ_COMPARE_TWICE 211 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 212 + #endif 233 213 /* Start rseq by storing table entry pointer into rseq_cs. */ 234 214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 235 215 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 277 251 278 252 __asm__ __volatile__ goto ( 279 253 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 254 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 255 + #ifdef RSEQ_COMPARE_TWICE 256 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 257 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 258 + #endif 280 259 /* Start rseq by storing table entry pointer into rseq_cs. */ 281 260 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 282 261 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 351 320 352 321 __asm__ __volatile__ goto ( 353 322 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 323 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 324 + #ifdef RSEQ_COMPARE_TWICE 325 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 326 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 327 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 328 + #endif 354 329 /* Start rseq by storing table entry pointer into rseq_cs. */ 355 330 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 356 331 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 423 386 424 387 __asm__ __volatile__ goto ( 425 388 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 389 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 390 + #ifdef RSEQ_COMPARE_TWICE 391 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 392 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 393 + #endif 426 394 "movq %[src], %[rseq_scratch0]\n\t" 427 395 "movq %[dst], %[rseq_scratch1]\n\t" 428 396 "movq %[len], %[rseq_scratch2]\n\t" ··· 577 535 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 578 536 (post_commit_ip - start_ip), abort_ip) 579 537 538 + /* 539 + * Exit points of a rseq critical section consist of all instructions outside 540 + * of the critical section where a critical section can either branch to or 541 + * reach through the normal course of its execution. The abort IP and the 542 + * post-commit IP are already part of the __rseq_table section and should not 543 + * be explicitly defined as additional exit points. Knowing all exit points is 544 + * useful to assist debuggers stepping over the critical section. 545 + */ 546 + #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 547 + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 548 + ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 549 + ".popsection\n\t" 550 + 580 551 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 581 552 RSEQ_INJECT_ASM(1) \ 582 553 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ ··· 624 569 625 570 __asm__ __volatile__ goto ( 626 571 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 572 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 573 + #ifdef RSEQ_COMPARE_TWICE 574 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 575 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 576 + #endif 627 577 /* Start rseq by storing table entry pointer into rseq_cs. */ 628 578 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 629 579 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 685 625 686 626 __asm__ __volatile__ goto ( 687 627 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 628 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 629 + #ifdef RSEQ_COMPARE_TWICE 630 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 631 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 632 + #endif 688 633 /* Start rseq by storing table entry pointer into rseq_cs. */ 689 634 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 690 635 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 748 683 749 684 __asm__ __volatile__ goto ( 750 685 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 686 + #ifdef RSEQ_COMPARE_TWICE 687 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 688 + #endif 751 689 /* Start rseq by storing table entry pointer into rseq_cs. */ 752 690 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 753 691 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 795 727 796 728 __asm__ __volatile__ goto ( 797 729 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 730 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 731 + #ifdef RSEQ_COMPARE_TWICE 732 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 733 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 734 + #endif 798 735 /* Start rseq by storing table entry pointer into rseq_cs. */ 799 736 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 800 737 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 861 788 862 789 __asm__ __volatile__ goto ( 863 790 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 791 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 792 + #ifdef RSEQ_COMPARE_TWICE 793 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 794 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 795 + #endif 864 796 /* Start rseq by storing table entry pointer into rseq_cs. */ 865 797 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 866 798 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 930 852 931 853 __asm__ __volatile__ goto ( 932 854 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 855 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 856 + #ifdef RSEQ_COMPARE_TWICE 857 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 858 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 859 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 860 + #endif 933 861 /* Start rseq by storing table entry pointer into rseq_cs. */ 934 862 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) 935 863 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) ··· 1004 920 1005 921 __asm__ __volatile__ goto ( 1006 922 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 923 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 924 + #ifdef RSEQ_COMPARE_TWICE 925 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 926 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 927 + #endif 1007 928 "movl %[src], %[rseq_scratch0]\n\t" 1008 929 "movl %[dst], %[rseq_scratch1]\n\t" 1009 930 "movl %[len], %[rseq_scratch2]\n\t" ··· 1116 1027 1117 1028 __asm__ __volatile__ goto ( 1118 1029 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1030 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 1031 + #ifdef RSEQ_COMPARE_TWICE 1032 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 1033 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 1034 + #endif 1119 1035 "movl %[src], %[rseq_scratch0]\n\t" 1120 1036 "movl %[dst], %[rseq_scratch1]\n\t" 1121 1037 "movl %[len], %[rseq_scratch2]\n\t"