mips: propagate the calling convention change down into __csum_partial_copy_..._user()

Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git

kernel os linux

and turn the exception handlers into simply returning 0, which
simplifies the hell out of things in csum_partial.S

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

Al Viro 5 years ago 1cd95ab8 f863c65c

+90 -196

2 changed files

expand all

arch

mips

include

asm

checksum.h

lib

csum_partial.S

+6 -20

arch/mips/include/asm/checksum.h

··· 34 34 */ 35 35 __wsum csum_partial(const void *buff, int len, __wsum sum); 36 36 37 - __wsum __csum_partial_copy_from_user(const void *src, void *dst, 38 - int len, __wsum sum, int *err_ptr); 39 - __wsum __csum_partial_copy_to_user(const void *src, void *dst, 40 - int len, __wsum sum, int *err_ptr); 37 + __wsum __csum_partial_copy_from_user(const void __user *src, void *dst, int len); 38 + __wsum __csum_partial_copy_to_user(const void *src, void __user *dst, int len); 41 39 42 40 #define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 43 41 static inline 44 42 __wsum csum_and_copy_from_user(const void __user *src, void *dst, int len) 45 43 { 46 - __wsum sum = ~0U; 47 - int err = 0; 48 - 49 44 might_fault(); 50 - 51 45 if (!access_ok(src, len)) 52 46 return 0; 53 - sum = __csum_partial_copy_from_user((__force void *)src, dst, 54 - len, sum, &err); 55 - return err ? 0 : sum; 47 + return __csum_partial_copy_from_user(src, dst, len); 56 48 } 57 49 58 50 /* ··· 54 62 static inline 55 63 __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len) 56 64 { 57 - int err = 0; 58 - __wsum sum = ~0U; 59 - 60 65 might_fault(); 61 66 if (!access_ok(dst, len)) 62 67 return 0; 63 - sum = __csum_partial_copy_to_user(src, 64 - (__force void *)dst, 65 - len, sum, &err); 66 - return err ? 0 : sum; 68 + return __csum_partial_copy_to_user(src, dst, len); 67 69 } 68 70 69 71 /* ··· 65 79 * we have just one address space, so this is identical to the above) 66 80 */ 67 81 #define _HAVE_ARCH_CSUM_AND_COPY 68 - __wsum __csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum); 82 + __wsum __csum_partial_copy_nocheck(const void *src, void *dst, int len); 69 83 static inline __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len) 70 84 { 71 - return __csum_partial_copy_nocheck(src, dst, len, 0); 85 + return __csum_partial_copy_nocheck(src, dst, len); 72 86 } 73 87 74 88 /*

+84 -176

arch/mips/lib/csum_partial.S

··· 308 308 /* 309 309 * checksum and copy routines based on memcpy.S 310 310 * 311 - * csum_partial_copy_nocheck(src, dst, len, sum) 312 - * __csum_partial_copy_kernel(src, dst, len, sum, errp) 311 + * csum_partial_copy_nocheck(src, dst, len) 312 + * __csum_partial_copy_kernel(src, dst, len) 313 313 * 314 314 * See "Spec" in memcpy.S for details. Unlike __copy_user, all 315 315 * function in this file use the standard calling convention. ··· 318 318 #define src a0 319 319 #define dst a1 320 320 #define len a2 321 - #define psum a3 322 321 #define sum v0 323 322 #define odd t8 324 - #define errptr t9 325 323 326 324 /* 327 - * The exception handler for loads requires that: 328 - * 1- AT contain the address of the byte just past the end of the source 329 - * of the copy, 330 - * 2- src_entry <= src < AT, and 331 - * 3- (dst - src) == (dst_entry - src_entry), 332 - * The _entry suffix denotes values when __copy_user was called. 333 - * 334 - * (1) is set up up by __csum_partial_copy_from_user and maintained by 335 - * not writing AT in __csum_partial_copy 336 - * (2) is met by incrementing src by the number of bytes copied 337 - * (3) is met by not doing loads between a pair of increments of dst and src 338 - * 339 - * The exception handlers for stores stores -EFAULT to errptr and return. 340 - * These handlers do not need to overwrite any data. 325 + * All exception handlers simply return 0. 341 326 */ 342 327 343 328 /* Instruction type */ ··· 343 358 * addr : Address 344 359 * handler : Exception handler 345 360 */ 346 - #define EXC(insn, type, reg, addr, handler) \ 361 + #define EXC(insn, type, reg, addr) \ 347 362 .if \mode == LEGACY_MODE; \ 348 363 9: insn reg, addr; \ 349 364 .section __ex_table,"a"; \ 350 - PTR 9b, handler; \ 365 + PTR 9b, .L_exc; \ 351 366 .previous; \ 352 367 /* This is enabled in EVA mode */ \ 353 368 .else; \ ··· 356 371 ((\to == USEROP) && (type == ST_INSN)); \ 357 372 9: __BUILD_EVA_INSN(insn##e, reg, addr); \ 358 373 .section __ex_table,"a"; \ 359 - PTR 9b, handler; \ 374 + PTR 9b, .L_exc; \ 360 375 .previous; \ 361 376 .else; \ 362 377 /* EVA without exception */ \ ··· 369 384 #ifdef USE_DOUBLE 370 385 371 386 #define LOADK ld /* No exception */ 372 - #define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler) 373 - #define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler) 374 - #define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler) 375 - #define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler) 376 - #define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler) 377 - #define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler) 378 - #define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler) 379 - #define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler) 387 + #define LOAD(reg, addr) EXC(ld, LD_INSN, reg, addr) 388 + #define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr) 389 + #define LOADL(reg, addr) EXC(ldl, LD_INSN, reg, addr) 390 + #define LOADR(reg, addr) EXC(ldr, LD_INSN, reg, addr) 391 + #define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr) 392 + #define STOREL(reg, addr) EXC(sdl, ST_INSN, reg, addr) 393 + #define STORER(reg, addr) EXC(sdr, ST_INSN, reg, addr) 394 + #define STORE(reg, addr) EXC(sd, ST_INSN, reg, addr) 380 395 #define ADD daddu 381 396 #define SUB dsubu 382 397 #define SRL dsrl ··· 389 404 #else 390 405 391 406 #define LOADK lw /* No exception */ 392 - #define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler) 393 - #define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler) 394 - #define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler) 395 - #define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler) 396 - #define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler) 397 - #define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler) 398 - #define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler) 399 - #define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler) 407 + #define LOAD(reg, addr) EXC(lw, LD_INSN, reg, addr) 408 + #define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr) 409 + #define LOADL(reg, addr) EXC(lwl, LD_INSN, reg, addr) 410 + #define LOADR(reg, addr) EXC(lwr, LD_INSN, reg, addr) 411 + #define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr) 412 + #define STOREL(reg, addr) EXC(swl, ST_INSN, reg, addr) 413 + #define STORER(reg, addr) EXC(swr, ST_INSN, reg, addr) 414 + #define STORE(reg, addr) EXC(sw, ST_INSN, reg, addr) 400 415 #define ADD addu 401 416 #define SUB subu 402 417 #define SRL srl ··· 435 450 .set at=v1 436 451 #endif 437 452 438 - .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to, __nocheck 453 + .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to 439 454 440 - PTR_ADDU AT, src, len /* See (1) above. */ 441 - /* initialize __nocheck if this the first time we execute this 442 - * macro 443 - */ 444 - #ifdef CONFIG_64BIT 445 - move errptr, a4 446 - #else 447 - lw errptr, 16(sp) 448 - #endif 449 - .if \__nocheck == 1 450 - FEXPORT(__csum_partial_copy_nocheck) 451 - EXPORT_SYMBOL(__csum_partial_copy_nocheck) 452 - .endif 453 - move sum, zero 455 + li sum, -1 454 456 move odd, zero 455 457 /* 456 458 * Note: dst & src may be unaligned, len may be 0 ··· 469 497 SUB len, 8*NBYTES # subtract here for bgez loop 470 498 .align 4 471 499 1: 472 - LOAD(t0, UNIT(0)(src), .Ll_exc\@) 473 - LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@) 474 - LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@) 475 - LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@) 476 - LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@) 477 - LOAD(t5, UNIT(5)(src), .Ll_exc_copy\@) 478 - LOAD(t6, UNIT(6)(src), .Ll_exc_copy\@) 479 - LOAD(t7, UNIT(7)(src), .Ll_exc_copy\@) 500 + LOAD(t0, UNIT(0)(src)) 501 + LOAD(t1, UNIT(1)(src)) 502 + LOAD(t2, UNIT(2)(src)) 503 + LOAD(t3, UNIT(3)(src)) 504 + LOAD(t4, UNIT(4)(src)) 505 + LOAD(t5, UNIT(5)(src)) 506 + LOAD(t6, UNIT(6)(src)) 507 + LOAD(t7, UNIT(7)(src)) 480 508 SUB len, len, 8*NBYTES 481 509 ADD src, src, 8*NBYTES 482 - STORE(t0, UNIT(0)(dst), .Ls_exc\@) 510 + STORE(t0, UNIT(0)(dst)) 483 511 ADDC(t0, t1) 484 - STORE(t1, UNIT(1)(dst), .Ls_exc\@) 512 + STORE(t1, UNIT(1)(dst)) 485 513 ADDC(sum, t0) 486 - STORE(t2, UNIT(2)(dst), .Ls_exc\@) 514 + STORE(t2, UNIT(2)(dst)) 487 515 ADDC(t2, t3) 488 - STORE(t3, UNIT(3)(dst), .Ls_exc\@) 516 + STORE(t3, UNIT(3)(dst)) 489 517 ADDC(sum, t2) 490 - STORE(t4, UNIT(4)(dst), .Ls_exc\@) 518 + STORE(t4, UNIT(4)(dst)) 491 519 ADDC(t4, t5) 492 - STORE(t5, UNIT(5)(dst), .Ls_exc\@) 520 + STORE(t5, UNIT(5)(dst)) 493 521 ADDC(sum, t4) 494 - STORE(t6, UNIT(6)(dst), .Ls_exc\@) 522 + STORE(t6, UNIT(6)(dst)) 495 523 ADDC(t6, t7) 496 - STORE(t7, UNIT(7)(dst), .Ls_exc\@) 524 + STORE(t7, UNIT(7)(dst)) 497 525 ADDC(sum, t6) 498 526 .set reorder /* DADDI_WAR */ 499 527 ADD dst, dst, 8*NBYTES ··· 513 541 /* 514 542 * len >= 4*NBYTES 515 543 */ 516 - LOAD(t0, UNIT(0)(src), .Ll_exc\@) 517 - LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@) 518 - LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@) 519 - LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@) 544 + LOAD(t0, UNIT(0)(src)) 545 + LOAD(t1, UNIT(1)(src)) 546 + LOAD(t2, UNIT(2)(src)) 547 + LOAD(t3, UNIT(3)(src)) 520 548 SUB len, len, 4*NBYTES 521 549 ADD src, src, 4*NBYTES 522 - STORE(t0, UNIT(0)(dst), .Ls_exc\@) 550 + STORE(t0, UNIT(0)(dst)) 523 551 ADDC(t0, t1) 524 - STORE(t1, UNIT(1)(dst), .Ls_exc\@) 552 + STORE(t1, UNIT(1)(dst)) 525 553 ADDC(sum, t0) 526 - STORE(t2, UNIT(2)(dst), .Ls_exc\@) 554 + STORE(t2, UNIT(2)(dst)) 527 555 ADDC(t2, t3) 528 - STORE(t3, UNIT(3)(dst), .Ls_exc\@) 556 + STORE(t3, UNIT(3)(dst)) 529 557 ADDC(sum, t2) 530 558 .set reorder /* DADDI_WAR */ 531 559 ADD dst, dst, 4*NBYTES ··· 538 566 beq rem, len, .Lcopy_bytes\@ 539 567 nop 540 568 1: 541 - LOAD(t0, 0(src), .Ll_exc\@) 569 + LOAD(t0, 0(src)) 542 570 ADD src, src, NBYTES 543 571 SUB len, len, NBYTES 544 - STORE(t0, 0(dst), .Ls_exc\@) 572 + STORE(t0, 0(dst)) 545 573 ADDC(sum, t0) 546 574 .set reorder /* DADDI_WAR */ 547 575 ADD dst, dst, NBYTES ··· 564 592 ADD t1, dst, len # t1 is just past last byte of dst 565 593 li bits, 8*NBYTES 566 594 SLL rem, len, 3 # rem = number of bits to keep 567 - LOAD(t0, 0(src), .Ll_exc\@) 595 + LOAD(t0, 0(src)) 568 596 SUB bits, bits, rem # bits = number of bits to discard 569 597 SHIFT_DISCARD t0, t0, bits 570 - STREST(t0, -1(t1), .Ls_exc\@) 598 + STREST(t0, -1(t1)) 571 599 SHIFT_DISCARD_REVERT t0, t0, bits 572 600 .set reorder 573 601 ADDC(sum, t0) ··· 584 612 * Set match = (src and dst have same alignment) 585 613 */ 586 614 #define match rem 587 - LDFIRST(t3, FIRST(0)(src), .Ll_exc\@) 615 + LDFIRST(t3, FIRST(0)(src)) 588 616 ADD t2, zero, NBYTES 589 - LDREST(t3, REST(0)(src), .Ll_exc_copy\@) 617 + LDREST(t3, REST(0)(src)) 590 618 SUB t2, t2, t1 # t2 = number of bytes copied 591 619 xor match, t0, t1 592 - STFIRST(t3, FIRST(0)(dst), .Ls_exc\@) 620 + STFIRST(t3, FIRST(0)(dst)) 593 621 SLL t4, t1, 3 # t4 = number of bits to discard 594 622 SHIFT_DISCARD t3, t3, t4 595 623 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */ ··· 611 639 * It's OK to load FIRST(N+1) before REST(N) because the two addresses 612 640 * are to the same unit (unless src is aligned, but it's not). 613 641 */ 614 - LDFIRST(t0, FIRST(0)(src), .Ll_exc\@) 615 - LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@) 642 + LDFIRST(t0, FIRST(0)(src)) 643 + LDFIRST(t1, FIRST(1)(src)) 616 644 SUB len, len, 4*NBYTES 617 - LDREST(t0, REST(0)(src), .Ll_exc_copy\@) 618 - LDREST(t1, REST(1)(src), .Ll_exc_copy\@) 619 - LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@) 620 - LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@) 621 - LDREST(t2, REST(2)(src), .Ll_exc_copy\@) 622 - LDREST(t3, REST(3)(src), .Ll_exc_copy\@) 645 + LDREST(t0, REST(0)(src)) 646 + LDREST(t1, REST(1)(src)) 647 + LDFIRST(t2, FIRST(2)(src)) 648 + LDFIRST(t3, FIRST(3)(src)) 649 + LDREST(t2, REST(2)(src)) 650 + LDREST(t3, REST(3)(src)) 623 651 ADD src, src, 4*NBYTES 624 652 #ifdef CONFIG_CPU_SB1 625 653 nop # improves slotting 626 654 #endif 627 - STORE(t0, UNIT(0)(dst), .Ls_exc\@) 655 + STORE(t0, UNIT(0)(dst)) 628 656 ADDC(t0, t1) 629 - STORE(t1, UNIT(1)(dst), .Ls_exc\@) 657 + STORE(t1, UNIT(1)(dst)) 630 658 ADDC(sum, t0) 631 - STORE(t2, UNIT(2)(dst), .Ls_exc\@) 659 + STORE(t2, UNIT(2)(dst)) 632 660 ADDC(t2, t3) 633 - STORE(t3, UNIT(3)(dst), .Ls_exc\@) 661 + STORE(t3, UNIT(3)(dst)) 634 662 ADDC(sum, t2) 635 663 .set reorder /* DADDI_WAR */ 636 664 ADD dst, dst, 4*NBYTES ··· 643 671 beq rem, len, .Lcopy_bytes\@ 644 672 nop 645 673 1: 646 - LDFIRST(t0, FIRST(0)(src), .Ll_exc\@) 647 - LDREST(t0, REST(0)(src), .Ll_exc_copy\@) 674 + LDFIRST(t0, FIRST(0)(src)) 675 + LDREST(t0, REST(0)(src)) 648 676 ADD src, src, NBYTES 649 677 SUB len, len, NBYTES 650 - STORE(t0, 0(dst), .Ls_exc\@) 678 + STORE(t0, 0(dst)) 651 679 ADDC(sum, t0) 652 680 .set reorder /* DADDI_WAR */ 653 681 ADD dst, dst, NBYTES ··· 668 696 #endif 669 697 move t2, zero # partial word 670 698 li t3, SHIFT_START # shift 671 - /* use .Ll_exc_copy here to return correct sum on fault */ 672 699 #define COPY_BYTE(N) \ 673 - LOADBU(t0, N(src), .Ll_exc_copy\@); \ 700 + LOADBU(t0, N(src)); \ 674 701 SUB len, len, 1; \ 675 - STOREB(t0, N(dst), .Ls_exc\@); \ 702 + STOREB(t0, N(dst)); \ 676 703 SLLV t0, t0, t3; \ 677 704 addu t3, SHIFT_INC; \ 678 705 beqz len, .Lcopy_bytes_done\@; \ ··· 685 714 COPY_BYTE(4) 686 715 COPY_BYTE(5) 687 716 #endif 688 - LOADBU(t0, NBYTES-2(src), .Ll_exc_copy\@) 717 + LOADBU(t0, NBYTES-2(src)) 689 718 SUB len, len, 1 690 - STOREB(t0, NBYTES-2(dst), .Ls_exc\@) 719 + STOREB(t0, NBYTES-2(dst)) 691 720 SLLV t0, t0, t3 692 721 or t2, t0 693 722 .Lcopy_bytes_done\@: ··· 724 753 #endif 725 754 .set pop 726 755 .set reorder 727 - ADDC32(sum, psum) 728 756 jr ra 729 757 .set noreorder 730 - 731 - .Ll_exc_copy\@: 732 - /* 733 - * Copy bytes from src until faulting load address (or until a 734 - * lb faults) 735 - * 736 - * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28) 737 - * may be more than a byte beyond the last address. 738 - * Hence, the lb below may get an exception. 739 - * 740 - * Assumes src < THREAD_BUADDR($28) 741 - */ 742 - LOADK t0, TI_TASK($28) 743 - li t2, SHIFT_START 744 - LOADK t0, THREAD_BUADDR(t0) 745 - 1: 746 - LOADBU(t1, 0(src), .Ll_exc\@) 747 - ADD src, src, 1 748 - sb t1, 0(dst) # can't fault -- we're copy_from_user 749 - SLLV t1, t1, t2 750 - addu t2, SHIFT_INC 751 - ADDC(sum, t1) 752 - .set reorder /* DADDI_WAR */ 753 - ADD dst, dst, 1 754 - bne src, t0, 1b 755 - .set noreorder 756 - .Ll_exc\@: 757 - LOADK t0, TI_TASK($28) 758 - nop 759 - LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address 760 - nop 761 - SUB len, AT, t0 # len number of uncopied bytes 762 - /* 763 - * Here's where we rely on src and dst being incremented in tandem, 764 - * See (3) above. 765 - * dst += (fault addr - src) to put dst at first byte to clear 766 - */ 767 - ADD dst, t0 # compute start address in a1 768 - SUB dst, src 769 - /* 770 - * Clear len bytes starting at dst. Can't call __bzero because it 771 - * might modify len. An inefficient loop for these rare times... 772 - */ 773 - .set reorder /* DADDI_WAR */ 774 - SUB src, len, 1 775 - beqz len, .Ldone\@ 776 - .set noreorder 777 - 1: sb zero, 0(dst) 778 - ADD dst, dst, 1 779 - .set push 780 - .set noat 781 - #ifndef CONFIG_CPU_DADDI_WORKAROUNDS 782 - bnez src, 1b 783 - SUB src, src, 1 784 - #else 785 - li v1, 1 786 - bnez src, 1b 787 - SUB src, src, v1 788 - #endif 789 - li v1, -EFAULT 790 - b .Ldone\@ 791 - sw v1, (errptr) 792 - 793 - .Ls_exc\@: 794 - li v0, -1 /* invalid checksum */ 795 - li v1, -EFAULT 796 - jr ra 797 - sw v1, (errptr) 798 - .set pop 799 758 .endm 800 759 760 + .set noreorder 761 + .L_exc: 762 + jr ra 763 + li v0, 0 764 + 765 + FEXPORT(__csum_partial_copy_nocheck) 766 + EXPORT_SYMBOL(__csum_partial_copy_nocheck) 801 767 #ifndef CONFIG_EVA 802 768 FEXPORT(__csum_partial_copy_to_user) 803 769 EXPORT_SYMBOL(__csum_partial_copy_to_user) 804 770 FEXPORT(__csum_partial_copy_from_user) 805 771 EXPORT_SYMBOL(__csum_partial_copy_from_user) 806 772 #endif 807 - __BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 1 773 + __BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 808 774 809 775 #ifdef CONFIG_EVA 810 776 LEAF(__csum_partial_copy_to_user) 811 - __BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP 0 777 + __BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP 812 778 END(__csum_partial_copy_to_user) 813 779 814 780 LEAF(__csum_partial_copy_from_user) 815 - __BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP 0 781 + __BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP 816 782 END(__csum_partial_copy_from_user) 817 783 #endif