tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 */
5
6#include <linux/linkage.h>
7#include <linux/const.h>
8#include <asm/assembler.h>
9#include <asm/page.h>
10#include <asm/cpufeature.h>
11#include <asm/alternative.h>
12
13/*
14 * Copy a page from src to dest (both are page aligned)
15 *
16 * Parameters:
17 * x0 - dest
18 * x1 - src
19 */
20SYM_FUNC_START(__pi_copy_page)
21alternative_if ARM64_HAS_NO_HW_PREFETCH
22 // Prefetch three cache lines ahead.
23 prfm pldl1strm, [x1, #128]
24 prfm pldl1strm, [x1, #256]
25 prfm pldl1strm, [x1, #384]
26alternative_else_nop_endif
27
28 ldp x2, x3, [x1]
29 ldp x4, x5, [x1, #16]
30 ldp x6, x7, [x1, #32]
31 ldp x8, x9, [x1, #48]
32 ldp x10, x11, [x1, #64]
33 ldp x12, x13, [x1, #80]
34 ldp x14, x15, [x1, #96]
35 ldp x16, x17, [x1, #112]
36
37 add x0, x0, #256
38 add x1, x1, #128
391:
40 tst x0, #(PAGE_SIZE - 1)
41
42alternative_if ARM64_HAS_NO_HW_PREFETCH
43 prfm pldl1strm, [x1, #384]
44alternative_else_nop_endif
45
46 stnp x2, x3, [x0, #-256]
47 ldp x2, x3, [x1]
48 stnp x4, x5, [x0, #16 - 256]
49 ldp x4, x5, [x1, #16]
50 stnp x6, x7, [x0, #32 - 256]
51 ldp x6, x7, [x1, #32]
52 stnp x8, x9, [x0, #48 - 256]
53 ldp x8, x9, [x1, #48]
54 stnp x10, x11, [x0, #64 - 256]
55 ldp x10, x11, [x1, #64]
56 stnp x12, x13, [x0, #80 - 256]
57 ldp x12, x13, [x1, #80]
58 stnp x14, x15, [x0, #96 - 256]
59 ldp x14, x15, [x1, #96]
60 stnp x16, x17, [x0, #112 - 256]
61 ldp x16, x17, [x1, #112]
62
63 add x0, x0, #128
64 add x1, x1, #128
65
66 b.ne 1b
67
68 stnp x2, x3, [x0, #-256]
69 stnp x4, x5, [x0, #16 - 256]
70 stnp x6, x7, [x0, #32 - 256]
71 stnp x8, x9, [x0, #48 - 256]
72 stnp x10, x11, [x0, #64 - 256]
73 stnp x12, x13, [x0, #80 - 256]
74 stnp x14, x15, [x0, #96 - 256]
75 stnp x16, x17, [x0, #112 - 256]
76
77 ret
78SYM_FUNC_END(__pi_copy_page)
79SYM_FUNC_ALIAS(copy_page, __pi_copy_page)
80EXPORT_SYMBOL(copy_page)