tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 */
5
6#include <linux/linkage.h>
7#include <linux/const.h>
8#include <asm/assembler.h>
9#include <asm/page.h>
10#include <asm/cpufeature.h>
11#include <asm/alternative.h>
12
13/*
14 * Copy a page from src to dest (both are page aligned)
15 *
16 * Parameters:
17 * x0 - dest
18 * x1 - src
19 */
20SYM_FUNC_START(__pi_copy_page)
21#ifdef CONFIG_AS_HAS_MOPS
22 .arch_extension mops
23alternative_if_not ARM64_HAS_MOPS
24 b .Lno_mops
25alternative_else_nop_endif
26
27 mov x2, #PAGE_SIZE
28 cpypwn [x0]!, [x1]!, x2!
29 cpymwn [x0]!, [x1]!, x2!
30 cpyewn [x0]!, [x1]!, x2!
31 ret
32.Lno_mops:
33#endif
34 ldp x2, x3, [x1]
35 ldp x4, x5, [x1, #16]
36 ldp x6, x7, [x1, #32]
37 ldp x8, x9, [x1, #48]
38 ldp x10, x11, [x1, #64]
39 ldp x12, x13, [x1, #80]
40 ldp x14, x15, [x1, #96]
41 ldp x16, x17, [x1, #112]
42
43 add x0, x0, #256
44 add x1, x1, #128
451:
46 tst x0, #(PAGE_SIZE - 1)
47
48 stnp x2, x3, [x0, #-256]
49 ldp x2, x3, [x1]
50 stnp x4, x5, [x0, #16 - 256]
51 ldp x4, x5, [x1, #16]
52 stnp x6, x7, [x0, #32 - 256]
53 ldp x6, x7, [x1, #32]
54 stnp x8, x9, [x0, #48 - 256]
55 ldp x8, x9, [x1, #48]
56 stnp x10, x11, [x0, #64 - 256]
57 ldp x10, x11, [x1, #64]
58 stnp x12, x13, [x0, #80 - 256]
59 ldp x12, x13, [x1, #80]
60 stnp x14, x15, [x0, #96 - 256]
61 ldp x14, x15, [x1, #96]
62 stnp x16, x17, [x0, #112 - 256]
63 ldp x16, x17, [x1, #112]
64
65 add x0, x0, #128
66 add x1, x1, #128
67
68 b.ne 1b
69
70 stnp x2, x3, [x0, #-256]
71 stnp x4, x5, [x0, #16 - 256]
72 stnp x6, x7, [x0, #32 - 256]
73 stnp x8, x9, [x0, #48 - 256]
74 stnp x10, x11, [x0, #64 - 256]
75 stnp x12, x13, [x0, #80 - 256]
76 stnp x14, x15, [x0, #96 - 256]
77 stnp x16, x17, [x0, #112 - 256]
78
79 ret
80SYM_FUNC_END(__pi_copy_page)
81SYM_FUNC_ALIAS(copy_page, __pi_copy_page)
82EXPORT_SYMBOL(copy_page)