tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2020 ARM Ltd.
4 */
5#include <linux/linkage.h>
6
7#include <asm/asm-uaccess.h>
8#include <asm/assembler.h>
9#include <asm/mte.h>
10#include <asm/page.h>
11#include <asm/sysreg.h>
12
13 .arch armv8.5-a+memtag
14
15/*
16 * multitag_transfer_size - set \reg to the block size that is accessed by the
17 * LDGM/STGM instructions.
18 */
19 .macro multitag_transfer_size, reg, tmp
20 mrs_s \reg, SYS_GMID_EL1
21 ubfx \reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE
22 mov \tmp, #4
23 lsl \reg, \tmp, \reg
24 .endm
25
26/*
27 * Clear the tags in a page
28 * x0 - address of the page to be cleared
29 */
30SYM_FUNC_START(mte_clear_page_tags)
31 multitag_transfer_size x1, x2
321: stgm xzr, [x0]
33 add x0, x0, x1
34 tst x0, #(PAGE_SIZE - 1)
35 b.ne 1b
36 ret
37SYM_FUNC_END(mte_clear_page_tags)
38
39/*
40 * Zero the page and tags at the same time
41 *
42 * Parameters:
43 * x0 - address to the beginning of the page
44 */
45SYM_FUNC_START(mte_zero_clear_page_tags)
46 mrs x1, dczid_el0
47 and w1, w1, #0xf
48 mov x2, #4
49 lsl x1, x2, x1
50 and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag
51
521: dc gzva, x0
53 add x0, x0, x1
54 tst x0, #(PAGE_SIZE - 1)
55 b.ne 1b
56 ret
57SYM_FUNC_END(mte_zero_clear_page_tags)
58
59/*
60 * Copy the tags from the source page to the destination one
61 * x0 - address of the destination page
62 * x1 - address of the source page
63 */
64SYM_FUNC_START(mte_copy_page_tags)
65 mov x2, x0
66 mov x3, x1
67 multitag_transfer_size x5, x6
681: ldgm x4, [x3]
69 stgm x4, [x2]
70 add x2, x2, x5
71 add x3, x3, x5
72 tst x2, #(PAGE_SIZE - 1)
73 b.ne 1b
74 ret
75SYM_FUNC_END(mte_copy_page_tags)
76
77/*
78 * Read tags from a user buffer (one tag per byte) and set the corresponding
79 * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
80 * x0 - kernel address (to)
81 * x1 - user buffer (from)
82 * x2 - number of tags/bytes (n)
83 * Returns:
84 * x0 - number of tags read/set
85 */
86SYM_FUNC_START(mte_copy_tags_from_user)
87 mov x3, x1
88 cbz x2, 2f
891:
90 user_ldst 2f, ldtrb, w4, x1, 0
91 lsl x4, x4, #MTE_TAG_SHIFT
92 stg x4, [x0], #MTE_GRANULE_SIZE
93 add x1, x1, #1
94 subs x2, x2, #1
95 b.ne 1b
96
97 // exception handling and function return
982: sub x0, x1, x3 // update the number of tags set
99 ret
100SYM_FUNC_END(mte_copy_tags_from_user)
101
102/*
103 * Get the tags from a kernel address range and write the tag values to the
104 * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
105 * x0 - user buffer (to)
106 * x1 - kernel address (from)
107 * x2 - number of tags/bytes (n)
108 * Returns:
109 * x0 - number of tags read/set
110 */
111SYM_FUNC_START(mte_copy_tags_to_user)
112 mov x3, x0
113 cbz x2, 2f
1141:
115 ldg x4, [x1]
116 ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
117 user_ldst 2f, sttrb, w4, x0, 0
118 add x0, x0, #1
119 add x1, x1, #MTE_GRANULE_SIZE
120 subs x2, x2, #1
121 b.ne 1b
122
123 // exception handling and function return
1242: sub x0, x0, x3 // update the number of tags copied
125 ret
126SYM_FUNC_END(mte_copy_tags_to_user)
127
128/*
129 * Save the tags in a page
130 * x0 - page address
131 * x1 - tag storage
132 */
133SYM_FUNC_START(mte_save_page_tags)
134 multitag_transfer_size x7, x5
1351:
136 mov x2, #0
1372:
138 ldgm x5, [x0]
139 orr x2, x2, x5
140 add x0, x0, x7
141 tst x0, #0xFF // 16 tag values fit in a register,
142 b.ne 2b // which is 16*16=256 bytes
143
144 str x2, [x1], #8
145
146 tst x0, #(PAGE_SIZE - 1)
147 b.ne 1b
148
149 ret
150SYM_FUNC_END(mte_save_page_tags)
151
152/*
153 * Restore the tags in a page
154 * x0 - page address
155 * x1 - tag storage
156 */
157SYM_FUNC_START(mte_restore_page_tags)
158 multitag_transfer_size x7, x5
1591:
160 ldr x2, [x1], #8
1612:
162 stgm x2, [x0]
163 add x0, x0, x7
164 tst x0, #0xFF
165 b.ne 2b
166
167 tst x0, #(PAGE_SIZE - 1)
168 b.ne 1b
169
170 ret
171SYM_FUNC_END(mte_restore_page_tags)