tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020 ARM Ltd.
4 */
5#ifndef __ASM_MTE_H
6#define __ASM_MTE_H
7
8#include <asm/compiler.h>
9#include <asm/mte-def.h>
10
11#ifndef __ASSEMBLY__
12
13#include <linux/bitfield.h>
14#include <linux/kasan-enabled.h>
15#include <linux/page-flags.h>
16#include <linux/sched.h>
17#include <linux/types.h>
18
19#include <asm/pgtable-types.h>
20
21void mte_clear_page_tags(void *addr);
22unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
23 unsigned long n);
24unsigned long mte_copy_tags_to_user(void __user *to, void *from,
25 unsigned long n);
26int mte_save_tags(struct page *page);
27void mte_save_page_tags(const void *page_addr, void *tag_storage);
28bool mte_restore_tags(swp_entry_t entry, struct page *page);
29void mte_restore_page_tags(void *page_addr, const void *tag_storage);
30void mte_invalidate_tags(int type, pgoff_t offset);
31void mte_invalidate_tags_area(int type);
32void *mte_allocate_tag_storage(void);
33void mte_free_tag_storage(char *storage);
34
35#ifdef CONFIG_ARM64_MTE
36
37/* track which pages have valid allocation tags */
38#define PG_mte_tagged PG_arch_2
39
40void mte_zero_clear_page_tags(void *addr);
41void mte_sync_tags(pte_t old_pte, pte_t pte);
42void mte_copy_page_tags(void *kto, const void *kfrom);
43void mte_thread_init_user(void);
44void mte_thread_switch(struct task_struct *next);
45void mte_suspend_enter(void);
46long set_mte_ctrl(struct task_struct *task, unsigned long arg);
47long get_mte_ctrl(struct task_struct *task);
48int mte_ptrace_copy_tags(struct task_struct *child, long request,
49 unsigned long addr, unsigned long data);
50size_t mte_probe_user_range(const char __user *uaddr, size_t size);
51
52#else /* CONFIG_ARM64_MTE */
53
54/* unused if !CONFIG_ARM64_MTE, silence the compiler */
55#define PG_mte_tagged 0
56
57static inline void mte_zero_clear_page_tags(void *addr)
58{
59}
60static inline void mte_sync_tags(pte_t old_pte, pte_t pte)
61{
62}
63static inline void mte_copy_page_tags(void *kto, const void *kfrom)
64{
65}
66static inline void mte_thread_init_user(void)
67{
68}
69static inline void mte_thread_switch(struct task_struct *next)
70{
71}
72static inline void mte_suspend_enter(void)
73{
74}
75static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg)
76{
77 return 0;
78}
79static inline long get_mte_ctrl(struct task_struct *task)
80{
81 return 0;
82}
83static inline int mte_ptrace_copy_tags(struct task_struct *child,
84 long request, unsigned long addr,
85 unsigned long data)
86{
87 return -EIO;
88}
89
90#endif /* CONFIG_ARM64_MTE */
91
92static inline void mte_disable_tco_entry(struct task_struct *task)
93{
94 if (!system_supports_mte())
95 return;
96
97 /*
98 * Re-enable tag checking (TCO set on exception entry). This is only
99 * necessary if MTE is enabled in either the kernel or the userspace
100 * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set
101 * for both). With MTE disabled in the kernel and disabled or
102 * asynchronous in userspace, tag check faults (including in uaccesses)
103 * are not reported, therefore there is no need to re-enable checking.
104 * This is beneficial on microarchitectures where re-enabling TCO is
105 * expensive.
106 */
107 if (kasan_hw_tags_enabled() ||
108 (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT)))
109 asm volatile(SET_PSTATE_TCO(0));
110}
111
112#ifdef CONFIG_KASAN_HW_TAGS
113/* Whether the MTE asynchronous mode is enabled. */
114DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
115
116static inline bool system_uses_mte_async_or_asymm_mode(void)
117{
118 return static_branch_unlikely(&mte_async_or_asymm_mode);
119}
120
121void mte_check_tfsr_el1(void);
122
123static inline void mte_check_tfsr_entry(void)
124{
125 if (!system_supports_mte())
126 return;
127
128 mte_check_tfsr_el1();
129}
130
131static inline void mte_check_tfsr_exit(void)
132{
133 if (!system_supports_mte())
134 return;
135
136 /*
137 * The asynchronous faults are sync'ed automatically with
138 * TFSR_EL1 on kernel entry but for exit an explicit dsb()
139 * is required.
140 */
141 dsb(nsh);
142 isb();
143
144 mte_check_tfsr_el1();
145}
146#else
147static inline bool system_uses_mte_async_or_asymm_mode(void)
148{
149 return false;
150}
151static inline void mte_check_tfsr_el1(void)
152{
153}
154static inline void mte_check_tfsr_entry(void)
155{
156}
157static inline void mte_check_tfsr_exit(void)
158{
159}
160#endif /* CONFIG_KASAN_HW_TAGS */
161
162#endif /* __ASSEMBLY__ */
163#endif /* __ASM_MTE_H */