Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 Regents of the University of California
4 */
5
6#ifndef _ASM_RISCV_PGTABLE_64_H
7#define _ASM_RISCV_PGTABLE_64_H
8
9#include <linux/bits.h>
10#include <linux/const.h>
11#include <asm/errata_list.h>
12
13extern bool pgtable_l4_enabled;
14extern bool pgtable_l5_enabled;
15
16#define PGDIR_SHIFT_L3 30
17#define PGDIR_SHIFT_L4 39
18#define PGDIR_SHIFT_L5 48
19#define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3)
20
21#define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \
22 (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3))
23/* Size of region mapped by a page global directory */
24#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
25#define PGDIR_MASK (~(PGDIR_SIZE - 1))
26
27/* p4d is folded into pgd in case of 4-level page table */
28#define P4D_SHIFT_L3 30
29#define P4D_SHIFT_L4 39
30#define P4D_SHIFT_L5 39
31#define P4D_SHIFT (pgtable_l5_enabled ? P4D_SHIFT_L5 : \
32 (pgtable_l4_enabled ? P4D_SHIFT_L4 : P4D_SHIFT_L3))
33#define P4D_SIZE (_AC(1, UL) << P4D_SHIFT)
34#define P4D_MASK (~(P4D_SIZE - 1))
35
36/* pud is folded into pgd in case of 3-level page table */
37#define PUD_SHIFT 30
38#define PUD_SIZE (_AC(1, UL) << PUD_SHIFT)
39#define PUD_MASK (~(PUD_SIZE - 1))
40
41#define PMD_SHIFT 21
42/* Size of region mapped by a page middle directory */
43#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT)
44#define PMD_MASK (~(PMD_SIZE - 1))
45
46/* Page 4th Directory entry */
47typedef struct {
48 unsigned long p4d;
49} p4d_t;
50
51#define p4d_val(x) ((x).p4d)
52#define __p4d(x) ((p4d_t) { (x) })
53#define PTRS_PER_P4D (PAGE_SIZE / sizeof(p4d_t))
54
55/* Page Upper Directory entry */
56typedef struct {
57 unsigned long pud;
58} pud_t;
59
60#define pud_val(x) ((x).pud)
61#define __pud(x) ((pud_t) { (x) })
62#define PTRS_PER_PUD (PAGE_SIZE / sizeof(pud_t))
63
64/* Page Middle Directory entry */
65typedef struct {
66 unsigned long pmd;
67} pmd_t;
68
69#define pmd_val(x) ((x).pmd)
70#define __pmd(x) ((pmd_t) { (x) })
71
72#define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t))
73
74/*
75 * rv64 PTE format:
76 * | 63 | 62 61 | 60 54 | 53 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
77 * N MT RSV PFN reserved for SW D A G U X W R V
78 */
79#define _PAGE_PFN_MASK GENMASK(53, 10)
80
81/*
82 * [62:61] Svpbmt Memory Type definitions:
83 *
84 * 00 - PMA Normal Cacheable, No change to implied PMA memory type
85 * 01 - NC Non-cacheable, idempotent, weakly-ordered Main Memory
86 * 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory
87 * 11 - Rsvd Reserved for future standard use
88 */
89#define _PAGE_NOCACHE_SVPBMT (1UL << 61)
90#define _PAGE_IO_SVPBMT (1UL << 62)
91#define _PAGE_MTMASK_SVPBMT (_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT)
92
93/*
94 * [63:59] T-Head Memory Type definitions:
95 *
96 * 00000 - NC Weakly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable
97 * 01110 - PMA Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable
98 * 10000 - IO Strongly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable
99 */
100#define _PAGE_PMA_THEAD ((1UL << 62) | (1UL << 61) | (1UL << 60))
101#define _PAGE_NOCACHE_THEAD 0UL
102#define _PAGE_IO_THEAD (1UL << 63)
103#define _PAGE_MTMASK_THEAD (_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59))
104
105static inline u64 riscv_page_mtmask(void)
106{
107 u64 val;
108
109 ALT_SVPBMT(val, _PAGE_MTMASK);
110 return val;
111}
112
113static inline u64 riscv_page_nocache(void)
114{
115 u64 val;
116
117 ALT_SVPBMT(val, _PAGE_NOCACHE);
118 return val;
119}
120
121static inline u64 riscv_page_io(void)
122{
123 u64 val;
124
125 ALT_SVPBMT(val, _PAGE_IO);
126 return val;
127}
128
129#define _PAGE_NOCACHE riscv_page_nocache()
130#define _PAGE_IO riscv_page_io()
131#define _PAGE_MTMASK riscv_page_mtmask()
132
133/* Set of bits to preserve across pte_modify() */
134#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \
135 _PAGE_WRITE | _PAGE_EXEC | \
136 _PAGE_USER | _PAGE_GLOBAL | \
137 _PAGE_MTMASK))
138
139static inline int pud_present(pud_t pud)
140{
141 return (pud_val(pud) & _PAGE_PRESENT);
142}
143
144static inline int pud_none(pud_t pud)
145{
146 return (pud_val(pud) == 0);
147}
148
149static inline int pud_bad(pud_t pud)
150{
151 return !pud_present(pud);
152}
153
154#define pud_leaf pud_leaf
155static inline int pud_leaf(pud_t pud)
156{
157 return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF);
158}
159
160static inline int pud_user(pud_t pud)
161{
162 return pud_val(pud) & _PAGE_USER;
163}
164
165static inline void set_pud(pud_t *pudp, pud_t pud)
166{
167 *pudp = pud;
168}
169
170static inline void pud_clear(pud_t *pudp)
171{
172 set_pud(pudp, __pud(0));
173}
174
175static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot)
176{
177 return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
178}
179
180static inline unsigned long _pud_pfn(pud_t pud)
181{
182 return __page_val_to_pfn(pud_val(pud));
183}
184
185static inline pmd_t *pud_pgtable(pud_t pud)
186{
187 return (pmd_t *)pfn_to_virt(__page_val_to_pfn(pud_val(pud)));
188}
189
190static inline struct page *pud_page(pud_t pud)
191{
192 return pfn_to_page(__page_val_to_pfn(pud_val(pud)));
193}
194
195#define mm_p4d_folded mm_p4d_folded
196static inline bool mm_p4d_folded(struct mm_struct *mm)
197{
198 if (pgtable_l5_enabled)
199 return false;
200
201 return true;
202}
203
204#define mm_pud_folded mm_pud_folded
205static inline bool mm_pud_folded(struct mm_struct *mm)
206{
207 if (pgtable_l4_enabled)
208 return false;
209
210 return true;
211}
212
213#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
214
215static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot)
216{
217 unsigned long prot_val = pgprot_val(prot);
218
219 ALT_THEAD_PMA(prot_val);
220
221 return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val);
222}
223
224static inline unsigned long _pmd_pfn(pmd_t pmd)
225{
226 return __page_val_to_pfn(pmd_val(pmd));
227}
228
229#define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot)
230
231#define pmd_ERROR(e) \
232 pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
233
234#define pud_ERROR(e) \
235 pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
236
237#define p4d_ERROR(e) \
238 pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
239
240static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
241{
242 if (pgtable_l4_enabled)
243 *p4dp = p4d;
244 else
245 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) });
246}
247
248static inline int p4d_none(p4d_t p4d)
249{
250 if (pgtable_l4_enabled)
251 return (p4d_val(p4d) == 0);
252
253 return 0;
254}
255
256static inline int p4d_present(p4d_t p4d)
257{
258 if (pgtable_l4_enabled)
259 return (p4d_val(p4d) & _PAGE_PRESENT);
260
261 return 1;
262}
263
264static inline int p4d_bad(p4d_t p4d)
265{
266 if (pgtable_l4_enabled)
267 return !p4d_present(p4d);
268
269 return 0;
270}
271
272static inline void p4d_clear(p4d_t *p4d)
273{
274 if (pgtable_l4_enabled)
275 set_p4d(p4d, __p4d(0));
276}
277
278static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot)
279{
280 return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
281}
282
283static inline unsigned long _p4d_pfn(p4d_t p4d)
284{
285 return __page_val_to_pfn(p4d_val(p4d));
286}
287
288static inline pud_t *p4d_pgtable(p4d_t p4d)
289{
290 if (pgtable_l4_enabled)
291 return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d)));
292
293 return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) });
294}
295#define p4d_page_vaddr(p4d) ((unsigned long)p4d_pgtable(p4d))
296
297static inline struct page *p4d_page(p4d_t p4d)
298{
299 return pfn_to_page(__page_val_to_pfn(p4d_val(p4d)));
300}
301
302#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
303
304#define pud_offset pud_offset
305static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
306{
307 if (pgtable_l4_enabled)
308 return p4d_pgtable(*p4d) + pud_index(address);
309
310 return (pud_t *)p4d;
311}
312
313static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
314{
315 if (pgtable_l5_enabled)
316 *pgdp = pgd;
317 else
318 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) });
319}
320
321static inline int pgd_none(pgd_t pgd)
322{
323 if (pgtable_l5_enabled)
324 return (pgd_val(pgd) == 0);
325
326 return 0;
327}
328
329static inline int pgd_present(pgd_t pgd)
330{
331 if (pgtable_l5_enabled)
332 return (pgd_val(pgd) & _PAGE_PRESENT);
333
334 return 1;
335}
336
337static inline int pgd_bad(pgd_t pgd)
338{
339 if (pgtable_l5_enabled)
340 return !pgd_present(pgd);
341
342 return 0;
343}
344
345static inline void pgd_clear(pgd_t *pgd)
346{
347 if (pgtable_l5_enabled)
348 set_pgd(pgd, __pgd(0));
349}
350
351static inline p4d_t *pgd_pgtable(pgd_t pgd)
352{
353 if (pgtable_l5_enabled)
354 return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd)));
355
356 return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) });
357}
358#define pgd_page_vaddr(pgd) ((unsigned long)pgd_pgtable(pgd))
359
360static inline struct page *pgd_page(pgd_t pgd)
361{
362 return pfn_to_page(__page_val_to_pfn(pgd_val(pgd)));
363}
364#define pgd_page(pgd) pgd_page(pgd)
365
366#define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
367
368#define p4d_offset p4d_offset
369static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
370{
371 if (pgtable_l5_enabled)
372 return pgd_pgtable(*pgd) + p4d_index(address);
373
374 return (p4d_t *)pgd;
375}
376
377#endif /* _ASM_RISCV_PGTABLE_64_H */