Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Re-map IO memory to kernel address space so that we can access it.
4 * This is needed for high PCI addresses that aren't mapped in the
5 * 640k-1MB IO memory area on PC's
6 *
7 * (C) Copyright 1995 1996 Linus Torvalds
8 */
9#include <linux/vmalloc.h>
10#include <linux/mm.h>
11#include <linux/sched.h>
12#include <linux/io.h>
13#include <linux/export.h>
14#include <asm/cacheflush.h>
15
16#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
17static int __read_mostly ioremap_p4d_capable;
18static int __read_mostly ioremap_pud_capable;
19static int __read_mostly ioremap_pmd_capable;
20static int __read_mostly ioremap_huge_disabled;
21
22static int __init set_nohugeiomap(char *str)
23{
24 ioremap_huge_disabled = 1;
25 return 0;
26}
27early_param("nohugeiomap", set_nohugeiomap);
28
29void __init ioremap_huge_init(void)
30{
31 if (!ioremap_huge_disabled) {
32 if (arch_ioremap_p4d_supported())
33 ioremap_p4d_capable = 1;
34 if (arch_ioremap_pud_supported())
35 ioremap_pud_capable = 1;
36 if (arch_ioremap_pmd_supported())
37 ioremap_pmd_capable = 1;
38 }
39}
40
41static inline int ioremap_p4d_enabled(void)
42{
43 return ioremap_p4d_capable;
44}
45
46static inline int ioremap_pud_enabled(void)
47{
48 return ioremap_pud_capable;
49}
50
51static inline int ioremap_pmd_enabled(void)
52{
53 return ioremap_pmd_capable;
54}
55
56#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
57static inline int ioremap_p4d_enabled(void) { return 0; }
58static inline int ioremap_pud_enabled(void) { return 0; }
59static inline int ioremap_pmd_enabled(void) { return 0; }
60#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
61
62static int ioremap_pte_range(pmd_t *pmd, unsigned long addr,
63 unsigned long end, phys_addr_t phys_addr, pgprot_t prot,
64 pgtbl_mod_mask *mask)
65{
66 pte_t *pte;
67 u64 pfn;
68
69 pfn = phys_addr >> PAGE_SHIFT;
70 pte = pte_alloc_kernel_track(pmd, addr, mask);
71 if (!pte)
72 return -ENOMEM;
73 do {
74 BUG_ON(!pte_none(*pte));
75 set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
76 pfn++;
77 } while (pte++, addr += PAGE_SIZE, addr != end);
78 *mask |= PGTBL_PTE_MODIFIED;
79 return 0;
80}
81
82static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr,
83 unsigned long end, phys_addr_t phys_addr,
84 pgprot_t prot)
85{
86 if (!ioremap_pmd_enabled())
87 return 0;
88
89 if ((end - addr) != PMD_SIZE)
90 return 0;
91
92 if (!IS_ALIGNED(addr, PMD_SIZE))
93 return 0;
94
95 if (!IS_ALIGNED(phys_addr, PMD_SIZE))
96 return 0;
97
98 if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr))
99 return 0;
100
101 return pmd_set_huge(pmd, phys_addr, prot);
102}
103
104static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,
105 unsigned long end, phys_addr_t phys_addr, pgprot_t prot,
106 pgtbl_mod_mask *mask)
107{
108 pmd_t *pmd;
109 unsigned long next;
110
111 pmd = pmd_alloc_track(&init_mm, pud, addr, mask);
112 if (!pmd)
113 return -ENOMEM;
114 do {
115 next = pmd_addr_end(addr, end);
116
117 if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot)) {
118 *mask |= PGTBL_PMD_MODIFIED;
119 continue;
120 }
121
122 if (ioremap_pte_range(pmd, addr, next, phys_addr, prot, mask))
123 return -ENOMEM;
124 } while (pmd++, phys_addr += (next - addr), addr = next, addr != end);
125 return 0;
126}
127
128static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr,
129 unsigned long end, phys_addr_t phys_addr,
130 pgprot_t prot)
131{
132 if (!ioremap_pud_enabled())
133 return 0;
134
135 if ((end - addr) != PUD_SIZE)
136 return 0;
137
138 if (!IS_ALIGNED(addr, PUD_SIZE))
139 return 0;
140
141 if (!IS_ALIGNED(phys_addr, PUD_SIZE))
142 return 0;
143
144 if (pud_present(*pud) && !pud_free_pmd_page(pud, addr))
145 return 0;
146
147 return pud_set_huge(pud, phys_addr, prot);
148}
149
150static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,
151 unsigned long end, phys_addr_t phys_addr, pgprot_t prot,
152 pgtbl_mod_mask *mask)
153{
154 pud_t *pud;
155 unsigned long next;
156
157 pud = pud_alloc_track(&init_mm, p4d, addr, mask);
158 if (!pud)
159 return -ENOMEM;
160 do {
161 next = pud_addr_end(addr, end);
162
163 if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot)) {
164 *mask |= PGTBL_PUD_MODIFIED;
165 continue;
166 }
167
168 if (ioremap_pmd_range(pud, addr, next, phys_addr, prot, mask))
169 return -ENOMEM;
170 } while (pud++, phys_addr += (next - addr), addr = next, addr != end);
171 return 0;
172}
173
174static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr,
175 unsigned long end, phys_addr_t phys_addr,
176 pgprot_t prot)
177{
178 if (!ioremap_p4d_enabled())
179 return 0;
180
181 if ((end - addr) != P4D_SIZE)
182 return 0;
183
184 if (!IS_ALIGNED(addr, P4D_SIZE))
185 return 0;
186
187 if (!IS_ALIGNED(phys_addr, P4D_SIZE))
188 return 0;
189
190 if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr))
191 return 0;
192
193 return p4d_set_huge(p4d, phys_addr, prot);
194}
195
196static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,
197 unsigned long end, phys_addr_t phys_addr, pgprot_t prot,
198 pgtbl_mod_mask *mask)
199{
200 p4d_t *p4d;
201 unsigned long next;
202
203 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask);
204 if (!p4d)
205 return -ENOMEM;
206 do {
207 next = p4d_addr_end(addr, end);
208
209 if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) {
210 *mask |= PGTBL_P4D_MODIFIED;
211 continue;
212 }
213
214 if (ioremap_pud_range(p4d, addr, next, phys_addr, prot, mask))
215 return -ENOMEM;
216 } while (p4d++, phys_addr += (next - addr), addr = next, addr != end);
217 return 0;
218}
219
220int ioremap_page_range(unsigned long addr,
221 unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
222{
223 pgd_t *pgd;
224 unsigned long start;
225 unsigned long next;
226 int err;
227 pgtbl_mod_mask mask = 0;
228
229 might_sleep();
230 BUG_ON(addr >= end);
231
232 start = addr;
233 pgd = pgd_offset_k(addr);
234 do {
235 next = pgd_addr_end(addr, end);
236 err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot,
237 &mask);
238 if (err)
239 break;
240 } while (pgd++, phys_addr += (next - addr), addr = next, addr != end);
241
242 flush_cache_vmap(start, end);
243
244 if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
245 arch_sync_kernel_mappings(start, end);
246
247 return err;
248}
249
250#ifdef CONFIG_GENERIC_IOREMAP
251void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot)
252{
253 unsigned long offset, vaddr;
254 phys_addr_t last_addr;
255 struct vm_struct *area;
256
257 /* Disallow wrap-around or zero size */
258 last_addr = addr + size - 1;
259 if (!size || last_addr < addr)
260 return NULL;
261
262 /* Page-align mappings */
263 offset = addr & (~PAGE_MASK);
264 addr -= offset;
265 size = PAGE_ALIGN(size + offset);
266
267 area = get_vm_area_caller(size, VM_IOREMAP,
268 __builtin_return_address(0));
269 if (!area)
270 return NULL;
271 vaddr = (unsigned long)area->addr;
272
273 if (ioremap_page_range(vaddr, vaddr + size, addr, __pgprot(prot))) {
274 free_vm_area(area);
275 return NULL;
276 }
277
278 return (void __iomem *)(vaddr + offset);
279}
280EXPORT_SYMBOL(ioremap_prot);
281
282void iounmap(volatile void __iomem *addr)
283{
284 vunmap((void *)((unsigned long)addr & PAGE_MASK));
285}
286EXPORT_SYMBOL(iounmap);
287#endif /* CONFIG_GENERIC_IOREMAP */