Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Re-map IO memory to kernel address space so that we can access it.
4 * This is needed for high PCI addresses that aren't mapped in the
5 * 640k-1MB IO memory area on PC's
6 *
7 * (C) Copyright 1995 1996 Linus Torvalds
8 */
9#include <linux/vmalloc.h>
10#include <linux/mm.h>
11#include <linux/sched.h>
12#include <linux/io.h>
13#include <linux/export.h>
14#include <asm/cacheflush.h>
15#include <asm/pgtable.h>
16
17#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
18static int __read_mostly ioremap_p4d_capable;
19static int __read_mostly ioremap_pud_capable;
20static int __read_mostly ioremap_pmd_capable;
21static int __read_mostly ioremap_huge_disabled;
22
23static int __init set_nohugeiomap(char *str)
24{
25 ioremap_huge_disabled = 1;
26 return 0;
27}
28early_param("nohugeiomap", set_nohugeiomap);
29
30void __init ioremap_huge_init(void)
31{
32 if (!ioremap_huge_disabled) {
33 if (arch_ioremap_pud_supported())
34 ioremap_pud_capable = 1;
35 if (arch_ioremap_pmd_supported())
36 ioremap_pmd_capable = 1;
37 }
38}
39
40static inline int ioremap_p4d_enabled(void)
41{
42 return ioremap_p4d_capable;
43}
44
45static inline int ioremap_pud_enabled(void)
46{
47 return ioremap_pud_capable;
48}
49
50static inline int ioremap_pmd_enabled(void)
51{
52 return ioremap_pmd_capable;
53}
54
55#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
56static inline int ioremap_p4d_enabled(void) { return 0; }
57static inline int ioremap_pud_enabled(void) { return 0; }
58static inline int ioremap_pmd_enabled(void) { return 0; }
59#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
60
61static int ioremap_pte_range(pmd_t *pmd, unsigned long addr,
62 unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
63{
64 pte_t *pte;
65 u64 pfn;
66
67 pfn = phys_addr >> PAGE_SHIFT;
68 pte = pte_alloc_kernel(pmd, addr);
69 if (!pte)
70 return -ENOMEM;
71 do {
72 BUG_ON(!pte_none(*pte));
73 set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
74 pfn++;
75 } while (pte++, addr += PAGE_SIZE, addr != end);
76 return 0;
77}
78
79static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,
80 unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
81{
82 pmd_t *pmd;
83 unsigned long next;
84
85 phys_addr -= addr;
86 pmd = pmd_alloc(&init_mm, pud, addr);
87 if (!pmd)
88 return -ENOMEM;
89 do {
90 next = pmd_addr_end(addr, end);
91
92 if (ioremap_pmd_enabled() &&
93 ((next - addr) == PMD_SIZE) &&
94 IS_ALIGNED(phys_addr + addr, PMD_SIZE)) {
95 if (pmd_set_huge(pmd, phys_addr + addr, prot))
96 continue;
97 }
98
99 if (ioremap_pte_range(pmd, addr, next, phys_addr + addr, prot))
100 return -ENOMEM;
101 } while (pmd++, addr = next, addr != end);
102 return 0;
103}
104
105static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,
106 unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
107{
108 pud_t *pud;
109 unsigned long next;
110
111 phys_addr -= addr;
112 pud = pud_alloc(&init_mm, p4d, addr);
113 if (!pud)
114 return -ENOMEM;
115 do {
116 next = pud_addr_end(addr, end);
117
118 if (ioremap_pud_enabled() &&
119 ((next - addr) == PUD_SIZE) &&
120 IS_ALIGNED(phys_addr + addr, PUD_SIZE)) {
121 if (pud_set_huge(pud, phys_addr + addr, prot))
122 continue;
123 }
124
125 if (ioremap_pmd_range(pud, addr, next, phys_addr + addr, prot))
126 return -ENOMEM;
127 } while (pud++, addr = next, addr != end);
128 return 0;
129}
130
131static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,
132 unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
133{
134 p4d_t *p4d;
135 unsigned long next;
136
137 phys_addr -= addr;
138 p4d = p4d_alloc(&init_mm, pgd, addr);
139 if (!p4d)
140 return -ENOMEM;
141 do {
142 next = p4d_addr_end(addr, end);
143
144 if (ioremap_p4d_enabled() &&
145 ((next - addr) == P4D_SIZE) &&
146 IS_ALIGNED(phys_addr + addr, P4D_SIZE)) {
147 if (p4d_set_huge(p4d, phys_addr + addr, prot))
148 continue;
149 }
150
151 if (ioremap_pud_range(p4d, addr, next, phys_addr + addr, prot))
152 return -ENOMEM;
153 } while (p4d++, addr = next, addr != end);
154 return 0;
155}
156
157int ioremap_page_range(unsigned long addr,
158 unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
159{
160 pgd_t *pgd;
161 unsigned long start;
162 unsigned long next;
163 int err;
164
165 might_sleep();
166 BUG_ON(addr >= end);
167
168 start = addr;
169 phys_addr -= addr;
170 pgd = pgd_offset_k(addr);
171 do {
172 next = pgd_addr_end(addr, end);
173 err = ioremap_p4d_range(pgd, addr, next, phys_addr+addr, prot);
174 if (err)
175 break;
176 } while (pgd++, addr = next, addr != end);
177
178 flush_cache_vmap(start, end);
179
180 return err;
181}