tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (c) 2024, Google LLC.
4 * Pasha Tatashin <pasha.tatashin@soleen.com>
5 */
6
7#ifndef __IOMMU_PAGES_H
8#define __IOMMU_PAGES_H
9
10#include <linux/vmstat.h>
11#include <linux/gfp.h>
12#include <linux/mm.h>
13
14/*
15 * All page allocations that should be reported to as "iommu-pagetables" to
16 * userspace must use one of the functions below. This includes allocations of
17 * page-tables and other per-iommu_domain configuration structures.
18 *
19 * This is necessary for the proper accounting as IOMMU state can be rather
20 * large, i.e. multiple gigabytes in size.
21 */
22
23/**
24 * __iommu_alloc_account - account for newly allocated page.
25 * @page: head struct page of the page.
26 * @order: order of the page
27 */
28static inline void __iommu_alloc_account(struct page *page, int order)
29{
30 const long pgcnt = 1l << order;
31
32 mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt);
33 mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt);
34}
35
36/**
37 * __iommu_free_account - account a page that is about to be freed.
38 * @page: head struct page of the page.
39 * @order: order of the page
40 */
41static inline void __iommu_free_account(struct page *page, int order)
42{
43 const long pgcnt = 1l << order;
44
45 mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt);
46 mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt);
47}
48
49/**
50 * __iommu_alloc_pages - allocate a zeroed page of a given order.
51 * @gfp: buddy allocator flags
52 * @order: page order
53 *
54 * returns the head struct page of the allocated page.
55 */
56static inline struct page *__iommu_alloc_pages(gfp_t gfp, int order)
57{
58 struct page *page;
59
60 page = alloc_pages(gfp | __GFP_ZERO, order);
61 if (unlikely(!page))
62 return NULL;
63
64 __iommu_alloc_account(page, order);
65
66 return page;
67}
68
69/**
70 * __iommu_free_pages - free page of a given order
71 * @page: head struct page of the page
72 * @order: page order
73 */
74static inline void __iommu_free_pages(struct page *page, int order)
75{
76 if (!page)
77 return;
78
79 __iommu_free_account(page, order);
80 __free_pages(page, order);
81}
82
83/**
84 * iommu_alloc_pages_node - allocate a zeroed page of a given order from
85 * specific NUMA node.
86 * @nid: memory NUMA node id
87 * @gfp: buddy allocator flags
88 * @order: page order
89 *
90 * returns the virtual address of the allocated page
91 */
92static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order)
93{
94 struct page *page = alloc_pages_node(nid, gfp | __GFP_ZERO, order);
95
96 if (unlikely(!page))
97 return NULL;
98
99 __iommu_alloc_account(page, order);
100
101 return page_address(page);
102}
103
104/**
105 * iommu_alloc_pages - allocate a zeroed page of a given order
106 * @gfp: buddy allocator flags
107 * @order: page order
108 *
109 * returns the virtual address of the allocated page
110 */
111static inline void *iommu_alloc_pages(gfp_t gfp, int order)
112{
113 struct page *page = __iommu_alloc_pages(gfp, order);
114
115 if (unlikely(!page))
116 return NULL;
117
118 return page_address(page);
119}
120
121/**
122 * iommu_alloc_page_node - allocate a zeroed page at specific NUMA node.
123 * @nid: memory NUMA node id
124 * @gfp: buddy allocator flags
125 *
126 * returns the virtual address of the allocated page
127 */
128static inline void *iommu_alloc_page_node(int nid, gfp_t gfp)
129{
130 return iommu_alloc_pages_node(nid, gfp, 0);
131}
132
133/**
134 * iommu_alloc_page - allocate a zeroed page
135 * @gfp: buddy allocator flags
136 *
137 * returns the virtual address of the allocated page
138 */
139static inline void *iommu_alloc_page(gfp_t gfp)
140{
141 return iommu_alloc_pages(gfp, 0);
142}
143
144/**
145 * iommu_free_pages - free page of a given order
146 * @virt: virtual address of the page to be freed.
147 * @order: page order
148 */
149static inline void iommu_free_pages(void *virt, int order)
150{
151 if (!virt)
152 return;
153
154 __iommu_free_pages(virt_to_page(virt), order);
155}
156
157/**
158 * iommu_free_page - free page
159 * @virt: virtual address of the page to be freed.
160 */
161static inline void iommu_free_page(void *virt)
162{
163 iommu_free_pages(virt, 0);
164}
165
166/**
167 * iommu_put_pages_list - free a list of pages.
168 * @page: the head of the lru list to be freed.
169 *
170 * There are no locking requirement for these pages, as they are going to be
171 * put on a free list as soon as refcount reaches 0. Pages are put on this LRU
172 * list once they are removed from the IOMMU page tables. However, they can
173 * still be access through debugfs.
174 */
175static inline void iommu_put_pages_list(struct list_head *page)
176{
177 while (!list_empty(page)) {
178 struct page *p = list_entry(page->prev, struct page, lru);
179
180 list_del(&p->lru);
181 __iommu_free_account(p, 0);
182 put_page(p);
183 }
184}
185
186#endif /* __IOMMU_PAGES_H */