Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/* Queue of folios definitions
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 *
7 * See:
8 *
9 * Documentation/core-api/folio_queue.rst
10 *
11 * for a description of the API.
12 */
13
14#ifndef _LINUX_FOLIO_QUEUE_H
15#define _LINUX_FOLIO_QUEUE_H
16
17#include <linux/pagevec.h>
18#include <linux/mm.h>
19
20/*
21 * Segment in a queue of running buffers. Each segment can hold a number of
22 * folios and a portion of the queue can be referenced with the ITER_FOLIOQ
23 * iterator. The possibility exists of inserting non-folio elements into the
24 * queue (such as gaps).
25 *
26 * Explicit prev and next pointers are used instead of a list_head to make it
27 * easier to add segments to tail and remove them from the head without the
28 * need for a lock.
29 */
30struct folio_queue {
31 struct folio_batch vec; /* Folios in the queue segment */
32 u8 orders[PAGEVEC_SIZE]; /* Order of each folio */
33 struct folio_queue *next; /* Next queue segment or NULL */
34 struct folio_queue *prev; /* Previous queue segment of NULL */
35 unsigned long marks; /* 1-bit mark per folio */
36 unsigned long marks2; /* Second 1-bit mark per folio */
37 unsigned long marks3; /* Third 1-bit mark per folio */
38#if PAGEVEC_SIZE > BITS_PER_LONG
39#error marks is not big enough
40#endif
41 unsigned int rreq_id;
42 unsigned int debug_id;
43};
44
45/**
46 * folioq_init - Initialise a folio queue segment
47 * @folioq: The segment to initialise
48 * @rreq_id: The request identifier to use in tracelines.
49 *
50 * Initialise a folio queue segment and set an identifier to be used in traces.
51 *
52 * Note that the folio pointers are left uninitialised.
53 */
54static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id)
55{
56 folio_batch_init(&folioq->vec);
57 folioq->next = NULL;
58 folioq->prev = NULL;
59 folioq->marks = 0;
60 folioq->marks2 = 0;
61 folioq->marks3 = 0;
62 folioq->rreq_id = rreq_id;
63 folioq->debug_id = 0;
64}
65
66/**
67 * folioq_nr_slots: Query the capacity of a folio queue segment
68 * @folioq: The segment to query
69 *
70 * Query the number of folios that a particular folio queue segment might hold.
71 * [!] NOTE: This must not be assumed to be the same for every segment!
72 */
73static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
74{
75 return PAGEVEC_SIZE;
76}
77
78/**
79 * folioq_count: Query the occupancy of a folio queue segment
80 * @folioq: The segment to query
81 *
82 * Query the number of folios that have been added to a folio queue segment.
83 * Note that this is not decreased as folios are removed from a segment.
84 */
85static inline unsigned int folioq_count(struct folio_queue *folioq)
86{
87 return folio_batch_count(&folioq->vec);
88}
89
90/**
91 * folioq_full: Query if a folio queue segment is full
92 * @folioq: The segment to query
93 *
94 * Query if a folio queue segment is fully occupied. Note that this does not
95 * change if folios are removed from a segment.
96 */
97static inline bool folioq_full(struct folio_queue *folioq)
98{
99 //return !folio_batch_space(&folioq->vec);
100 return folioq_count(folioq) >= folioq_nr_slots(folioq);
101}
102
103/**
104 * folioq_is_marked: Check first folio mark in a folio queue segment
105 * @folioq: The segment to query
106 * @slot: The slot number of the folio to query
107 *
108 * Determine if the first mark is set for the folio in the specified slot in a
109 * folio queue segment.
110 */
111static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot)
112{
113 return test_bit(slot, &folioq->marks);
114}
115
116/**
117 * folioq_mark: Set the first mark on a folio in a folio queue segment
118 * @folioq: The segment to modify
119 * @slot: The slot number of the folio to modify
120 *
121 * Set the first mark for the folio in the specified slot in a folio queue
122 * segment.
123 */
124static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot)
125{
126 set_bit(slot, &folioq->marks);
127}
128
129/**
130 * folioq_unmark: Clear the first mark on a folio in a folio queue segment
131 * @folioq: The segment to modify
132 * @slot: The slot number of the folio to modify
133 *
134 * Clear the first mark for the folio in the specified slot in a folio queue
135 * segment.
136 */
137static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot)
138{
139 clear_bit(slot, &folioq->marks);
140}
141
142/**
143 * folioq_is_marked2: Check second folio mark in a folio queue segment
144 * @folioq: The segment to query
145 * @slot: The slot number of the folio to query
146 *
147 * Determine if the second mark is set for the folio in the specified slot in a
148 * folio queue segment.
149 */
150static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot)
151{
152 return test_bit(slot, &folioq->marks2);
153}
154
155/**
156 * folioq_mark2: Set the second mark on a folio in a folio queue segment
157 * @folioq: The segment to modify
158 * @slot: The slot number of the folio to modify
159 *
160 * Set the second mark for the folio in the specified slot in a folio queue
161 * segment.
162 */
163static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot)
164{
165 set_bit(slot, &folioq->marks2);
166}
167
168/**
169 * folioq_unmark2: Clear the second mark on a folio in a folio queue segment
170 * @folioq: The segment to modify
171 * @slot: The slot number of the folio to modify
172 *
173 * Clear the second mark for the folio in the specified slot in a folio queue
174 * segment.
175 */
176static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot)
177{
178 clear_bit(slot, &folioq->marks2);
179}
180
181/**
182 * folioq_is_marked3: Check third folio mark in a folio queue segment
183 * @folioq: The segment to query
184 * @slot: The slot number of the folio to query
185 *
186 * Determine if the third mark is set for the folio in the specified slot in a
187 * folio queue segment.
188 */
189static inline bool folioq_is_marked3(const struct folio_queue *folioq, unsigned int slot)
190{
191 return test_bit(slot, &folioq->marks3);
192}
193
194/**
195 * folioq_mark3: Set the third mark on a folio in a folio queue segment
196 * @folioq: The segment to modify
197 * @slot: The slot number of the folio to modify
198 *
199 * Set the third mark for the folio in the specified slot in a folio queue
200 * segment.
201 */
202static inline void folioq_mark3(struct folio_queue *folioq, unsigned int slot)
203{
204 set_bit(slot, &folioq->marks3);
205}
206
207/**
208 * folioq_unmark3: Clear the third mark on a folio in a folio queue segment
209 * @folioq: The segment to modify
210 * @slot: The slot number of the folio to modify
211 *
212 * Clear the third mark for the folio in the specified slot in a folio queue
213 * segment.
214 */
215static inline void folioq_unmark3(struct folio_queue *folioq, unsigned int slot)
216{
217 clear_bit(slot, &folioq->marks3);
218}
219
220/**
221 * folioq_append: Add a folio to a folio queue segment
222 * @folioq: The segment to add to
223 * @folio: The folio to add
224 *
225 * Add a folio to the tail of the sequence in a folio queue segment, increasing
226 * the occupancy count and returning the slot number for the folio just added.
227 * The folio size is extracted and stored in the queue and the marks are left
228 * unmodified.
229 *
230 * Note that it's left up to the caller to check that the segment capacity will
231 * not be exceeded and to extend the queue.
232 */
233static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio)
234{
235 unsigned int slot = folioq->vec.nr++;
236
237 folioq->vec.folios[slot] = folio;
238 folioq->orders[slot] = folio_order(folio);
239 return slot;
240}
241
242/**
243 * folioq_append_mark: Add a folio to a folio queue segment
244 * @folioq: The segment to add to
245 * @folio: The folio to add
246 *
247 * Add a folio to the tail of the sequence in a folio queue segment, increasing
248 * the occupancy count and returning the slot number for the folio just added.
249 * The folio size is extracted and stored in the queue, the first mark is set
250 * and and the second and third marks are left unmodified.
251 *
252 * Note that it's left up to the caller to check that the segment capacity will
253 * not be exceeded and to extend the queue.
254 */
255static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio)
256{
257 unsigned int slot = folioq->vec.nr++;
258
259 folioq->vec.folios[slot] = folio;
260 folioq->orders[slot] = folio_order(folio);
261 folioq_mark(folioq, slot);
262 return slot;
263}
264
265/**
266 * folioq_folio: Get a folio from a folio queue segment
267 * @folioq: The segment to access
268 * @slot: The folio slot to access
269 *
270 * Retrieve the folio in the specified slot from a folio queue segment. Note
271 * that no bounds check is made and if the slot hasn't been added into yet, the
272 * pointer will be undefined. If the slot has been cleared, NULL will be
273 * returned.
274 */
275static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot)
276{
277 return folioq->vec.folios[slot];
278}
279
280/**
281 * folioq_folio_order: Get the order of a folio from a folio queue segment
282 * @folioq: The segment to access
283 * @slot: The folio slot to access
284 *
285 * Retrieve the order of the folio in the specified slot from a folio queue
286 * segment. Note that no bounds check is made and if the slot hasn't been
287 * added into yet, the order returned will be 0.
288 */
289static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot)
290{
291 return folioq->orders[slot];
292}
293
294/**
295 * folioq_folio_size: Get the size of a folio from a folio queue segment
296 * @folioq: The segment to access
297 * @slot: The folio slot to access
298 *
299 * Retrieve the size of the folio in the specified slot from a folio queue
300 * segment. Note that no bounds check is made and if the slot hasn't been
301 * added into yet, the size returned will be PAGE_SIZE.
302 */
303static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot)
304{
305 return PAGE_SIZE << folioq_folio_order(folioq, slot);
306}
307
308/**
309 * folioq_clear: Clear a folio from a folio queue segment
310 * @folioq: The segment to clear
311 * @slot: The folio slot to clear
312 *
313 * Clear a folio from a sequence in a folio queue segment and clear its marks.
314 * The occupancy count is left unchanged.
315 */
316static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot)
317{
318 folioq->vec.folios[slot] = NULL;
319 folioq_unmark(folioq, slot);
320 folioq_unmark2(folioq, slot);
321 folioq_unmark3(folioq, slot);
322}
323
324#endif /* _LINUX_FOLIO_QUEUE_H */