Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* FS-Cache worker operation management routines
3 *
4 * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 *
7 * See Documentation/filesystems/caching/operations.rst
8 */
9
10#define FSCACHE_DEBUG_LEVEL OPERATION
11#include <linux/module.h>
12#include <linux/seq_file.h>
13#include <linux/slab.h>
14#include "internal.h"
15
16atomic_t fscache_op_debug_id;
17EXPORT_SYMBOL(fscache_op_debug_id);
18
19static void fscache_operation_dummy_cancel(struct fscache_operation *op)
20{
21}
22
23/**
24 * fscache_operation_init - Do basic initialisation of an operation
25 * @cookie: The cookie to operate on
26 * @op: The operation to initialise
27 * @processor: The function to perform the operation
28 * @cancel: A function to handle operation cancellation
29 * @release: The release function to assign
30 *
31 * Do basic initialisation of an operation. The caller must still set flags,
32 * object and processor if needed.
33 */
34void fscache_operation_init(struct fscache_cookie *cookie,
35 struct fscache_operation *op,
36 fscache_operation_processor_t processor,
37 fscache_operation_cancel_t cancel,
38 fscache_operation_release_t release)
39{
40 INIT_WORK(&op->work, fscache_op_work_func);
41 atomic_set(&op->usage, 1);
42 op->state = FSCACHE_OP_ST_INITIALISED;
43 op->debug_id = atomic_inc_return(&fscache_op_debug_id);
44 op->processor = processor;
45 op->cancel = cancel ?: fscache_operation_dummy_cancel;
46 op->release = release;
47 INIT_LIST_HEAD(&op->pend_link);
48 fscache_stat(&fscache_n_op_initialised);
49 trace_fscache_op(cookie, op, fscache_op_init);
50}
51EXPORT_SYMBOL(fscache_operation_init);
52
53/**
54 * fscache_enqueue_operation - Enqueue an operation for processing
55 * @op: The operation to enqueue
56 *
57 * Enqueue an operation for processing by the FS-Cache thread pool.
58 *
59 * This will get its own ref on the object.
60 */
61void fscache_enqueue_operation(struct fscache_operation *op)
62{
63 struct fscache_cookie *cookie = op->object->cookie;
64
65 _enter("{OBJ%x OP%x,%u}",
66 op->object->debug_id, op->debug_id, atomic_read(&op->usage));
67
68 ASSERT(list_empty(&op->pend_link));
69 ASSERT(op->processor != NULL);
70 ASSERT(fscache_object_is_available(op->object));
71 ASSERTCMP(atomic_read(&op->usage), >, 0);
72 ASSERTIFCMP(op->state != FSCACHE_OP_ST_IN_PROGRESS,
73 op->state, ==, FSCACHE_OP_ST_CANCELLED);
74
75 fscache_stat(&fscache_n_op_enqueue);
76 switch (op->flags & FSCACHE_OP_TYPE) {
77 case FSCACHE_OP_ASYNC:
78 trace_fscache_op(cookie, op, fscache_op_enqueue_async);
79 _debug("queue async");
80 atomic_inc(&op->usage);
81 if (!queue_work(fscache_op_wq, &op->work))
82 fscache_put_operation(op);
83 break;
84 case FSCACHE_OP_MYTHREAD:
85 trace_fscache_op(cookie, op, fscache_op_enqueue_mythread);
86 _debug("queue for caller's attention");
87 break;
88 default:
89 pr_err("Unexpected op type %lx", op->flags);
90 BUG();
91 break;
92 }
93}
94EXPORT_SYMBOL(fscache_enqueue_operation);
95
96/*
97 * start an op running
98 */
99static void fscache_run_op(struct fscache_object *object,
100 struct fscache_operation *op)
101{
102 ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING);
103
104 op->state = FSCACHE_OP_ST_IN_PROGRESS;
105 object->n_in_progress++;
106 if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags))
107 wake_up_bit(&op->flags, FSCACHE_OP_WAITING);
108 if (op->processor)
109 fscache_enqueue_operation(op);
110 else
111 trace_fscache_op(object->cookie, op, fscache_op_run);
112 fscache_stat(&fscache_n_op_run);
113}
114
115/*
116 * report an unexpected submission
117 */
118static void fscache_report_unexpected_submission(struct fscache_object *object,
119 struct fscache_operation *op,
120 const struct fscache_state *ostate)
121{
122 static bool once_only;
123 struct fscache_operation *p;
124 unsigned n;
125
126 if (once_only)
127 return;
128 once_only = true;
129
130 kdebug("unexpected submission OP%x [OBJ%x %s]",
131 op->debug_id, object->debug_id, object->state->name);
132 kdebug("objstate=%s [%s]", object->state->name, ostate->name);
133 kdebug("objflags=%lx", object->flags);
134 kdebug("objevent=%lx [%lx]", object->events, object->event_mask);
135 kdebug("ops=%u inp=%u exc=%u",
136 object->n_ops, object->n_in_progress, object->n_exclusive);
137
138 if (!list_empty(&object->pending_ops)) {
139 n = 0;
140 list_for_each_entry(p, &object->pending_ops, pend_link) {
141 ASSERTCMP(p->object, ==, object);
142 kdebug("%p %p", op->processor, op->release);
143 n++;
144 }
145
146 kdebug("n=%u", n);
147 }
148
149 dump_stack();
150}
151
152/*
153 * submit an exclusive operation for an object
154 * - other ops are excluded from running simultaneously with this one
155 * - this gets any extra refs it needs on an op
156 */
157int fscache_submit_exclusive_op(struct fscache_object *object,
158 struct fscache_operation *op)
159{
160 const struct fscache_state *ostate;
161 unsigned long flags;
162 int ret;
163
164 _enter("{OBJ%x OP%x},", object->debug_id, op->debug_id);
165
166 trace_fscache_op(object->cookie, op, fscache_op_submit_ex);
167
168 ASSERTCMP(op->state, ==, FSCACHE_OP_ST_INITIALISED);
169 ASSERTCMP(atomic_read(&op->usage), >, 0);
170
171 spin_lock(&object->lock);
172 ASSERTCMP(object->n_ops, >=, object->n_in_progress);
173 ASSERTCMP(object->n_ops, >=, object->n_exclusive);
174 ASSERT(list_empty(&op->pend_link));
175
176 ostate = object->state;
177 smp_rmb();
178
179 op->state = FSCACHE_OP_ST_PENDING;
180 flags = READ_ONCE(object->flags);
181 if (unlikely(!(flags & BIT(FSCACHE_OBJECT_IS_LIVE)))) {
182 fscache_stat(&fscache_n_op_rejected);
183 op->cancel(op);
184 op->state = FSCACHE_OP_ST_CANCELLED;
185 ret = -ENOBUFS;
186 } else if (unlikely(fscache_cache_is_broken(object))) {
187 op->cancel(op);
188 op->state = FSCACHE_OP_ST_CANCELLED;
189 ret = -EIO;
190 } else if (flags & BIT(FSCACHE_OBJECT_IS_AVAILABLE)) {
191 op->object = object;
192 object->n_ops++;
193 object->n_exclusive++; /* reads and writes must wait */
194
195 if (object->n_in_progress > 0) {
196 atomic_inc(&op->usage);
197 list_add_tail(&op->pend_link, &object->pending_ops);
198 fscache_stat(&fscache_n_op_pend);
199 } else if (!list_empty(&object->pending_ops)) {
200 atomic_inc(&op->usage);
201 list_add_tail(&op->pend_link, &object->pending_ops);
202 fscache_stat(&fscache_n_op_pend);
203 fscache_start_operations(object);
204 } else {
205 ASSERTCMP(object->n_in_progress, ==, 0);
206 fscache_run_op(object, op);
207 }
208
209 /* need to issue a new write op after this */
210 clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
211 ret = 0;
212 } else if (flags & BIT(FSCACHE_OBJECT_IS_LOOKED_UP)) {
213 op->object = object;
214 object->n_ops++;
215 object->n_exclusive++; /* reads and writes must wait */
216 atomic_inc(&op->usage);
217 list_add_tail(&op->pend_link, &object->pending_ops);
218 fscache_stat(&fscache_n_op_pend);
219 ret = 0;
220 } else if (flags & BIT(FSCACHE_OBJECT_KILLED_BY_CACHE)) {
221 op->cancel(op);
222 op->state = FSCACHE_OP_ST_CANCELLED;
223 ret = -ENOBUFS;
224 } else {
225 fscache_report_unexpected_submission(object, op, ostate);
226 op->cancel(op);
227 op->state = FSCACHE_OP_ST_CANCELLED;
228 ret = -ENOBUFS;
229 }
230
231 spin_unlock(&object->lock);
232 return ret;
233}
234
235/*
236 * submit an operation for an object
237 * - objects may be submitted only in the following states:
238 * - during object creation (write ops may be submitted)
239 * - whilst the object is active
240 * - after an I/O error incurred in one of the two above states (op rejected)
241 * - this gets any extra refs it needs on an op
242 */
243int fscache_submit_op(struct fscache_object *object,
244 struct fscache_operation *op)
245{
246 const struct fscache_state *ostate;
247 unsigned long flags;
248 int ret;
249
250 _enter("{OBJ%x OP%x},{%u}",
251 object->debug_id, op->debug_id, atomic_read(&op->usage));
252
253 trace_fscache_op(object->cookie, op, fscache_op_submit);
254
255 ASSERTCMP(op->state, ==, FSCACHE_OP_ST_INITIALISED);
256 ASSERTCMP(atomic_read(&op->usage), >, 0);
257
258 spin_lock(&object->lock);
259 ASSERTCMP(object->n_ops, >=, object->n_in_progress);
260 ASSERTCMP(object->n_ops, >=, object->n_exclusive);
261 ASSERT(list_empty(&op->pend_link));
262
263 ostate = object->state;
264 smp_rmb();
265
266 op->state = FSCACHE_OP_ST_PENDING;
267 flags = READ_ONCE(object->flags);
268 if (unlikely(!(flags & BIT(FSCACHE_OBJECT_IS_LIVE)))) {
269 fscache_stat(&fscache_n_op_rejected);
270 op->cancel(op);
271 op->state = FSCACHE_OP_ST_CANCELLED;
272 ret = -ENOBUFS;
273 } else if (unlikely(fscache_cache_is_broken(object))) {
274 op->cancel(op);
275 op->state = FSCACHE_OP_ST_CANCELLED;
276 ret = -EIO;
277 } else if (flags & BIT(FSCACHE_OBJECT_IS_AVAILABLE)) {
278 op->object = object;
279 object->n_ops++;
280
281 if (object->n_exclusive > 0) {
282 atomic_inc(&op->usage);
283 list_add_tail(&op->pend_link, &object->pending_ops);
284 fscache_stat(&fscache_n_op_pend);
285 } else if (!list_empty(&object->pending_ops)) {
286 atomic_inc(&op->usage);
287 list_add_tail(&op->pend_link, &object->pending_ops);
288 fscache_stat(&fscache_n_op_pend);
289 fscache_start_operations(object);
290 } else {
291 ASSERTCMP(object->n_exclusive, ==, 0);
292 fscache_run_op(object, op);
293 }
294 ret = 0;
295 } else if (flags & BIT(FSCACHE_OBJECT_IS_LOOKED_UP)) {
296 op->object = object;
297 object->n_ops++;
298 atomic_inc(&op->usage);
299 list_add_tail(&op->pend_link, &object->pending_ops);
300 fscache_stat(&fscache_n_op_pend);
301 ret = 0;
302 } else if (flags & BIT(FSCACHE_OBJECT_KILLED_BY_CACHE)) {
303 op->cancel(op);
304 op->state = FSCACHE_OP_ST_CANCELLED;
305 ret = -ENOBUFS;
306 } else {
307 fscache_report_unexpected_submission(object, op, ostate);
308 ASSERT(!fscache_object_is_active(object));
309 op->cancel(op);
310 op->state = FSCACHE_OP_ST_CANCELLED;
311 ret = -ENOBUFS;
312 }
313
314 spin_unlock(&object->lock);
315 return ret;
316}
317
318/*
319 * queue an object for withdrawal on error, aborting all following asynchronous
320 * operations
321 */
322void fscache_abort_object(struct fscache_object *object)
323{
324 _enter("{OBJ%x}", object->debug_id);
325
326 fscache_raise_event(object, FSCACHE_OBJECT_EV_ERROR);
327}
328
329/*
330 * Jump start the operation processing on an object. The caller must hold
331 * object->lock.
332 */
333void fscache_start_operations(struct fscache_object *object)
334{
335 struct fscache_operation *op;
336 bool stop = false;
337
338 while (!list_empty(&object->pending_ops) && !stop) {
339 op = list_entry(object->pending_ops.next,
340 struct fscache_operation, pend_link);
341
342 if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags)) {
343 if (object->n_in_progress > 0)
344 break;
345 stop = true;
346 }
347 list_del_init(&op->pend_link);
348 fscache_run_op(object, op);
349
350 /* the pending queue was holding a ref on the object */
351 fscache_put_operation(op);
352 }
353
354 ASSERTCMP(object->n_in_progress, <=, object->n_ops);
355
356 _debug("woke %d ops on OBJ%x",
357 object->n_in_progress, object->debug_id);
358}
359
360/*
361 * cancel an operation that's pending on an object
362 */
363int fscache_cancel_op(struct fscache_operation *op,
364 bool cancel_in_progress_op)
365{
366 struct fscache_object *object = op->object;
367 bool put = false;
368 int ret;
369
370 _enter("OBJ%x OP%x}", op->object->debug_id, op->debug_id);
371
372 trace_fscache_op(object->cookie, op, fscache_op_cancel);
373
374 ASSERTCMP(op->state, >=, FSCACHE_OP_ST_PENDING);
375 ASSERTCMP(op->state, !=, FSCACHE_OP_ST_CANCELLED);
376 ASSERTCMP(atomic_read(&op->usage), >, 0);
377
378 spin_lock(&object->lock);
379
380 ret = -EBUSY;
381 if (op->state == FSCACHE_OP_ST_PENDING) {
382 ASSERT(!list_empty(&op->pend_link));
383 list_del_init(&op->pend_link);
384 put = true;
385
386 fscache_stat(&fscache_n_op_cancelled);
387 op->cancel(op);
388 op->state = FSCACHE_OP_ST_CANCELLED;
389 if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags))
390 object->n_exclusive--;
391 if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags))
392 wake_up_bit(&op->flags, FSCACHE_OP_WAITING);
393 ret = 0;
394 } else if (op->state == FSCACHE_OP_ST_IN_PROGRESS && cancel_in_progress_op) {
395 ASSERTCMP(object->n_in_progress, >, 0);
396 if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags))
397 object->n_exclusive--;
398 object->n_in_progress--;
399 if (object->n_in_progress == 0)
400 fscache_start_operations(object);
401
402 fscache_stat(&fscache_n_op_cancelled);
403 op->cancel(op);
404 op->state = FSCACHE_OP_ST_CANCELLED;
405 if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags))
406 object->n_exclusive--;
407 if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags))
408 wake_up_bit(&op->flags, FSCACHE_OP_WAITING);
409 ret = 0;
410 }
411
412 if (put)
413 fscache_put_operation(op);
414 spin_unlock(&object->lock);
415 _leave(" = %d", ret);
416 return ret;
417}
418
419/*
420 * Cancel all pending operations on an object
421 */
422void fscache_cancel_all_ops(struct fscache_object *object)
423{
424 struct fscache_operation *op;
425
426 _enter("OBJ%x", object->debug_id);
427
428 spin_lock(&object->lock);
429
430 while (!list_empty(&object->pending_ops)) {
431 op = list_entry(object->pending_ops.next,
432 struct fscache_operation, pend_link);
433 fscache_stat(&fscache_n_op_cancelled);
434 list_del_init(&op->pend_link);
435
436 trace_fscache_op(object->cookie, op, fscache_op_cancel_all);
437
438 ASSERTCMP(op->state, ==, FSCACHE_OP_ST_PENDING);
439 op->cancel(op);
440 op->state = FSCACHE_OP_ST_CANCELLED;
441
442 if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags))
443 object->n_exclusive--;
444 if (test_and_clear_bit(FSCACHE_OP_WAITING, &op->flags))
445 wake_up_bit(&op->flags, FSCACHE_OP_WAITING);
446 fscache_put_operation(op);
447 cond_resched_lock(&object->lock);
448 }
449
450 spin_unlock(&object->lock);
451 _leave("");
452}
453
454/*
455 * Record the completion or cancellation of an in-progress operation.
456 */
457void fscache_op_complete(struct fscache_operation *op, bool cancelled)
458{
459 struct fscache_object *object = op->object;
460
461 _enter("OBJ%x", object->debug_id);
462
463 ASSERTCMP(op->state, ==, FSCACHE_OP_ST_IN_PROGRESS);
464 ASSERTCMP(object->n_in_progress, >, 0);
465 ASSERTIFCMP(test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags),
466 object->n_exclusive, >, 0);
467 ASSERTIFCMP(test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags),
468 object->n_in_progress, ==, 1);
469
470 spin_lock(&object->lock);
471
472 if (!cancelled) {
473 trace_fscache_op(object->cookie, op, fscache_op_completed);
474 op->state = FSCACHE_OP_ST_COMPLETE;
475 } else {
476 op->cancel(op);
477 trace_fscache_op(object->cookie, op, fscache_op_cancelled);
478 op->state = FSCACHE_OP_ST_CANCELLED;
479 }
480
481 if (test_bit(FSCACHE_OP_EXCLUSIVE, &op->flags))
482 object->n_exclusive--;
483 object->n_in_progress--;
484 if (object->n_in_progress == 0)
485 fscache_start_operations(object);
486
487 spin_unlock(&object->lock);
488 _leave("");
489}
490EXPORT_SYMBOL(fscache_op_complete);
491
492/*
493 * release an operation
494 * - queues pending ops if this is the last in-progress op
495 */
496void fscache_put_operation(struct fscache_operation *op)
497{
498 struct fscache_object *object;
499 struct fscache_cache *cache;
500
501 _enter("{OBJ%x OP%x,%d}",
502 op->object ? op->object->debug_id : 0,
503 op->debug_id, atomic_read(&op->usage));
504
505 ASSERTCMP(atomic_read(&op->usage), >, 0);
506
507 if (!atomic_dec_and_test(&op->usage))
508 return;
509
510 trace_fscache_op(op->object ? op->object->cookie : NULL, op, fscache_op_put);
511
512 _debug("PUT OP");
513 ASSERTIFCMP(op->state != FSCACHE_OP_ST_INITIALISED &&
514 op->state != FSCACHE_OP_ST_COMPLETE,
515 op->state, ==, FSCACHE_OP_ST_CANCELLED);
516
517 fscache_stat(&fscache_n_op_release);
518
519 if (op->release) {
520 op->release(op);
521 op->release = NULL;
522 }
523 op->state = FSCACHE_OP_ST_DEAD;
524
525 object = op->object;
526 if (likely(object)) {
527 if (test_bit(FSCACHE_OP_DEC_READ_CNT, &op->flags))
528 atomic_dec(&object->n_reads);
529 if (test_bit(FSCACHE_OP_UNUSE_COOKIE, &op->flags))
530 fscache_unuse_cookie(object);
531
532 /* now... we may get called with the object spinlock held, so we
533 * complete the cleanup here only if we can immediately acquire the
534 * lock, and defer it otherwise */
535 if (!spin_trylock(&object->lock)) {
536 _debug("defer put");
537 fscache_stat(&fscache_n_op_deferred_release);
538
539 cache = object->cache;
540 spin_lock(&cache->op_gc_list_lock);
541 list_add_tail(&op->pend_link, &cache->op_gc_list);
542 spin_unlock(&cache->op_gc_list_lock);
543 schedule_work(&cache->op_gc);
544 _leave(" [defer]");
545 return;
546 }
547
548 ASSERTCMP(object->n_ops, >, 0);
549 object->n_ops--;
550 if (object->n_ops == 0)
551 fscache_raise_event(object, FSCACHE_OBJECT_EV_CLEARED);
552
553 spin_unlock(&object->lock);
554 }
555
556 kfree(op);
557 _leave(" [done]");
558}
559EXPORT_SYMBOL(fscache_put_operation);
560
561/*
562 * garbage collect operations that have had their release deferred
563 */
564void fscache_operation_gc(struct work_struct *work)
565{
566 struct fscache_operation *op;
567 struct fscache_object *object;
568 struct fscache_cache *cache =
569 container_of(work, struct fscache_cache, op_gc);
570 int count = 0;
571
572 _enter("");
573
574 do {
575 spin_lock(&cache->op_gc_list_lock);
576 if (list_empty(&cache->op_gc_list)) {
577 spin_unlock(&cache->op_gc_list_lock);
578 break;
579 }
580
581 op = list_entry(cache->op_gc_list.next,
582 struct fscache_operation, pend_link);
583 list_del(&op->pend_link);
584 spin_unlock(&cache->op_gc_list_lock);
585
586 object = op->object;
587 trace_fscache_op(object->cookie, op, fscache_op_gc);
588
589 spin_lock(&object->lock);
590
591 _debug("GC DEFERRED REL OBJ%x OP%x",
592 object->debug_id, op->debug_id);
593 fscache_stat(&fscache_n_op_gc);
594
595 ASSERTCMP(atomic_read(&op->usage), ==, 0);
596 ASSERTCMP(op->state, ==, FSCACHE_OP_ST_DEAD);
597
598 ASSERTCMP(object->n_ops, >, 0);
599 object->n_ops--;
600 if (object->n_ops == 0)
601 fscache_raise_event(object, FSCACHE_OBJECT_EV_CLEARED);
602
603 spin_unlock(&object->lock);
604 kfree(op);
605
606 } while (count++ < 20);
607
608 if (!list_empty(&cache->op_gc_list))
609 schedule_work(&cache->op_gc);
610
611 _leave("");
612}
613
614/*
615 * execute an operation using fs_op_wq to provide processing context -
616 * the caller holds a ref to this object, so we don't need to hold one
617 */
618void fscache_op_work_func(struct work_struct *work)
619{
620 struct fscache_operation *op =
621 container_of(work, struct fscache_operation, work);
622
623 _enter("{OBJ%x OP%x,%d}",
624 op->object->debug_id, op->debug_id, atomic_read(&op->usage));
625
626 trace_fscache_op(op->object->cookie, op, fscache_op_work);
627
628 ASSERT(op->processor != NULL);
629 op->processor(op);
630 fscache_put_operation(op);
631
632 _leave("");
633}