tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef INT_BLK_MQ_H
3#define INT_BLK_MQ_H
4
5#include "blk-stat.h"
6#include "blk-mq-tag.h"
7
8struct blk_mq_tag_set;
9
10struct blk_mq_ctx {
11 struct {
12 spinlock_t lock;
13 struct list_head rq_list;
14 } ____cacheline_aligned_in_smp;
15
16 unsigned int cpu;
17 unsigned int index_hw;
18
19 /* incremented at dispatch time */
20 unsigned long rq_dispatched[2];
21 unsigned long rq_merged;
22
23 /* incremented at completion time */
24 unsigned long ____cacheline_aligned_in_smp rq_completed[2];
25
26 struct request_queue *queue;
27 struct kobject kobj;
28} ____cacheline_aligned_in_smp;
29
30void blk_mq_freeze_queue(struct request_queue *q);
31void blk_mq_free_queue(struct request_queue *q);
32int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
33void blk_mq_wake_waiters(struct request_queue *q);
34bool blk_mq_dispatch_rq_list(struct request_queue *, struct list_head *, bool);
35void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
36bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
37 bool wait);
38struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
39 struct blk_mq_ctx *start);
40
41/*
42 * Internal helpers for allocating/freeing the request map
43 */
44void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
45 unsigned int hctx_idx);
46void blk_mq_free_rq_map(struct blk_mq_tags *tags);
47struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
48 unsigned int hctx_idx,
49 unsigned int nr_tags,
50 unsigned int reserved_tags);
51int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
52 unsigned int hctx_idx, unsigned int depth);
53
54/*
55 * Internal helpers for request insertion into sw queues
56 */
57void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
58 bool at_head);
59void blk_mq_request_bypass_insert(struct request *rq, bool run_queue);
60void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
61 struct list_head *list);
62
63/*
64 * CPU -> queue mappings
65 */
66extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int);
67
68static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
69 int cpu)
70{
71 return q->queue_hw_ctx[q->mq_map[cpu]];
72}
73
74/*
75 * sysfs helpers
76 */
77extern void blk_mq_sysfs_init(struct request_queue *q);
78extern void blk_mq_sysfs_deinit(struct request_queue *q);
79extern int __blk_mq_register_dev(struct device *dev, struct request_queue *q);
80extern int blk_mq_sysfs_register(struct request_queue *q);
81extern void blk_mq_sysfs_unregister(struct request_queue *q);
82extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
83
84extern void blk_mq_rq_timed_out(struct request *req, bool reserved);
85
86void blk_mq_release(struct request_queue *q);
87
88static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
89 unsigned int cpu)
90{
91 return per_cpu_ptr(q->queue_ctx, cpu);
92}
93
94/*
95 * This assumes per-cpu software queueing queues. They could be per-node
96 * as well, for instance. For now this is hardcoded as-is. Note that we don't
97 * care about preemption, since we know the ctx's are persistent. This does
98 * mean that we can't rely on ctx always matching the currently running CPU.
99 */
100static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
101{
102 return __blk_mq_get_ctx(q, get_cpu());
103}
104
105static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
106{
107 put_cpu();
108}
109
110struct blk_mq_alloc_data {
111 /* input parameter */
112 struct request_queue *q;
113 blk_mq_req_flags_t flags;
114 unsigned int shallow_depth;
115
116 /* input & output parameter */
117 struct blk_mq_ctx *ctx;
118 struct blk_mq_hw_ctx *hctx;
119};
120
121static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data)
122{
123 if (data->flags & BLK_MQ_REQ_INTERNAL)
124 return data->hctx->sched_tags;
125
126 return data->hctx->tags;
127}
128
129static inline bool blk_mq_hctx_stopped(struct blk_mq_hw_ctx *hctx)
130{
131 return test_bit(BLK_MQ_S_STOPPED, &hctx->state);
132}
133
134static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx)
135{
136 return hctx->nr_ctx && hctx->tags;
137}
138
139void blk_mq_in_flight(struct request_queue *q, struct hd_struct *part,
140 unsigned int inflight[2]);
141
142static inline void blk_mq_put_dispatch_budget(struct blk_mq_hw_ctx *hctx)
143{
144 struct request_queue *q = hctx->queue;
145
146 if (q->mq_ops->put_budget)
147 q->mq_ops->put_budget(hctx);
148}
149
150static inline bool blk_mq_get_dispatch_budget(struct blk_mq_hw_ctx *hctx)
151{
152 struct request_queue *q = hctx->queue;
153
154 if (q->mq_ops->get_budget)
155 return q->mq_ops->get_budget(hctx);
156 return true;
157}
158
159static inline void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
160 struct request *rq)
161{
162 blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
163 rq->tag = -1;
164
165 if (rq->rq_flags & RQF_MQ_INFLIGHT) {
166 rq->rq_flags &= ~RQF_MQ_INFLIGHT;
167 atomic_dec(&hctx->nr_active);
168 }
169}
170
171static inline void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx,
172 struct request *rq)
173{
174 if (rq->tag == -1 || rq->internal_tag == -1)
175 return;
176
177 __blk_mq_put_driver_tag(hctx, rq);
178}
179
180static inline void blk_mq_put_driver_tag(struct request *rq)
181{
182 struct blk_mq_hw_ctx *hctx;
183
184 if (rq->tag == -1 || rq->internal_tag == -1)
185 return;
186
187 hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu);
188 __blk_mq_put_driver_tag(hctx, rq);
189}
190
191#endif