tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#ifndef _IDXD_H_
4#define _IDXD_H_
5
6#include <linux/sbitmap.h>
7#include <linux/dmaengine.h>
8#include <linux/percpu-rwsem.h>
9#include <linux/wait.h>
10#include <linux/cdev.h>
11#include "registers.h"
12
13#define IDXD_DRIVER_VERSION "1.00"
14
15extern struct kmem_cache *idxd_desc_pool;
16
17#define IDXD_REG_TIMEOUT 50
18#define IDXD_DRAIN_TIMEOUT 5000
19
20enum idxd_type {
21 IDXD_TYPE_UNKNOWN = -1,
22 IDXD_TYPE_DSA = 0,
23 IDXD_TYPE_MAX
24};
25
26#define IDXD_NAME_SIZE 128
27
28struct idxd_device_driver {
29 struct device_driver drv;
30};
31
32struct idxd_irq_entry {
33 struct idxd_device *idxd;
34 int id;
35 struct llist_head pending_llist;
36 struct list_head work_list;
37};
38
39struct idxd_group {
40 struct device conf_dev;
41 struct idxd_device *idxd;
42 struct grpcfg grpcfg;
43 int id;
44 int num_engines;
45 int num_wqs;
46 bool use_token_limit;
47 u8 tokens_allowed;
48 u8 tokens_reserved;
49 int tc_a;
50 int tc_b;
51};
52
53#define IDXD_MAX_PRIORITY 0xf
54
55enum idxd_wq_state {
56 IDXD_WQ_DISABLED = 0,
57 IDXD_WQ_ENABLED,
58};
59
60enum idxd_wq_flag {
61 WQ_FLAG_DEDICATED = 0,
62};
63
64enum idxd_wq_type {
65 IDXD_WQT_NONE = 0,
66 IDXD_WQT_KERNEL,
67 IDXD_WQT_USER,
68};
69
70struct idxd_cdev {
71 struct cdev cdev;
72 struct device *dev;
73 int minor;
74 struct wait_queue_head err_queue;
75};
76
77#define IDXD_ALLOCATED_BATCH_SIZE 128U
78#define WQ_NAME_SIZE 1024
79#define WQ_TYPE_SIZE 10
80
81enum idxd_op_type {
82 IDXD_OP_BLOCK = 0,
83 IDXD_OP_NONBLOCK = 1,
84};
85
86enum idxd_complete_type {
87 IDXD_COMPLETE_NORMAL = 0,
88 IDXD_COMPLETE_ABORT,
89};
90
91struct idxd_wq {
92 void __iomem *dportal;
93 struct device conf_dev;
94 struct idxd_cdev idxd_cdev;
95 struct idxd_device *idxd;
96 int id;
97 enum idxd_wq_type type;
98 struct idxd_group *group;
99 int client_count;
100 struct mutex wq_lock; /* mutex for workqueue */
101 u32 size;
102 u32 threshold;
103 u32 priority;
104 enum idxd_wq_state state;
105 unsigned long flags;
106 union wqcfg wqcfg;
107 atomic_t dq_count; /* dedicated queue flow control */
108 u32 vec_ptr; /* interrupt steering */
109 struct dsa_hw_desc **hw_descs;
110 int num_descs;
111 struct dsa_completion_record *compls;
112 dma_addr_t compls_addr;
113 int compls_size;
114 struct idxd_desc **descs;
115 struct sbitmap sbmap;
116 struct dma_chan dma_chan;
117 struct percpu_rw_semaphore submit_lock;
118 wait_queue_head_t submit_waitq;
119 char name[WQ_NAME_SIZE + 1];
120};
121
122struct idxd_engine {
123 struct device conf_dev;
124 int id;
125 struct idxd_group *group;
126 struct idxd_device *idxd;
127};
128
129/* shadow registers */
130struct idxd_hw {
131 u32 version;
132 union gen_cap_reg gen_cap;
133 union wq_cap_reg wq_cap;
134 union group_cap_reg group_cap;
135 union engine_cap_reg engine_cap;
136 struct opcap opcap;
137};
138
139enum idxd_device_state {
140 IDXD_DEV_HALTED = -1,
141 IDXD_DEV_DISABLED = 0,
142 IDXD_DEV_CONF_READY,
143 IDXD_DEV_ENABLED,
144};
145
146enum idxd_device_flag {
147 IDXD_FLAG_CONFIGURABLE = 0,
148};
149
150struct idxd_device {
151 enum idxd_type type;
152 struct device conf_dev;
153 struct list_head list;
154 struct idxd_hw hw;
155 enum idxd_device_state state;
156 unsigned long flags;
157 int id;
158 int major;
159
160 struct pci_dev *pdev;
161 void __iomem *reg_base;
162
163 spinlock_t dev_lock; /* spinlock for device */
164 struct idxd_group *groups;
165 struct idxd_wq *wqs;
166 struct idxd_engine *engines;
167
168 int num_groups;
169
170 u32 msix_perm_offset;
171 u32 wqcfg_offset;
172 u32 grpcfg_offset;
173 u32 perfmon_offset;
174
175 u64 max_xfer_bytes;
176 u32 max_batch_size;
177 int max_groups;
178 int max_engines;
179 int max_tokens;
180 int max_wqs;
181 int max_wq_size;
182 int token_limit;
183 int nr_tokens; /* non-reserved tokens */
184
185 union sw_err_reg sw_err;
186
187 struct msix_entry *msix_entries;
188 int num_wq_irqs;
189 struct idxd_irq_entry *irq_entries;
190
191 struct dma_device dma_dev;
192};
193
194/* IDXD software descriptor */
195struct idxd_desc {
196 struct dsa_hw_desc *hw;
197 dma_addr_t desc_dma;
198 struct dsa_completion_record *completion;
199 dma_addr_t compl_dma;
200 struct dma_async_tx_descriptor txd;
201 struct llist_node llnode;
202 struct list_head list;
203 int id;
204 struct idxd_wq *wq;
205};
206
207#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
208#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
209
210extern struct bus_type dsa_bus_type;
211
212static inline bool wq_dedicated(struct idxd_wq *wq)
213{
214 return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
215}
216
217enum idxd_portal_prot {
218 IDXD_PORTAL_UNLIMITED = 0,
219 IDXD_PORTAL_LIMITED,
220};
221
222static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
223{
224 return prot * 0x1000;
225}
226
227static inline int idxd_get_wq_portal_full_offset(int wq_id,
228 enum idxd_portal_prot prot)
229{
230 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
231}
232
233static inline void idxd_set_type(struct idxd_device *idxd)
234{
235 struct pci_dev *pdev = idxd->pdev;
236
237 if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
238 idxd->type = IDXD_TYPE_DSA;
239 else
240 idxd->type = IDXD_TYPE_UNKNOWN;
241}
242
243static inline void idxd_wq_get(struct idxd_wq *wq)
244{
245 wq->client_count++;
246}
247
248static inline void idxd_wq_put(struct idxd_wq *wq)
249{
250 wq->client_count--;
251}
252
253static inline int idxd_wq_refcount(struct idxd_wq *wq)
254{
255 return wq->client_count;
256};
257
258const char *idxd_get_dev_name(struct idxd_device *idxd);
259int idxd_register_bus_type(void);
260void idxd_unregister_bus_type(void);
261int idxd_setup_sysfs(struct idxd_device *idxd);
262void idxd_cleanup_sysfs(struct idxd_device *idxd);
263int idxd_register_driver(void);
264void idxd_unregister_driver(void);
265struct bus_type *idxd_get_bus_type(struct idxd_device *idxd);
266
267/* device interrupt control */
268irqreturn_t idxd_irq_handler(int vec, void *data);
269irqreturn_t idxd_misc_thread(int vec, void *data);
270irqreturn_t idxd_wq_thread(int irq, void *data);
271void idxd_mask_error_interrupts(struct idxd_device *idxd);
272void idxd_unmask_error_interrupts(struct idxd_device *idxd);
273void idxd_mask_msix_vectors(struct idxd_device *idxd);
274int idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
275int idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
276
277/* device control */
278int idxd_device_enable(struct idxd_device *idxd);
279int idxd_device_disable(struct idxd_device *idxd);
280int idxd_device_reset(struct idxd_device *idxd);
281int __idxd_device_reset(struct idxd_device *idxd);
282void idxd_device_cleanup(struct idxd_device *idxd);
283int idxd_device_config(struct idxd_device *idxd);
284void idxd_device_wqs_clear_state(struct idxd_device *idxd);
285
286/* work queue control */
287int idxd_wq_alloc_resources(struct idxd_wq *wq);
288void idxd_wq_free_resources(struct idxd_wq *wq);
289int idxd_wq_enable(struct idxd_wq *wq);
290int idxd_wq_disable(struct idxd_wq *wq);
291int idxd_wq_map_portal(struct idxd_wq *wq);
292void idxd_wq_unmap_portal(struct idxd_wq *wq);
293
294/* submission */
295int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
296struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
297void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
298
299/* dmaengine */
300int idxd_register_dma_device(struct idxd_device *idxd);
301void idxd_unregister_dma_device(struct idxd_device *idxd);
302int idxd_register_dma_channel(struct idxd_wq *wq);
303void idxd_unregister_dma_channel(struct idxd_wq *wq);
304void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
305void idxd_dma_complete_txd(struct idxd_desc *desc,
306 enum idxd_complete_type comp_type);
307dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx);
308
309/* cdev */
310int idxd_cdev_register(void);
311void idxd_cdev_remove(void);
312int idxd_cdev_get_major(struct idxd_device *idxd);
313int idxd_wq_add_cdev(struct idxd_wq *wq);
314void idxd_wq_del_cdev(struct idxd_wq *wq);
315
316#endif