tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#ifndef _IDXD_H_
4#define _IDXD_H_
5
6#include <linux/sbitmap.h>
7#include <linux/dmaengine.h>
8#include <linux/percpu-rwsem.h>
9#include <linux/wait.h>
10#include <linux/cdev.h>
11#include <linux/idr.h>
12#include <linux/pci.h>
13#include <linux/ioasid.h>
14#include <linux/perf_event.h>
15#include <uapi/linux/idxd.h>
16#include "registers.h"
17
18#define IDXD_DRIVER_VERSION "1.00"
19
20extern struct kmem_cache *idxd_desc_pool;
21extern bool tc_override;
22
23struct idxd_wq;
24struct idxd_dev;
25
26enum idxd_dev_type {
27 IDXD_DEV_NONE = -1,
28 IDXD_DEV_DSA = 0,
29 IDXD_DEV_IAX,
30 IDXD_DEV_WQ,
31 IDXD_DEV_GROUP,
32 IDXD_DEV_ENGINE,
33 IDXD_DEV_CDEV,
34 IDXD_DEV_MAX_TYPE,
35};
36
37struct idxd_dev {
38 struct device conf_dev;
39 enum idxd_dev_type type;
40};
41
42#define IDXD_REG_TIMEOUT 50
43#define IDXD_DRAIN_TIMEOUT 5000
44
45enum idxd_type {
46 IDXD_TYPE_UNKNOWN = -1,
47 IDXD_TYPE_DSA = 0,
48 IDXD_TYPE_IAX,
49 IDXD_TYPE_MAX,
50};
51
52#define IDXD_NAME_SIZE 128
53#define IDXD_PMU_EVENT_MAX 64
54
55#define IDXD_ENQCMDS_RETRIES 32
56#define IDXD_ENQCMDS_MAX_RETRIES 64
57
58struct idxd_device_driver {
59 const char *name;
60 enum idxd_dev_type *type;
61 int (*probe)(struct idxd_dev *idxd_dev);
62 void (*remove)(struct idxd_dev *idxd_dev);
63 struct device_driver drv;
64};
65
66extern struct idxd_device_driver dsa_drv;
67extern struct idxd_device_driver idxd_drv;
68extern struct idxd_device_driver idxd_dmaengine_drv;
69extern struct idxd_device_driver idxd_user_drv;
70
71#define INVALID_INT_HANDLE -1
72struct idxd_irq_entry {
73 int id;
74 int vector;
75 struct llist_head pending_llist;
76 struct list_head work_list;
77 /*
78 * Lock to protect access between irq thread process descriptor
79 * and irq thread processing error descriptor.
80 */
81 spinlock_t list_lock;
82 int int_handle;
83 ioasid_t pasid;
84};
85
86struct idxd_group {
87 struct idxd_dev idxd_dev;
88 struct idxd_device *idxd;
89 struct grpcfg grpcfg;
90 int id;
91 int num_engines;
92 int num_wqs;
93 bool use_rdbuf_limit;
94 u8 rdbufs_allowed;
95 u8 rdbufs_reserved;
96 int tc_a;
97 int tc_b;
98};
99
100struct idxd_pmu {
101 struct idxd_device *idxd;
102
103 struct perf_event *event_list[IDXD_PMU_EVENT_MAX];
104 int n_events;
105
106 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);
107
108 struct pmu pmu;
109 char name[IDXD_NAME_SIZE];
110 int cpu;
111
112 int n_counters;
113 int counter_width;
114 int n_event_categories;
115
116 bool per_counter_caps_supported;
117 unsigned long supported_event_categories;
118
119 unsigned long supported_filters;
120 int n_filters;
121
122 struct hlist_node cpuhp_node;
123};
124
125#define IDXD_MAX_PRIORITY 0xf
126
127enum idxd_wq_state {
128 IDXD_WQ_DISABLED = 0,
129 IDXD_WQ_ENABLED,
130};
131
132enum idxd_wq_flag {
133 WQ_FLAG_DEDICATED = 0,
134 WQ_FLAG_BLOCK_ON_FAULT,
135};
136
137enum idxd_wq_type {
138 IDXD_WQT_NONE = 0,
139 IDXD_WQT_KERNEL,
140 IDXD_WQT_USER,
141};
142
143struct idxd_cdev {
144 struct idxd_wq *wq;
145 struct cdev cdev;
146 struct idxd_dev idxd_dev;
147 int minor;
148};
149
150#define IDXD_ALLOCATED_BATCH_SIZE 128U
151#define WQ_NAME_SIZE 1024
152#define WQ_TYPE_SIZE 10
153
154#define WQ_DEFAULT_QUEUE_DEPTH 16
155#define WQ_DEFAULT_MAX_XFER SZ_2M
156#define WQ_DEFAULT_MAX_BATCH 32
157
158enum idxd_op_type {
159 IDXD_OP_BLOCK = 0,
160 IDXD_OP_NONBLOCK = 1,
161};
162
163enum idxd_complete_type {
164 IDXD_COMPLETE_NORMAL = 0,
165 IDXD_COMPLETE_ABORT,
166 IDXD_COMPLETE_DEV_FAIL,
167};
168
169struct idxd_dma_chan {
170 struct dma_chan chan;
171 struct idxd_wq *wq;
172};
173
174struct idxd_wq {
175 void __iomem *portal;
176 u32 portal_offset;
177 unsigned int enqcmds_retries;
178 struct percpu_ref wq_active;
179 struct completion wq_dead;
180 struct completion wq_resurrect;
181 struct idxd_dev idxd_dev;
182 struct idxd_cdev *idxd_cdev;
183 struct wait_queue_head err_queue;
184 struct idxd_device *idxd;
185 int id;
186 struct idxd_irq_entry ie;
187 enum idxd_wq_type type;
188 struct idxd_group *group;
189 int client_count;
190 struct mutex wq_lock; /* mutex for workqueue */
191 u32 size;
192 u32 threshold;
193 u32 priority;
194 enum idxd_wq_state state;
195 unsigned long flags;
196 union wqcfg *wqcfg;
197 struct dsa_hw_desc **hw_descs;
198 int num_descs;
199 union {
200 struct dsa_completion_record *compls;
201 struct iax_completion_record *iax_compls;
202 };
203 dma_addr_t compls_addr;
204 int compls_size;
205 struct idxd_desc **descs;
206 struct sbitmap_queue sbq;
207 struct idxd_dma_chan *idxd_chan;
208 char name[WQ_NAME_SIZE + 1];
209 u64 max_xfer_bytes;
210 u32 max_batch_size;
211 bool ats_dis;
212};
213
214struct idxd_engine {
215 struct idxd_dev idxd_dev;
216 int id;
217 struct idxd_group *group;
218 struct idxd_device *idxd;
219};
220
221/* shadow registers */
222struct idxd_hw {
223 u32 version;
224 union gen_cap_reg gen_cap;
225 union wq_cap_reg wq_cap;
226 union group_cap_reg group_cap;
227 union engine_cap_reg engine_cap;
228 struct opcap opcap;
229 u32 cmd_cap;
230};
231
232enum idxd_device_state {
233 IDXD_DEV_HALTED = -1,
234 IDXD_DEV_DISABLED = 0,
235 IDXD_DEV_ENABLED,
236};
237
238enum idxd_device_flag {
239 IDXD_FLAG_CONFIGURABLE = 0,
240 IDXD_FLAG_CMD_RUNNING,
241 IDXD_FLAG_PASID_ENABLED,
242 IDXD_FLAG_USER_PASID_ENABLED,
243};
244
245struct idxd_dma_dev {
246 struct idxd_device *idxd;
247 struct dma_device dma;
248};
249
250struct idxd_driver_data {
251 const char *name_prefix;
252 enum idxd_type type;
253 struct device_type *dev_type;
254 int compl_size;
255 int align;
256};
257
258struct idxd_device {
259 struct idxd_dev idxd_dev;
260 struct idxd_driver_data *data;
261 struct list_head list;
262 struct idxd_hw hw;
263 enum idxd_device_state state;
264 unsigned long flags;
265 int id;
266 int major;
267 u32 cmd_status;
268 struct idxd_irq_entry ie; /* misc irq, msix 0 */
269
270 struct pci_dev *pdev;
271 void __iomem *reg_base;
272
273 spinlock_t dev_lock; /* spinlock for device */
274 spinlock_t cmd_lock; /* spinlock for device commands */
275 struct completion *cmd_done;
276 struct idxd_group **groups;
277 struct idxd_wq **wqs;
278 struct idxd_engine **engines;
279
280 struct iommu_sva *sva;
281 unsigned int pasid;
282
283 int num_groups;
284 int irq_cnt;
285 bool request_int_handles;
286
287 u32 msix_perm_offset;
288 u32 wqcfg_offset;
289 u32 grpcfg_offset;
290 u32 perfmon_offset;
291
292 u64 max_xfer_bytes;
293 u32 max_batch_size;
294 int max_groups;
295 int max_engines;
296 int max_rdbufs;
297 int max_wqs;
298 int max_wq_size;
299 int rdbuf_limit;
300 int nr_rdbufs; /* non-reserved read buffers */
301 unsigned int wqcfg_size;
302
303 union sw_err_reg sw_err;
304 wait_queue_head_t cmd_waitq;
305
306 struct idxd_dma_dev *idxd_dma;
307 struct workqueue_struct *wq;
308 struct work_struct work;
309
310 struct idxd_pmu *idxd_pmu;
311};
312
313/* IDXD software descriptor */
314struct idxd_desc {
315 union {
316 struct dsa_hw_desc *hw;
317 struct iax_hw_desc *iax_hw;
318 };
319 dma_addr_t desc_dma;
320 union {
321 struct dsa_completion_record *completion;
322 struct iax_completion_record *iax_completion;
323 };
324 dma_addr_t compl_dma;
325 struct dma_async_tx_descriptor txd;
326 struct llist_node llnode;
327 struct list_head list;
328 int id;
329 int cpu;
330 struct idxd_wq *wq;
331};
332
333/*
334 * This is software defined error for the completion status. We overload the error code
335 * that will never appear in completion status and only SWERR register.
336 */
337enum idxd_completion_status {
338 IDXD_COMP_DESC_ABORT = 0xff,
339};
340
341#define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev
342#define wq_confdev(wq) &wq->idxd_dev.conf_dev
343#define engine_confdev(engine) &engine->idxd_dev.conf_dev
344#define group_confdev(group) &group->idxd_dev.conf_dev
345#define cdev_dev(cdev) &cdev->idxd_dev.conf_dev
346
347#define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)
348#define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)
349#define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)
350
351static inline struct idxd_device *confdev_to_idxd(struct device *dev)
352{
353 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
354
355 return idxd_dev_to_idxd(idxd_dev);
356}
357
358static inline struct idxd_wq *confdev_to_wq(struct device *dev)
359{
360 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
361
362 return idxd_dev_to_wq(idxd_dev);
363}
364
365static inline struct idxd_engine *confdev_to_engine(struct device *dev)
366{
367 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
368
369 return container_of(idxd_dev, struct idxd_engine, idxd_dev);
370}
371
372static inline struct idxd_group *confdev_to_group(struct device *dev)
373{
374 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
375
376 return container_of(idxd_dev, struct idxd_group, idxd_dev);
377}
378
379static inline struct idxd_cdev *dev_to_cdev(struct device *dev)
380{
381 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
382
383 return container_of(idxd_dev, struct idxd_cdev, idxd_dev);
384}
385
386static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)
387{
388 if (type >= IDXD_DEV_MAX_TYPE) {
389 idev->type = IDXD_DEV_NONE;
390 return;
391 }
392
393 idev->type = type;
394}
395
396static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx)
397{
398 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie;
399}
400
401static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie)
402{
403 return container_of(ie, struct idxd_wq, ie);
404}
405
406static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)
407{
408 return container_of(ie, struct idxd_device, ie);
409}
410
411extern struct bus_type dsa_bus_type;
412
413extern bool support_enqcmd;
414extern struct ida idxd_ida;
415extern struct device_type dsa_device_type;
416extern struct device_type iax_device_type;
417extern struct device_type idxd_wq_device_type;
418extern struct device_type idxd_engine_device_type;
419extern struct device_type idxd_group_device_type;
420
421static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)
422{
423 return idxd_dev->type == IDXD_DEV_DSA;
424}
425
426static inline bool is_iax_dev(struct idxd_dev *idxd_dev)
427{
428 return idxd_dev->type == IDXD_DEV_IAX;
429}
430
431static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)
432{
433 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);
434}
435
436static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)
437{
438 return idxd_dev->type == IDXD_DEV_WQ;
439}
440
441static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
442{
443 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
444 return true;
445 return false;
446}
447
448static inline bool is_idxd_wq_user(struct idxd_wq *wq)
449{
450 return wq->type == IDXD_WQT_USER;
451}
452
453static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)
454{
455 return wq->type == IDXD_WQT_KERNEL;
456}
457
458static inline bool wq_dedicated(struct idxd_wq *wq)
459{
460 return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
461}
462
463static inline bool wq_shared(struct idxd_wq *wq)
464{
465 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
466}
467
468static inline bool device_pasid_enabled(struct idxd_device *idxd)
469{
470 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
471}
472
473static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
474{
475 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
476}
477
478static inline bool wq_pasid_enabled(struct idxd_wq *wq)
479{
480 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
481 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
482}
483
484static inline bool wq_shared_supported(struct idxd_wq *wq)
485{
486 return (support_enqcmd && wq_pasid_enabled(wq));
487}
488
489enum idxd_portal_prot {
490 IDXD_PORTAL_UNLIMITED = 0,
491 IDXD_PORTAL_LIMITED,
492};
493
494enum idxd_interrupt_type {
495 IDXD_IRQ_MSIX = 0,
496 IDXD_IRQ_IMS,
497};
498
499static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
500{
501 return prot * 0x1000;
502}
503
504static inline int idxd_get_wq_portal_full_offset(int wq_id,
505 enum idxd_portal_prot prot)
506{
507 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
508}
509
510#define IDXD_PORTAL_MASK (PAGE_SIZE - 1)
511
512/*
513 * Even though this function can be accessed by multiple threads, it is safe to use.
514 * At worst the address gets used more than once before it gets incremented. We don't
515 * hit a threshold until iops becomes many million times a second. So the occasional
516 * reuse of the same address is tolerable compare to using an atomic variable. This is
517 * safe on a system that has atomic load/store for 32bit integers. Given that this is an
518 * Intel iEP device, that should not be a problem.
519 */
520static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)
521{
522 int ofs = wq->portal_offset;
523
524 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;
525 return wq->portal + ofs;
526}
527
528static inline void idxd_wq_get(struct idxd_wq *wq)
529{
530 wq->client_count++;
531}
532
533static inline void idxd_wq_put(struct idxd_wq *wq)
534{
535 wq->client_count--;
536}
537
538static inline int idxd_wq_refcount(struct idxd_wq *wq)
539{
540 return wq->client_count;
541};
542
543int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,
544 struct module *module, const char *mod_name);
545#define idxd_driver_register(driver) \
546 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
547
548void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);
549
550#define module_idxd_driver(__idxd_driver) \
551 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)
552
553int idxd_register_bus_type(void);
554void idxd_unregister_bus_type(void);
555int idxd_register_devices(struct idxd_device *idxd);
556void idxd_unregister_devices(struct idxd_device *idxd);
557int idxd_register_driver(void);
558void idxd_unregister_driver(void);
559void idxd_wqs_quiesce(struct idxd_device *idxd);
560bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc);
561
562/* device interrupt control */
563irqreturn_t idxd_misc_thread(int vec, void *data);
564irqreturn_t idxd_wq_thread(int irq, void *data);
565void idxd_mask_error_interrupts(struct idxd_device *idxd);
566void idxd_unmask_error_interrupts(struct idxd_device *idxd);
567
568/* device control */
569int idxd_register_idxd_drv(void);
570void idxd_unregister_idxd_drv(void);
571int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
572void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
573int drv_enable_wq(struct idxd_wq *wq);
574void drv_disable_wq(struct idxd_wq *wq);
575int idxd_device_init_reset(struct idxd_device *idxd);
576int idxd_device_enable(struct idxd_device *idxd);
577int idxd_device_disable(struct idxd_device *idxd);
578void idxd_device_reset(struct idxd_device *idxd);
579void idxd_device_clear_state(struct idxd_device *idxd);
580int idxd_device_config(struct idxd_device *idxd);
581void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
582int idxd_device_load_config(struct idxd_device *idxd);
583int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
584 enum idxd_interrupt_type irq_type);
585int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
586 enum idxd_interrupt_type irq_type);
587
588/* work queue control */
589void idxd_wqs_unmap_portal(struct idxd_device *idxd);
590int idxd_wq_alloc_resources(struct idxd_wq *wq);
591void idxd_wq_free_resources(struct idxd_wq *wq);
592int idxd_wq_enable(struct idxd_wq *wq);
593int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);
594void idxd_wq_drain(struct idxd_wq *wq);
595void idxd_wq_reset(struct idxd_wq *wq);
596int idxd_wq_map_portal(struct idxd_wq *wq);
597void idxd_wq_unmap_portal(struct idxd_wq *wq);
598int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
599int idxd_wq_disable_pasid(struct idxd_wq *wq);
600void __idxd_wq_quiesce(struct idxd_wq *wq);
601void idxd_wq_quiesce(struct idxd_wq *wq);
602int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
603void idxd_wq_free_irq(struct idxd_wq *wq);
604int idxd_wq_request_irq(struct idxd_wq *wq);
605
606/* submission */
607int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
608struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
609void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
610int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);
611
612/* dmaengine */
613int idxd_register_dma_device(struct idxd_device *idxd);
614void idxd_unregister_dma_device(struct idxd_device *idxd);
615void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
616void idxd_dma_complete_txd(struct idxd_desc *desc,
617 enum idxd_complete_type comp_type, bool free_desc);
618
619/* cdev */
620int idxd_cdev_register(void);
621void idxd_cdev_remove(void);
622int idxd_cdev_get_major(struct idxd_device *idxd);
623int idxd_wq_add_cdev(struct idxd_wq *wq);
624void idxd_wq_del_cdev(struct idxd_wq *wq);
625
626/* perfmon */
627#if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
628int perfmon_pmu_init(struct idxd_device *idxd);
629void perfmon_pmu_remove(struct idxd_device *idxd);
630void perfmon_counter_overflow(struct idxd_device *idxd);
631void perfmon_init(void);
632void perfmon_exit(void);
633#else
634static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }
635static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}
636static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}
637static inline void perfmon_init(void) {}
638static inline void perfmon_exit(void) {}
639#endif
640
641#endif