Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <joerg.roedel@amd.com>
5 */
6
7#ifndef __LINUX_IOMMU_H
8#define __LINUX_IOMMU_H
9
10#include <linux/scatterlist.h>
11#include <linux/device.h>
12#include <linux/types.h>
13#include <linux/errno.h>
14#include <linux/err.h>
15#include <linux/of.h>
16#include <linux/ioasid.h>
17#include <uapi/linux/iommu.h>
18
19#define IOMMU_READ (1 << 0)
20#define IOMMU_WRITE (1 << 1)
21#define IOMMU_CACHE (1 << 2) /* DMA cache coherency */
22#define IOMMU_NOEXEC (1 << 3)
23#define IOMMU_MMIO (1 << 4) /* e.g. things like MSI doorbells */
24/*
25 * Where the bus hardware includes a privilege level as part of its access type
26 * markings, and certain devices are capable of issuing transactions marked as
27 * either 'supervisor' or 'user', the IOMMU_PRIV flag requests that the other
28 * given permission flags only apply to accesses at the higher privilege level,
29 * and that unprivileged transactions should have as little access as possible.
30 * This would usually imply the same permissions as kernel mappings on the CPU,
31 * if the IOMMU page table format is equivalent.
32 */
33#define IOMMU_PRIV (1 << 5)
34/*
35 * Non-coherent masters can use this page protection flag to set cacheable
36 * memory attributes for only a transparent outer level of cache, also known as
37 * the last-level or system cache.
38 */
39#define IOMMU_SYS_CACHE_ONLY (1 << 6)
40
41struct iommu_ops;
42struct iommu_group;
43struct bus_type;
44struct device;
45struct iommu_domain;
46struct notifier_block;
47struct iommu_sva;
48struct iommu_fault_event;
49
50/* iommu fault flags */
51#define IOMMU_FAULT_READ 0x0
52#define IOMMU_FAULT_WRITE 0x1
53
54typedef int (*iommu_fault_handler_t)(struct iommu_domain *,
55 struct device *, unsigned long, int, void *);
56typedef int (*iommu_mm_exit_handler_t)(struct device *dev, struct iommu_sva *,
57 void *);
58typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *);
59
60struct iommu_domain_geometry {
61 dma_addr_t aperture_start; /* First address that can be mapped */
62 dma_addr_t aperture_end; /* Last address that can be mapped */
63 bool force_aperture; /* DMA only allowed in mappable range? */
64};
65
66/* Domain feature flags */
67#define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */
68#define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API
69 implementation */
70#define __IOMMU_DOMAIN_PT (1U << 2) /* Domain is identity mapped */
71
72/*
73 * This are the possible domain-types
74 *
75 * IOMMU_DOMAIN_BLOCKED - All DMA is blocked, can be used to isolate
76 * devices
77 * IOMMU_DOMAIN_IDENTITY - DMA addresses are system physical addresses
78 * IOMMU_DOMAIN_UNMANAGED - DMA mappings managed by IOMMU-API user, used
79 * for VMs
80 * IOMMU_DOMAIN_DMA - Internally used for DMA-API implementations.
81 * This flag allows IOMMU drivers to implement
82 * certain optimizations for these domains
83 */
84#define IOMMU_DOMAIN_BLOCKED (0U)
85#define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT)
86#define IOMMU_DOMAIN_UNMANAGED (__IOMMU_DOMAIN_PAGING)
87#define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING | \
88 __IOMMU_DOMAIN_DMA_API)
89
90struct iommu_domain {
91 unsigned type;
92 const struct iommu_ops *ops;
93 unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */
94 iommu_fault_handler_t handler;
95 void *handler_token;
96 struct iommu_domain_geometry geometry;
97 void *iova_cookie;
98};
99
100enum iommu_cap {
101 IOMMU_CAP_CACHE_COHERENCY, /* IOMMU can enforce cache coherent DMA
102 transactions */
103 IOMMU_CAP_INTR_REMAP, /* IOMMU supports interrupt isolation */
104 IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */
105};
106
107/*
108 * Following constraints are specifc to FSL_PAMUV1:
109 * -aperture must be power of 2, and naturally aligned
110 * -number of windows must be power of 2, and address space size
111 * of each window is determined by aperture size / # of windows
112 * -the actual size of the mapped region of a window must be power
113 * of 2 starting with 4KB and physical address must be naturally
114 * aligned.
115 * DOMAIN_ATTR_FSL_PAMUV1 corresponds to the above mentioned contraints.
116 * The caller can invoke iommu_domain_get_attr to check if the underlying
117 * iommu implementation supports these constraints.
118 */
119
120enum iommu_attr {
121 DOMAIN_ATTR_GEOMETRY,
122 DOMAIN_ATTR_PAGING,
123 DOMAIN_ATTR_WINDOWS,
124 DOMAIN_ATTR_FSL_PAMU_STASH,
125 DOMAIN_ATTR_FSL_PAMU_ENABLE,
126 DOMAIN_ATTR_FSL_PAMUV1,
127 DOMAIN_ATTR_NESTING, /* two stages of translation */
128 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
129 DOMAIN_ATTR_MAX,
130};
131
132/* These are the possible reserved region types */
133enum iommu_resv_type {
134 /* Memory regions which must be mapped 1:1 at all times */
135 IOMMU_RESV_DIRECT,
136 /*
137 * Memory regions which are advertised to be 1:1 but are
138 * commonly considered relaxable in some conditions,
139 * for instance in device assignment use case (USB, Graphics)
140 */
141 IOMMU_RESV_DIRECT_RELAXABLE,
142 /* Arbitrary "never map this or give it to a device" address ranges */
143 IOMMU_RESV_RESERVED,
144 /* Hardware MSI region (untranslated) */
145 IOMMU_RESV_MSI,
146 /* Software-managed MSI translation window */
147 IOMMU_RESV_SW_MSI,
148};
149
150/**
151 * struct iommu_resv_region - descriptor for a reserved memory region
152 * @list: Linked list pointers
153 * @start: System physical start address of the region
154 * @length: Length of the region in bytes
155 * @prot: IOMMU Protection flags (READ/WRITE/...)
156 * @type: Type of the reserved region
157 */
158struct iommu_resv_region {
159 struct list_head list;
160 phys_addr_t start;
161 size_t length;
162 int prot;
163 enum iommu_resv_type type;
164};
165
166/* Per device IOMMU features */
167enum iommu_dev_features {
168 IOMMU_DEV_FEAT_AUX, /* Aux-domain feature */
169 IOMMU_DEV_FEAT_SVA, /* Shared Virtual Addresses */
170};
171
172#define IOMMU_PASID_INVALID (-1U)
173
174/**
175 * struct iommu_sva_ops - device driver callbacks for an SVA context
176 *
177 * @mm_exit: called when the mm is about to be torn down by exit_mmap. After
178 * @mm_exit returns, the device must not issue any more transaction
179 * with the PASID given as argument.
180 *
181 * The @mm_exit handler is allowed to sleep. Be careful about the
182 * locks taken in @mm_exit, because they might lead to deadlocks if
183 * they are also held when dropping references to the mm. Consider the
184 * following call chain:
185 * mutex_lock(A); mmput(mm) -> exit_mm() -> @mm_exit() -> mutex_lock(A)
186 * Using mmput_async() prevents this scenario.
187 *
188 */
189struct iommu_sva_ops {
190 iommu_mm_exit_handler_t mm_exit;
191};
192
193#ifdef CONFIG_IOMMU_API
194
195/**
196 * struct iommu_iotlb_gather - Range information for a pending IOTLB flush
197 *
198 * @start: IOVA representing the start of the range to be flushed
199 * @end: IOVA representing the end of the range to be flushed (exclusive)
200 * @pgsize: The interval at which to perform the flush
201 *
202 * This structure is intended to be updated by multiple calls to the
203 * ->unmap() function in struct iommu_ops before eventually being passed
204 * into ->iotlb_sync().
205 */
206struct iommu_iotlb_gather {
207 unsigned long start;
208 unsigned long end;
209 size_t pgsize;
210};
211
212/**
213 * struct iommu_ops - iommu ops and capabilities
214 * @capable: check capability
215 * @domain_alloc: allocate iommu domain
216 * @domain_free: free iommu domain
217 * @attach_dev: attach device to an iommu domain
218 * @detach_dev: detach device from an iommu domain
219 * @map: map a physically contiguous memory region to an iommu domain
220 * @unmap: unmap a physically contiguous memory region from an iommu domain
221 * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
222 * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
223 * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
224 * queue
225 * @iova_to_phys: translate iova to physical address
226 * @add_device: add device to iommu grouping
227 * @remove_device: remove device from iommu grouping
228 * @device_group: find iommu group for a particular device
229 * @domain_get_attr: Query domain attributes
230 * @domain_set_attr: Change domain attributes
231 * @get_resv_regions: Request list of reserved regions for a device
232 * @put_resv_regions: Free list of reserved regions for a device
233 * @apply_resv_region: Temporary helper call-back for iova reserved ranges
234 * @domain_window_enable: Configure and enable a particular window for a domain
235 * @domain_window_disable: Disable a particular window for a domain
236 * @of_xlate: add OF master IDs to iommu grouping
237 * @is_attach_deferred: Check if domain attach should be deferred from iommu
238 * driver init to device driver init (default no)
239 * @dev_has/enable/disable_feat: per device entries to check/enable/disable
240 * iommu specific features.
241 * @dev_feat_enabled: check enabled feature
242 * @aux_attach/detach_dev: aux-domain specific attach/detach entries.
243 * @aux_get_pasid: get the pasid given an aux-domain
244 * @sva_bind: Bind process address space to device
245 * @sva_unbind: Unbind process address space from device
246 * @sva_get_pasid: Get PASID associated to a SVA handle
247 * @page_response: handle page request response
248 * @cache_invalidate: invalidate translation caches
249 * @sva_bind_gpasid: bind guest pasid and mm
250 * @sva_unbind_gpasid: unbind guest pasid and mm
251 * @pgsize_bitmap: bitmap of all possible supported page sizes
252 * @owner: Driver module providing these ops
253 */
254struct iommu_ops {
255 bool (*capable)(enum iommu_cap);
256
257 /* Domain allocation and freeing by the iommu driver */
258 struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type);
259 void (*domain_free)(struct iommu_domain *);
260
261 int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
262 void (*detach_dev)(struct iommu_domain *domain, struct device *dev);
263 int (*map)(struct iommu_domain *domain, unsigned long iova,
264 phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
265 size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
266 size_t size, struct iommu_iotlb_gather *iotlb_gather);
267 void (*flush_iotlb_all)(struct iommu_domain *domain);
268 void (*iotlb_sync_map)(struct iommu_domain *domain);
269 void (*iotlb_sync)(struct iommu_domain *domain,
270 struct iommu_iotlb_gather *iotlb_gather);
271 phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova);
272 int (*add_device)(struct device *dev);
273 void (*remove_device)(struct device *dev);
274 struct iommu_group *(*device_group)(struct device *dev);
275 int (*domain_get_attr)(struct iommu_domain *domain,
276 enum iommu_attr attr, void *data);
277 int (*domain_set_attr)(struct iommu_domain *domain,
278 enum iommu_attr attr, void *data);
279
280 /* Request/Free a list of reserved regions for a device */
281 void (*get_resv_regions)(struct device *dev, struct list_head *list);
282 void (*put_resv_regions)(struct device *dev, struct list_head *list);
283 void (*apply_resv_region)(struct device *dev,
284 struct iommu_domain *domain,
285 struct iommu_resv_region *region);
286
287 /* Window handling functions */
288 int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr,
289 phys_addr_t paddr, u64 size, int prot);
290 void (*domain_window_disable)(struct iommu_domain *domain, u32 wnd_nr);
291
292 int (*of_xlate)(struct device *dev, struct of_phandle_args *args);
293 bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev);
294
295 /* Per device IOMMU features */
296 bool (*dev_has_feat)(struct device *dev, enum iommu_dev_features f);
297 bool (*dev_feat_enabled)(struct device *dev, enum iommu_dev_features f);
298 int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f);
299 int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f);
300
301 /* Aux-domain specific attach/detach entries */
302 int (*aux_attach_dev)(struct iommu_domain *domain, struct device *dev);
303 void (*aux_detach_dev)(struct iommu_domain *domain, struct device *dev);
304 int (*aux_get_pasid)(struct iommu_domain *domain, struct device *dev);
305
306 struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm,
307 void *drvdata);
308 void (*sva_unbind)(struct iommu_sva *handle);
309 int (*sva_get_pasid)(struct iommu_sva *handle);
310
311 int (*page_response)(struct device *dev,
312 struct iommu_fault_event *evt,
313 struct iommu_page_response *msg);
314 int (*cache_invalidate)(struct iommu_domain *domain, struct device *dev,
315 struct iommu_cache_invalidate_info *inv_info);
316 int (*sva_bind_gpasid)(struct iommu_domain *domain,
317 struct device *dev, struct iommu_gpasid_bind_data *data);
318
319 int (*sva_unbind_gpasid)(struct device *dev, int pasid);
320
321 unsigned long pgsize_bitmap;
322 struct module *owner;
323};
324
325/**
326 * struct iommu_device - IOMMU core representation of one IOMMU hardware
327 * instance
328 * @list: Used by the iommu-core to keep a list of registered iommus
329 * @ops: iommu-ops for talking to this iommu
330 * @dev: struct device for sysfs handling
331 */
332struct iommu_device {
333 struct list_head list;
334 const struct iommu_ops *ops;
335 struct fwnode_handle *fwnode;
336 struct device *dev;
337};
338
339/**
340 * struct iommu_fault_event - Generic fault event
341 *
342 * Can represent recoverable faults such as a page requests or
343 * unrecoverable faults such as DMA or IRQ remapping faults.
344 *
345 * @fault: fault descriptor
346 * @list: pending fault event list, used for tracking responses
347 */
348struct iommu_fault_event {
349 struct iommu_fault fault;
350 struct list_head list;
351};
352
353/**
354 * struct iommu_fault_param - per-device IOMMU fault data
355 * @handler: Callback function to handle IOMMU faults at device level
356 * @data: handler private data
357 * @faults: holds the pending faults which needs response
358 * @lock: protect pending faults list
359 */
360struct iommu_fault_param {
361 iommu_dev_fault_handler_t handler;
362 void *data;
363 struct list_head faults;
364 struct mutex lock;
365};
366
367/**
368 * struct dev_iommu - Collection of per-device IOMMU data
369 *
370 * @fault_param: IOMMU detected device fault reporting data
371 * @fwspec: IOMMU fwspec data
372 * @priv: IOMMU Driver private data
373 *
374 * TODO: migrate other per device data pointers under iommu_dev_data, e.g.
375 * struct iommu_group *iommu_group;
376 */
377struct dev_iommu {
378 struct mutex lock;
379 struct iommu_fault_param *fault_param;
380 struct iommu_fwspec *fwspec;
381 void *priv;
382};
383
384int iommu_device_register(struct iommu_device *iommu);
385void iommu_device_unregister(struct iommu_device *iommu);
386int iommu_device_sysfs_add(struct iommu_device *iommu,
387 struct device *parent,
388 const struct attribute_group **groups,
389 const char *fmt, ...) __printf(4, 5);
390void iommu_device_sysfs_remove(struct iommu_device *iommu);
391int iommu_device_link(struct iommu_device *iommu, struct device *link);
392void iommu_device_unlink(struct iommu_device *iommu, struct device *link);
393
394static inline void __iommu_device_set_ops(struct iommu_device *iommu,
395 const struct iommu_ops *ops)
396{
397 iommu->ops = ops;
398}
399
400#define iommu_device_set_ops(iommu, ops) \
401do { \
402 struct iommu_ops *__ops = (struct iommu_ops *)(ops); \
403 __ops->owner = THIS_MODULE; \
404 __iommu_device_set_ops(iommu, __ops); \
405} while (0)
406
407static inline void iommu_device_set_fwnode(struct iommu_device *iommu,
408 struct fwnode_handle *fwnode)
409{
410 iommu->fwnode = fwnode;
411}
412
413static inline struct iommu_device *dev_to_iommu_device(struct device *dev)
414{
415 return (struct iommu_device *)dev_get_drvdata(dev);
416}
417
418static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
419{
420 *gather = (struct iommu_iotlb_gather) {
421 .start = ULONG_MAX,
422 };
423}
424
425#define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */
426#define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */
427#define IOMMU_GROUP_NOTIFY_BIND_DRIVER 3 /* Pre Driver bind */
428#define IOMMU_GROUP_NOTIFY_BOUND_DRIVER 4 /* Post Driver bind */
429#define IOMMU_GROUP_NOTIFY_UNBIND_DRIVER 5 /* Pre Driver unbind */
430#define IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER 6 /* Post Driver unbind */
431
432extern int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops);
433extern bool iommu_present(struct bus_type *bus);
434extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap);
435extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus);
436extern struct iommu_group *iommu_group_get_by_id(int id);
437extern void iommu_domain_free(struct iommu_domain *domain);
438extern int iommu_attach_device(struct iommu_domain *domain,
439 struct device *dev);
440extern void iommu_detach_device(struct iommu_domain *domain,
441 struct device *dev);
442extern int iommu_cache_invalidate(struct iommu_domain *domain,
443 struct device *dev,
444 struct iommu_cache_invalidate_info *inv_info);
445extern int iommu_sva_bind_gpasid(struct iommu_domain *domain,
446 struct device *dev, struct iommu_gpasid_bind_data *data);
447extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
448 struct device *dev, ioasid_t pasid);
449extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
450extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
451extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
452 phys_addr_t paddr, size_t size, int prot);
453extern int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
454 phys_addr_t paddr, size_t size, int prot);
455extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
456 size_t size);
457extern size_t iommu_unmap_fast(struct iommu_domain *domain,
458 unsigned long iova, size_t size,
459 struct iommu_iotlb_gather *iotlb_gather);
460extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
461 struct scatterlist *sg,unsigned int nents, int prot);
462extern size_t iommu_map_sg_atomic(struct iommu_domain *domain,
463 unsigned long iova, struct scatterlist *sg,
464 unsigned int nents, int prot);
465extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
466extern void iommu_set_fault_handler(struct iommu_domain *domain,
467 iommu_fault_handler_t handler, void *token);
468
469extern void iommu_get_resv_regions(struct device *dev, struct list_head *list);
470extern void iommu_put_resv_regions(struct device *dev, struct list_head *list);
471extern void generic_iommu_put_resv_regions(struct device *dev,
472 struct list_head *list);
473extern int iommu_request_dm_for_dev(struct device *dev);
474extern int iommu_request_dma_domain_for_dev(struct device *dev);
475extern void iommu_set_default_passthrough(bool cmd_line);
476extern void iommu_set_default_translated(bool cmd_line);
477extern bool iommu_default_passthrough(void);
478extern struct iommu_resv_region *
479iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot,
480 enum iommu_resv_type type);
481extern int iommu_get_group_resv_regions(struct iommu_group *group,
482 struct list_head *head);
483
484extern int iommu_attach_group(struct iommu_domain *domain,
485 struct iommu_group *group);
486extern void iommu_detach_group(struct iommu_domain *domain,
487 struct iommu_group *group);
488extern struct iommu_group *iommu_group_alloc(void);
489extern void *iommu_group_get_iommudata(struct iommu_group *group);
490extern void iommu_group_set_iommudata(struct iommu_group *group,
491 void *iommu_data,
492 void (*release)(void *iommu_data));
493extern int iommu_group_set_name(struct iommu_group *group, const char *name);
494extern int iommu_group_add_device(struct iommu_group *group,
495 struct device *dev);
496extern void iommu_group_remove_device(struct device *dev);
497extern int iommu_group_for_each_dev(struct iommu_group *group, void *data,
498 int (*fn)(struct device *, void *));
499extern struct iommu_group *iommu_group_get(struct device *dev);
500extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group);
501extern void iommu_group_put(struct iommu_group *group);
502extern int iommu_group_register_notifier(struct iommu_group *group,
503 struct notifier_block *nb);
504extern int iommu_group_unregister_notifier(struct iommu_group *group,
505 struct notifier_block *nb);
506extern int iommu_register_device_fault_handler(struct device *dev,
507 iommu_dev_fault_handler_t handler,
508 void *data);
509
510extern int iommu_unregister_device_fault_handler(struct device *dev);
511
512extern int iommu_report_device_fault(struct device *dev,
513 struct iommu_fault_event *evt);
514extern int iommu_page_response(struct device *dev,
515 struct iommu_page_response *msg);
516
517extern int iommu_group_id(struct iommu_group *group);
518extern struct iommu_group *iommu_group_get_for_dev(struct device *dev);
519extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *);
520
521extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr,
522 void *data);
523extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr,
524 void *data);
525
526/* Window handling function prototypes */
527extern int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
528 phys_addr_t offset, u64 size,
529 int prot);
530extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr);
531
532extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
533 unsigned long iova, int flags);
534
535static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
536{
537 if (domain->ops->flush_iotlb_all)
538 domain->ops->flush_iotlb_all(domain);
539}
540
541static inline void iommu_tlb_sync(struct iommu_domain *domain,
542 struct iommu_iotlb_gather *iotlb_gather)
543{
544 if (domain->ops->iotlb_sync)
545 domain->ops->iotlb_sync(domain, iotlb_gather);
546
547 iommu_iotlb_gather_init(iotlb_gather);
548}
549
550static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
551 struct iommu_iotlb_gather *gather,
552 unsigned long iova, size_t size)
553{
554 unsigned long start = iova, end = start + size;
555
556 /*
557 * If the new page is disjoint from the current range or is mapped at
558 * a different granularity, then sync the TLB so that the gather
559 * structure can be rewritten.
560 */
561 if (gather->pgsize != size ||
562 end < gather->start || start > gather->end) {
563 if (gather->pgsize)
564 iommu_tlb_sync(domain, gather);
565 gather->pgsize = size;
566 }
567
568 if (gather->end < end)
569 gather->end = end;
570
571 if (gather->start > start)
572 gather->start = start;
573}
574
575/* PCI device grouping function */
576extern struct iommu_group *pci_device_group(struct device *dev);
577/* Generic device grouping function */
578extern struct iommu_group *generic_device_group(struct device *dev);
579/* FSL-MC device grouping function */
580struct iommu_group *fsl_mc_device_group(struct device *dev);
581
582/**
583 * struct iommu_fwspec - per-device IOMMU instance data
584 * @ops: ops for this device's IOMMU
585 * @iommu_fwnode: firmware handle for this device's IOMMU
586 * @iommu_priv: IOMMU driver private data for this device
587 * @num_pasid_bits: number of PASID bits supported by this device
588 * @num_ids: number of associated device IDs
589 * @ids: IDs which this device may present to the IOMMU
590 */
591struct iommu_fwspec {
592 const struct iommu_ops *ops;
593 struct fwnode_handle *iommu_fwnode;
594 u32 flags;
595 u32 num_pasid_bits;
596 unsigned int num_ids;
597 u32 ids[];
598};
599
600/* ATS is supported */
601#define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0)
602
603/**
604 * struct iommu_sva - handle to a device-mm bond
605 */
606struct iommu_sva {
607 struct device *dev;
608 const struct iommu_sva_ops *ops;
609};
610
611int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
612 const struct iommu_ops *ops);
613void iommu_fwspec_free(struct device *dev);
614int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids);
615const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode);
616
617static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
618{
619 if (dev->iommu)
620 return dev->iommu->fwspec;
621 else
622 return NULL;
623}
624
625static inline void dev_iommu_fwspec_set(struct device *dev,
626 struct iommu_fwspec *fwspec)
627{
628 dev->iommu->fwspec = fwspec;
629}
630
631static inline void *dev_iommu_priv_get(struct device *dev)
632{
633 return dev->iommu->priv;
634}
635
636static inline void dev_iommu_priv_set(struct device *dev, void *priv)
637{
638 dev->iommu->priv = priv;
639}
640
641int iommu_probe_device(struct device *dev);
642void iommu_release_device(struct device *dev);
643
644bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features f);
645int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
646int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
647bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f);
648int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev);
649void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev);
650int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev);
651
652struct iommu_sva *iommu_sva_bind_device(struct device *dev,
653 struct mm_struct *mm,
654 void *drvdata);
655void iommu_sva_unbind_device(struct iommu_sva *handle);
656int iommu_sva_set_ops(struct iommu_sva *handle,
657 const struct iommu_sva_ops *ops);
658int iommu_sva_get_pasid(struct iommu_sva *handle);
659
660#else /* CONFIG_IOMMU_API */
661
662struct iommu_ops {};
663struct iommu_group {};
664struct iommu_fwspec {};
665struct iommu_device {};
666struct iommu_fault_param {};
667struct iommu_iotlb_gather {};
668
669static inline bool iommu_present(struct bus_type *bus)
670{
671 return false;
672}
673
674static inline bool iommu_capable(struct bus_type *bus, enum iommu_cap cap)
675{
676 return false;
677}
678
679static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
680{
681 return NULL;
682}
683
684static inline struct iommu_group *iommu_group_get_by_id(int id)
685{
686 return NULL;
687}
688
689static inline void iommu_domain_free(struct iommu_domain *domain)
690{
691}
692
693static inline int iommu_attach_device(struct iommu_domain *domain,
694 struct device *dev)
695{
696 return -ENODEV;
697}
698
699static inline void iommu_detach_device(struct iommu_domain *domain,
700 struct device *dev)
701{
702}
703
704static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
705{
706 return NULL;
707}
708
709static inline int iommu_map(struct iommu_domain *domain, unsigned long iova,
710 phys_addr_t paddr, size_t size, int prot)
711{
712 return -ENODEV;
713}
714
715static inline int iommu_map_atomic(struct iommu_domain *domain,
716 unsigned long iova, phys_addr_t paddr,
717 size_t size, int prot)
718{
719 return -ENODEV;
720}
721
722static inline size_t iommu_unmap(struct iommu_domain *domain,
723 unsigned long iova, size_t size)
724{
725 return 0;
726}
727
728static inline size_t iommu_unmap_fast(struct iommu_domain *domain,
729 unsigned long iova, int gfp_order,
730 struct iommu_iotlb_gather *iotlb_gather)
731{
732 return 0;
733}
734
735static inline size_t iommu_map_sg(struct iommu_domain *domain,
736 unsigned long iova, struct scatterlist *sg,
737 unsigned int nents, int prot)
738{
739 return 0;
740}
741
742static inline size_t iommu_map_sg_atomic(struct iommu_domain *domain,
743 unsigned long iova, struct scatterlist *sg,
744 unsigned int nents, int prot)
745{
746 return 0;
747}
748
749static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
750{
751}
752
753static inline void iommu_tlb_sync(struct iommu_domain *domain,
754 struct iommu_iotlb_gather *iotlb_gather)
755{
756}
757
758static inline int iommu_domain_window_enable(struct iommu_domain *domain,
759 u32 wnd_nr, phys_addr_t paddr,
760 u64 size, int prot)
761{
762 return -ENODEV;
763}
764
765static inline void iommu_domain_window_disable(struct iommu_domain *domain,
766 u32 wnd_nr)
767{
768}
769
770static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
771{
772 return 0;
773}
774
775static inline void iommu_set_fault_handler(struct iommu_domain *domain,
776 iommu_fault_handler_t handler, void *token)
777{
778}
779
780static inline void iommu_get_resv_regions(struct device *dev,
781 struct list_head *list)
782{
783}
784
785static inline void iommu_put_resv_regions(struct device *dev,
786 struct list_head *list)
787{
788}
789
790static inline int iommu_get_group_resv_regions(struct iommu_group *group,
791 struct list_head *head)
792{
793 return -ENODEV;
794}
795
796static inline int iommu_request_dm_for_dev(struct device *dev)
797{
798 return -ENODEV;
799}
800
801static inline int iommu_request_dma_domain_for_dev(struct device *dev)
802{
803 return -ENODEV;
804}
805
806static inline void iommu_set_default_passthrough(bool cmd_line)
807{
808}
809
810static inline void iommu_set_default_translated(bool cmd_line)
811{
812}
813
814static inline bool iommu_default_passthrough(void)
815{
816 return true;
817}
818
819static inline int iommu_attach_group(struct iommu_domain *domain,
820 struct iommu_group *group)
821{
822 return -ENODEV;
823}
824
825static inline void iommu_detach_group(struct iommu_domain *domain,
826 struct iommu_group *group)
827{
828}
829
830static inline struct iommu_group *iommu_group_alloc(void)
831{
832 return ERR_PTR(-ENODEV);
833}
834
835static inline void *iommu_group_get_iommudata(struct iommu_group *group)
836{
837 return NULL;
838}
839
840static inline void iommu_group_set_iommudata(struct iommu_group *group,
841 void *iommu_data,
842 void (*release)(void *iommu_data))
843{
844}
845
846static inline int iommu_group_set_name(struct iommu_group *group,
847 const char *name)
848{
849 return -ENODEV;
850}
851
852static inline int iommu_group_add_device(struct iommu_group *group,
853 struct device *dev)
854{
855 return -ENODEV;
856}
857
858static inline void iommu_group_remove_device(struct device *dev)
859{
860}
861
862static inline int iommu_group_for_each_dev(struct iommu_group *group,
863 void *data,
864 int (*fn)(struct device *, void *))
865{
866 return -ENODEV;
867}
868
869static inline struct iommu_group *iommu_group_get(struct device *dev)
870{
871 return NULL;
872}
873
874static inline void iommu_group_put(struct iommu_group *group)
875{
876}
877
878static inline int iommu_group_register_notifier(struct iommu_group *group,
879 struct notifier_block *nb)
880{
881 return -ENODEV;
882}
883
884static inline int iommu_group_unregister_notifier(struct iommu_group *group,
885 struct notifier_block *nb)
886{
887 return 0;
888}
889
890static inline
891int iommu_register_device_fault_handler(struct device *dev,
892 iommu_dev_fault_handler_t handler,
893 void *data)
894{
895 return -ENODEV;
896}
897
898static inline int iommu_unregister_device_fault_handler(struct device *dev)
899{
900 return 0;
901}
902
903static inline
904int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)
905{
906 return -ENODEV;
907}
908
909static inline int iommu_page_response(struct device *dev,
910 struct iommu_page_response *msg)
911{
912 return -ENODEV;
913}
914
915static inline int iommu_group_id(struct iommu_group *group)
916{
917 return -ENODEV;
918}
919
920static inline int iommu_domain_get_attr(struct iommu_domain *domain,
921 enum iommu_attr attr, void *data)
922{
923 return -EINVAL;
924}
925
926static inline int iommu_domain_set_attr(struct iommu_domain *domain,
927 enum iommu_attr attr, void *data)
928{
929 return -EINVAL;
930}
931
932static inline int iommu_device_register(struct iommu_device *iommu)
933{
934 return -ENODEV;
935}
936
937static inline void iommu_device_set_ops(struct iommu_device *iommu,
938 const struct iommu_ops *ops)
939{
940}
941
942static inline void iommu_device_set_fwnode(struct iommu_device *iommu,
943 struct fwnode_handle *fwnode)
944{
945}
946
947static inline struct iommu_device *dev_to_iommu_device(struct device *dev)
948{
949 return NULL;
950}
951
952static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
953{
954}
955
956static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
957 struct iommu_iotlb_gather *gather,
958 unsigned long iova, size_t size)
959{
960}
961
962static inline void iommu_device_unregister(struct iommu_device *iommu)
963{
964}
965
966static inline int iommu_device_sysfs_add(struct iommu_device *iommu,
967 struct device *parent,
968 const struct attribute_group **groups,
969 const char *fmt, ...)
970{
971 return -ENODEV;
972}
973
974static inline void iommu_device_sysfs_remove(struct iommu_device *iommu)
975{
976}
977
978static inline int iommu_device_link(struct device *dev, struct device *link)
979{
980 return -EINVAL;
981}
982
983static inline void iommu_device_unlink(struct device *dev, struct device *link)
984{
985}
986
987static inline int iommu_fwspec_init(struct device *dev,
988 struct fwnode_handle *iommu_fwnode,
989 const struct iommu_ops *ops)
990{
991 return -ENODEV;
992}
993
994static inline void iommu_fwspec_free(struct device *dev)
995{
996}
997
998static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids,
999 int num_ids)
1000{
1001 return -ENODEV;
1002}
1003
1004static inline
1005const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
1006{
1007 return NULL;
1008}
1009
1010static inline bool
1011iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat)
1012{
1013 return false;
1014}
1015
1016static inline bool
1017iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat)
1018{
1019 return false;
1020}
1021
1022static inline int
1023iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
1024{
1025 return -ENODEV;
1026}
1027
1028static inline int
1029iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
1030{
1031 return -ENODEV;
1032}
1033
1034static inline int
1035iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev)
1036{
1037 return -ENODEV;
1038}
1039
1040static inline void
1041iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev)
1042{
1043}
1044
1045static inline int
1046iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
1047{
1048 return -ENODEV;
1049}
1050
1051static inline struct iommu_sva *
1052iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
1053{
1054 return NULL;
1055}
1056
1057static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
1058{
1059}
1060
1061static inline int iommu_sva_set_ops(struct iommu_sva *handle,
1062 const struct iommu_sva_ops *ops)
1063{
1064 return -EINVAL;
1065}
1066
1067static inline int iommu_sva_get_pasid(struct iommu_sva *handle)
1068{
1069 return IOMMU_PASID_INVALID;
1070}
1071
1072static inline int
1073iommu_cache_invalidate(struct iommu_domain *domain,
1074 struct device *dev,
1075 struct iommu_cache_invalidate_info *inv_info)
1076{
1077 return -ENODEV;
1078}
1079static inline int iommu_sva_bind_gpasid(struct iommu_domain *domain,
1080 struct device *dev, struct iommu_gpasid_bind_data *data)
1081{
1082 return -ENODEV;
1083}
1084
1085static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
1086 struct device *dev, int pasid)
1087{
1088 return -ENODEV;
1089}
1090
1091static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
1092{
1093 return NULL;
1094}
1095#endif /* CONFIG_IOMMU_API */
1096
1097#ifdef CONFIG_IOMMU_DEBUGFS
1098extern struct dentry *iommu_debugfs_dir;
1099void iommu_debugfs_setup(void);
1100#else
1101static inline void iommu_debugfs_setup(void) {}
1102#endif
1103
1104#endif /* __LINUX_IOMMU_H */