Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401/* SPDX-License-Identifier: GPL-2.0-only *//* * Copyright (C) 2021 Intel Corporation * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */#ifndef __LINUX_IOMMUFD_H#define __LINUX_IOMMUFD_H
#include <linux/err.h>#include <linux/errno.h>#include <linux/iommu.h>#include <linux/refcount.h>#include <linux/types.h>#include <linux/xarray.h>#include <uapi/linux/iommufd.h>
struct device;struct file;struct iommu_group;struct iommu_user_data;struct iommu_user_data_array;struct iommufd_access;struct iommufd_ctx;struct iommufd_device;struct iommufd_viommu_ops;struct page;
enum iommufd_object_type { IOMMUFD_OBJ_NONE, IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE, IOMMUFD_OBJ_DEVICE, IOMMUFD_OBJ_HWPT_PAGING, IOMMUFD_OBJ_HWPT_NESTED, IOMMUFD_OBJ_IOAS, IOMMUFD_OBJ_ACCESS, IOMMUFD_OBJ_FAULT, IOMMUFD_OBJ_VIOMMU, IOMMUFD_OBJ_VDEVICE, IOMMUFD_OBJ_VEVENTQ, IOMMUFD_OBJ_HW_QUEUE,#ifdef CONFIG_IOMMUFD_TEST IOMMUFD_OBJ_SELFTEST,#endif IOMMUFD_OBJ_MAX,};
/* Base struct for all objects with a userspace ID handle. */struct iommufd_object { /* * Destroy will sleep and wait for wait_cnt to go to zero. This allows * concurrent users of the ID to reliably avoid causing a spurious * destroy failure. Incrementing this count should either be short * lived or be revoked and blocked during pre_destroy(). */ refcount_t wait_cnt; refcount_t users; enum iommufd_object_type type; unsigned int id;};
struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx, struct device *dev, u32 *id);void iommufd_device_unbind(struct iommufd_device *idev);
int iommufd_device_attach(struct iommufd_device *idev, ioasid_t pasid, u32 *pt_id);int iommufd_device_replace(struct iommufd_device *idev, ioasid_t pasid, u32 *pt_id);void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid);
struct iommufd_ctx *iommufd_device_to_ictx(struct iommufd_device *idev);u32 iommufd_device_to_id(struct iommufd_device *idev);
struct iommufd_access_ops { u8 needs_pin_pages : 1; void (*unmap)(void *data, unsigned long iova, unsigned long length);};
enum { IOMMUFD_ACCESS_RW_READ = 0, IOMMUFD_ACCESS_RW_WRITE = 1 << 0, /* Set if the caller is in a kthread then rw will use kthread_use_mm() */ IOMMUFD_ACCESS_RW_KTHREAD = 1 << 1,
/* Only for use by selftest */ __IOMMUFD_ACCESS_RW_SLOW_PATH = 1 << 2,};
struct iommufd_access *iommufd_access_create(struct iommufd_ctx *ictx, const struct iommufd_access_ops *ops, void *data, u32 *id);void iommufd_access_destroy(struct iommufd_access *access);int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id);int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id);void iommufd_access_detach(struct iommufd_access *access);
void iommufd_ctx_get(struct iommufd_ctx *ictx);
struct iommufd_viommu { struct iommufd_object obj; struct iommufd_ctx *ictx; struct iommu_device *iommu_dev; struct iommufd_hwpt_paging *hwpt;
const struct iommufd_viommu_ops *ops;
struct xarray vdevs; struct list_head veventqs; struct rw_semaphore veventqs_rwsem;
enum iommu_viommu_type type;};
struct iommufd_vdevice { struct iommufd_object obj; struct iommufd_viommu *viommu; struct iommufd_device *idev;
/* * Virtual device ID per vIOMMU, e.g. vSID of ARM SMMUv3, vDeviceID of * AMD IOMMU, and vRID of Intel VT-d */ u64 virt_id;
/* Clean up all driver-specific parts of an iommufd_vdevice */ void (*destroy)(struct iommufd_vdevice *vdev);};
struct iommufd_hw_queue { struct iommufd_object obj; struct iommufd_viommu *viommu; struct iommufd_access *access;
u64 base_addr; /* in guest physical address space */ size_t length;
enum iommu_hw_queue_type type;
/* Clean up all driver-specific parts of an iommufd_hw_queue */ void (*destroy)(struct iommufd_hw_queue *hw_queue);};
/** * struct iommufd_viommu_ops - vIOMMU specific operations * @destroy: Clean up all driver-specific parts of an iommufd_viommu. The memory * of the vIOMMU will be free-ed by iommufd core after calling this op * @alloc_domain_nested: Allocate a IOMMU_DOMAIN_NESTED on a vIOMMU that holds a * nesting parent domain (IOMMU_DOMAIN_PAGING). @user_data * must be defined in include/uapi/linux/iommufd.h. * It must fully initialize the new iommu_domain before * returning. Upon failure, ERR_PTR must be returned. * @cache_invalidate: Flush hardware cache used by a vIOMMU. It can be used for * any IOMMU hardware specific cache: TLB and device cache. * The @array passes in the cache invalidation requests, in * form of a driver data structure. A driver must update the * array->entry_num to report the number of handled requests. * The data structure of the array entry must be defined in * include/uapi/linux/iommufd.h * @vdevice_size: Size of the driver-defined vDEVICE structure per this vIOMMU * @vdevice_init: Initialize the driver-level structure of a vDEVICE object, or * related HW procedure. @vdev is already initialized by iommufd * core: vdev->dev and vdev->viommu pointers; vdev->id carries a * per-vIOMMU virtual ID (refer to struct iommu_vdevice_alloc in * include/uapi/linux/iommufd.h) * If driver has a deinit function to revert what vdevice_init op * does, it should set it to the @vdev->destroy function pointer * @get_hw_queue_size: Get the size of a driver-defined HW queue structure for a * given @viommu corresponding to @queue_type. Driver should * return 0 if HW queue aren't supported accordingly. It is * required for driver to use the HW_QUEUE_STRUCT_SIZE macro * to sanitize the driver-level HW queue structure related * to the core one * @hw_queue_init_phys: Initialize the driver-level structure of a HW queue that * is initialized with its core-level structure that holds * all the info about a guest queue memory. * Driver providing this op indicates that HW accesses the * guest queue memory via physical addresses. * @index carries the logical HW QUEUE ID per vIOMMU in a * guest VM, for a multi-queue model. @base_addr_pa carries * the physical location of the guest queue * If driver has a deinit function to revert what this op * does, it should set it to the @hw_queue->destroy pointer */struct iommufd_viommu_ops { void (*destroy)(struct iommufd_viommu *viommu); struct iommu_domain *(*alloc_domain_nested)( struct iommufd_viommu *viommu, u32 flags, const struct iommu_user_data *user_data); int (*cache_invalidate)(struct iommufd_viommu *viommu, struct iommu_user_data_array *array); const size_t vdevice_size; int (*vdevice_init)(struct iommufd_vdevice *vdev); size_t (*get_hw_queue_size)(struct iommufd_viommu *viommu, enum iommu_hw_queue_type queue_type); /* AMD's HW will add hw_queue_init simply using @hw_queue->base_addr */ int (*hw_queue_init_phys)(struct iommufd_hw_queue *hw_queue, u32 index, phys_addr_t base_addr_pa);};
#if IS_ENABLED(CONFIG_IOMMUFD)struct iommufd_ctx *iommufd_ctx_from_file(struct file *file);struct iommufd_ctx *iommufd_ctx_from_fd(int fd);void iommufd_ctx_put(struct iommufd_ctx *ictx);bool iommufd_ctx_has_group(struct iommufd_ctx *ictx, struct iommu_group *group);
int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, unsigned int flags);void iommufd_access_unpin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length);int iommufd_access_rw(struct iommufd_access *access, unsigned long iova, void *data, size_t len, unsigned int flags);int iommufd_vfio_compat_ioas_get_id(struct iommufd_ctx *ictx, u32 *out_ioas_id);int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx);int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx);#else /* !CONFIG_IOMMUFD */static inline struct iommufd_ctx *iommufd_ctx_from_file(struct file *file){ return ERR_PTR(-EOPNOTSUPP);}
static inline void iommufd_ctx_put(struct iommufd_ctx *ictx){}
static inline int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, unsigned int flags){ return -EOPNOTSUPP;}
static inline void iommufd_access_unpin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length){}
static inline int iommufd_access_rw(struct iommufd_access *access, unsigned long iova, void *data, size_t len, unsigned int flags){ return -EOPNOTSUPP;}
static inline int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx){ return -EOPNOTSUPP;}
static inline int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx){ return -EOPNOTSUPP;}#endif /* CONFIG_IOMMUFD */
#if IS_ENABLED(CONFIG_IOMMUFD_DRIVER_CORE)int _iommufd_object_depend(struct iommufd_object *obj_dependent, struct iommufd_object *obj_depended);void _iommufd_object_undepend(struct iommufd_object *obj_dependent, struct iommufd_object *obj_depended);int _iommufd_alloc_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, phys_addr_t mmio_addr, size_t length, unsigned long *offset);void _iommufd_destroy_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, unsigned long offset);struct device *iommufd_vdevice_to_device(struct iommufd_vdevice *vdev);struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu, unsigned long vdev_id);int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu, struct device *dev, unsigned long *vdev_id);int iommufd_viommu_report_event(struct iommufd_viommu *viommu, enum iommu_veventq_type type, void *event_data, size_t data_len);#else /* !CONFIG_IOMMUFD_DRIVER_CORE */static inline int _iommufd_object_depend(struct iommufd_object *obj_dependent, struct iommufd_object *obj_depended){ return -EOPNOTSUPP;}
static inline void_iommufd_object_undepend(struct iommufd_object *obj_dependent, struct iommufd_object *obj_depended){}
static inline int _iommufd_alloc_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, phys_addr_t mmio_addr, size_t length, unsigned long *offset){ return -EOPNOTSUPP;}
static inline void _iommufd_destroy_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, unsigned long offset){}
static inline struct device *iommufd_vdevice_to_device(struct iommufd_vdevice *vdev){ return NULL;}
static inline struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu, unsigned long vdev_id){ return NULL;}
static inline int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu, struct device *dev, unsigned long *vdev_id){ return -ENOENT;}
static inline int iommufd_viommu_report_event(struct iommufd_viommu *viommu, enum iommu_veventq_type type, void *event_data, size_t data_len){ return -EOPNOTSUPP;}#endif /* CONFIG_IOMMUFD_DRIVER_CORE */
#define VIOMMU_STRUCT_SIZE(drv_struct, member) \ (sizeof(drv_struct) + \ BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \ BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_viommu, \ ((drv_struct *)NULL)->member)))
#define VDEVICE_STRUCT_SIZE(drv_struct, member) \ (sizeof(drv_struct) + \ BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \ BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_vdevice, \ ((drv_struct *)NULL)->member)))
#define HW_QUEUE_STRUCT_SIZE(drv_struct, member) \ (sizeof(drv_struct) + \ BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \ BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_hw_queue, \ ((drv_struct *)NULL)->member)))
/* * Helpers for IOMMU driver to build/destroy a dependency between two sibling * structures created by one of the allocators above */#define iommufd_hw_queue_depend(dependent, depended, member) \ ({ \ int ret = -EINVAL; \ \ static_assert(__same_type(struct iommufd_hw_queue, \ dependent->member)); \ static_assert(__same_type(typeof(*dependent), *depended)); \ if (!WARN_ON_ONCE(dependent->member.viommu != \ depended->member.viommu)) \ ret = _iommufd_object_depend(&dependent->member.obj, \ &depended->member.obj); \ ret; \ })
#define iommufd_hw_queue_undepend(dependent, depended, member) \ ({ \ static_assert(__same_type(struct iommufd_hw_queue, \ dependent->member)); \ static_assert(__same_type(typeof(*dependent), *depended)); \ WARN_ON_ONCE(dependent->member.viommu != \ depended->member.viommu); \ _iommufd_object_undepend(&dependent->member.obj, \ &depended->member.obj); \ })
/* * Helpers for IOMMU driver to alloc/destroy an mmapable area for a structure. * * To support an mmappable MMIO region, kernel driver must first register it to * iommufd core to allocate an @offset, during a driver-structure initialization * (e.g. viommu_init op). Then, it should report to user space this @offset and * the @length of the MMIO region for mmap syscall. */static inline int iommufd_viommu_alloc_mmap(struct iommufd_viommu *viommu, phys_addr_t mmio_addr, size_t length, unsigned long *offset){ return _iommufd_alloc_mmap(viommu->ictx, &viommu->obj, mmio_addr, length, offset);}
static inline void iommufd_viommu_destroy_mmap(struct iommufd_viommu *viommu, unsigned long offset){ _iommufd_destroy_mmap(viommu->ictx, &viommu->obj, offset);}#endif