Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309/* SPDX-License-Identifier: GPL-2.0 */#ifndef __LINUX_SWIOTLB_H#define __LINUX_SWIOTLB_H
#include <linux/device.h>#include <linux/dma-direction.h>#include <linux/init.h>#include <linux/types.h>#include <linux/limits.h>#include <linux/spinlock.h>#include <linux/workqueue.h>
struct device;struct page;struct scatterlist;
#define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */#define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */#define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */
/* * Maximum allowable number of contiguous slabs to map, * must be a power of 2. What is the appropriate value ? * The complexity of {map,unmap}_single is linearly dependent on this value. */#define IO_TLB_SEGSIZE 128
/* * log of the size of each IO TLB slab. The number of slabs is command line * controllable. */#define IO_TLB_SHIFT 11#define IO_TLB_SIZE (1 << IO_TLB_SHIFT)
/* default to 64MB */#define IO_TLB_DEFAULT_SIZE (64UL<<20)
unsigned long swiotlb_size_or_default(void);void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, int (*remap)(void *tlb, unsigned long nslabs));int swiotlb_init_late(size_t size, gfp_t gfp_mask, int (*remap)(void *tlb, unsigned long nslabs));extern void __init swiotlb_update_mem_attributes(void);
#ifdef CONFIG_SWIOTLB
/** * struct io_tlb_pool - IO TLB memory pool descriptor * @start: The start address of the swiotlb memory pool. Used to do a quick * range check to see if the memory was in fact allocated by this * API. * @end: The end address of the swiotlb memory pool. Used to do a quick * range check to see if the memory was in fact allocated by this * API. * @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool * may be remapped in the memory encrypted case and store virtual * address for bounce buffer operation. * @nslabs: The number of IO TLB slots between @start and @end. For the * default swiotlb, this can be adjusted with a boot parameter, * see setup_io_tlb_npages(). * @late_alloc: %true if allocated using the page allocator. * @nareas: Number of areas in the pool. * @area_nslabs: Number of slots in each area. * @areas: Array of memory area descriptors. * @slots: Array of slot descriptors. * @node: Member of the IO TLB memory pool list. * @rcu: RCU head for swiotlb_dyn_free(). * @transient: %true if transient memory pool. */struct io_tlb_pool { phys_addr_t start; phys_addr_t end; void *vaddr; unsigned long nslabs; bool late_alloc; unsigned int nareas; unsigned int area_nslabs; struct io_tlb_area *areas; struct io_tlb_slot *slots;#ifdef CONFIG_SWIOTLB_DYNAMIC struct list_head node; struct rcu_head rcu; bool transient;#endif};
/** * struct io_tlb_mem - Software IO TLB allocator * @defpool: Default (initial) IO TLB memory pool descriptor. * @pool: IO TLB memory pool descriptor (if not dynamic). * @nslabs: Total number of IO TLB slabs in all pools. * @debugfs: The dentry to debugfs. * @force_bounce: %true if swiotlb bouncing is forced * @for_alloc: %true if the pool is used for memory allocation * @can_grow: %true if more pools can be allocated dynamically. * @phys_limit: Maximum allowed physical address. * @lock: Lock to synchronize changes to the list. * @pools: List of IO TLB memory pool descriptors (if dynamic). * @dyn_alloc: Dynamic IO TLB pool allocation work. * @total_used: The total number of slots in the pool that are currently used * across all areas. Used only for calculating used_hiwater in * debugfs. * @used_hiwater: The high water mark for total_used. Used only for reporting * in debugfs. * @transient_nslabs: The total number of slots in all transient pools that * are currently used across all areas. */struct io_tlb_mem { struct io_tlb_pool defpool; unsigned long nslabs; struct dentry *debugfs; bool force_bounce; bool for_alloc;#ifdef CONFIG_SWIOTLB_DYNAMIC bool can_grow; u64 phys_limit; spinlock_t lock; struct list_head pools; struct work_struct dyn_alloc;#endif#ifdef CONFIG_DEBUG_FS atomic_long_t total_used; atomic_long_t used_hiwater; atomic_long_t transient_nslabs;#endif};
struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
/** * swiotlb_find_pool() - find swiotlb pool to which a physical address belongs * @dev: Device which has mapped the buffer. * @paddr: Physical address within the DMA buffer. * * Find the swiotlb pool that @paddr points into. * * Return: * * pool address if @paddr points into a bounce buffer * * NULL if @paddr does not point into a bounce buffer. As such, this function * can be used to determine if @paddr denotes a swiotlb bounce buffer. */static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr){ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
if (!mem) return NULL;
#ifdef CONFIG_SWIOTLB_DYNAMIC /* * All SWIOTLB buffer addresses must have been returned by * swiotlb_tbl_map_single() and passed to a device driver. * If a SWIOTLB address is checked on another CPU, then it was * presumably loaded by the device driver from an unspecified private * data structure. Make sure that this load is ordered before reading * dev->dma_uses_io_tlb here and mem->pools in __swiotlb_find_pool(). * * This barrier pairs with smp_mb() in swiotlb_find_slots(). */ smp_rmb(); if (READ_ONCE(dev->dma_uses_io_tlb)) return __swiotlb_find_pool(dev, paddr);#else if (paddr >= mem->defpool.start && paddr < mem->defpool.end) return &mem->defpool;#endif
return NULL;}
static inline bool is_swiotlb_force_bounce(struct device *dev){ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
return mem && mem->force_bounce;}
void swiotlb_init(bool addressing_limited, unsigned int flags);void __init swiotlb_exit(void);void swiotlb_dev_init(struct device *dev);size_t swiotlb_max_mapping_size(struct device *dev);bool is_swiotlb_allocated(void);bool is_swiotlb_active(struct device *dev);void __init swiotlb_adjust_size(unsigned long size);phys_addr_t default_swiotlb_base(void);phys_addr_t default_swiotlb_limit(void);#elsestatic inline void swiotlb_init(bool addressing_limited, unsigned int flags){}
static inline void swiotlb_dev_init(struct device *dev){}
static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr){ return NULL;}static inline bool is_swiotlb_force_bounce(struct device *dev){ return false;}static inline void swiotlb_exit(void){}static inline size_t swiotlb_max_mapping_size(struct device *dev){ return SIZE_MAX;}
static inline bool is_swiotlb_allocated(void){ return false;}
static inline bool is_swiotlb_active(struct device *dev){ return false;}
static inline void swiotlb_adjust_size(unsigned long size){}
static inline phys_addr_t default_swiotlb_base(void){ return 0;}
static inline phys_addr_t default_swiotlb_limit(void){ return 0;}#endif /* CONFIG_SWIOTLB */
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, size_t mapping_size, unsigned int alloc_aligned_mask, enum dma_data_direction dir, unsigned long attrs);dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs);
void __swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, size_t mapping_size, enum dma_data_direction dir, unsigned long attrs, struct io_tlb_pool *pool);static inline void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs){ struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
if (unlikely(pool)) __swiotlb_tbl_unmap_single(dev, addr, size, dir, attrs, pool);}
void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr, size_t size, enum dma_data_direction dir, struct io_tlb_pool *pool);static inline void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t addr, size_t size, enum dma_data_direction dir){ struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
if (unlikely(pool)) __swiotlb_sync_single_for_device(dev, addr, size, dir, pool);}
void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr, size_t size, enum dma_data_direction dir, struct io_tlb_pool *pool);static inline void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t addr, size_t size, enum dma_data_direction dir){ struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
if (unlikely(pool)) __swiotlb_sync_single_for_cpu(dev, addr, size, dir, pool);}
extern void swiotlb_print_info(void);
#ifdef CONFIG_DMA_RESTRICTED_POOLstruct page *swiotlb_alloc(struct device *dev, size_t size);bool swiotlb_free(struct device *dev, struct page *page, size_t size);
static inline bool is_swiotlb_for_alloc(struct device *dev){ return dev->dma_io_tlb_mem->for_alloc;}#elsestatic inline struct page *swiotlb_alloc(struct device *dev, size_t size){ return NULL;}static inline bool swiotlb_free(struct device *dev, struct page *page, size_t size){ return false;}static inline bool is_swiotlb_for_alloc(struct device *dev){ return false;}#endif /* CONFIG_DMA_RESTRICTED_POOL */
#endif /* __LINUX_SWIOTLB_H */