Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_KASAN_H#define _LINUX_KASAN_H
#include <linux/bug.h>#include <linux/kasan-enabled.h>#include <linux/kasan-tags.h>#include <linux/kernel.h>#include <linux/static_key.h>#include <linux/types.h>
struct kmem_cache;struct page;struct slab;struct vm_struct;struct task_struct;
#ifdef CONFIG_KASAN
#include <linux/linkage.h>#include <asm/kasan.h>
#endif
typedef unsigned int __bitwise kasan_vmalloc_flags_t;
#define KASAN_VMALLOC_NONE ((__force kasan_vmalloc_flags_t)0x00u)#define KASAN_VMALLOC_INIT ((__force kasan_vmalloc_flags_t)0x01u)#define KASAN_VMALLOC_VM_ALLOC ((__force kasan_vmalloc_flags_t)0x02u)#define KASAN_VMALLOC_PROT_NORMAL ((__force kasan_vmalloc_flags_t)0x04u)#define KASAN_VMALLOC_KEEP_TAG ((__force kasan_vmalloc_flags_t)0x08u)
#define KASAN_VMALLOC_PAGE_RANGE 0x1 /* Apply exsiting page range */#define KASAN_VMALLOC_TLB_FLUSH 0x2 /* TLB flush */
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
#include <linux/pgtable.h>
/* Software KASAN implementations use shadow memory. */
#ifdef CONFIG_KASAN_SW_TAGS/* This matches KASAN_TAG_INVALID. */#define KASAN_SHADOW_INIT 0xFE#else#define KASAN_SHADOW_INIT 0#endif
#ifndef PTE_HWTABLE_PTRS#define PTE_HWTABLE_PTRS 0#endif
extern unsigned char kasan_early_shadow_page[PAGE_SIZE];extern pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS];extern pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD];extern pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD];extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];
int kasan_populate_early_shadow(const void *shadow_start, const void *shadow_end);
#ifndef kasan_mem_to_shadowstatic inline void *kasan_mem_to_shadow(const void *addr){ return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT) + KASAN_SHADOW_OFFSET;}#endif
int kasan_add_zero_shadow(void *start, unsigned long size);void kasan_remove_zero_shadow(void *start, unsigned long size);
/* Enable reporting bugs after kasan_disable_current() */extern void kasan_enable_current(void);
/* Disable reporting bugs for current task */extern void kasan_disable_current(void);
#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
static inline int kasan_add_zero_shadow(void *start, unsigned long size){ return 0;}static inline void kasan_remove_zero_shadow(void *start, unsigned long size){}
static inline void kasan_enable_current(void) {}static inline void kasan_disable_current(void) {}
#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
#ifdef CONFIG_KASAN_HW_TAGS
#else /* CONFIG_KASAN_HW_TAGS */
#endif /* CONFIG_KASAN_HW_TAGS */
static inline bool kasan_has_integrated_init(void){ return kasan_hw_tags_enabled();}
#ifdef CONFIG_KASANvoid __kasan_unpoison_range(const void *addr, size_t size);static __always_inline void kasan_unpoison_range(const void *addr, size_t size){ if (kasan_enabled()) __kasan_unpoison_range(addr, size);}
void __kasan_poison_pages(struct page *page, unsigned int order, bool init);static __always_inline void kasan_poison_pages(struct page *page, unsigned int order, bool init){ if (kasan_enabled()) __kasan_poison_pages(page, order, init);}
bool __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);static __always_inline bool kasan_unpoison_pages(struct page *page, unsigned int order, bool init){ if (kasan_enabled()) return __kasan_unpoison_pages(page, order, init); return false;}
void __kasan_poison_slab(struct slab *slab);static __always_inline void kasan_poison_slab(struct slab *slab){ if (kasan_enabled()) __kasan_poison_slab(slab);}
void __kasan_unpoison_new_object(struct kmem_cache *cache, void *object);/** * kasan_unpoison_new_object - Temporarily unpoison a new slab object. * @cache: Cache the object belong to. * @object: Pointer to the object. * * This function is intended for the slab allocator's internal use. It * temporarily unpoisons an object from a newly allocated slab without doing * anything else. The object must later be repoisoned by * kasan_poison_new_object(). */static __always_inline void kasan_unpoison_new_object(struct kmem_cache *cache, void *object){ if (kasan_enabled()) __kasan_unpoison_new_object(cache, object);}
void __kasan_poison_new_object(struct kmem_cache *cache, void *object);/** * kasan_poison_new_object - Repoison a new slab object. * @cache: Cache the object belong to. * @object: Pointer to the object. * * This function is intended for the slab allocator's internal use. It * repoisons an object that was previously unpoisoned by * kasan_unpoison_new_object() without doing anything else. */static __always_inline void kasan_poison_new_object(struct kmem_cache *cache, void *object){ if (kasan_enabled()) __kasan_poison_new_object(cache, object);}
void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache, const void *object);static __always_inline void * __must_check kasan_init_slab_obj( struct kmem_cache *cache, const void *object){ if (kasan_enabled()) return __kasan_init_slab_obj(cache, object); return (void *)object;}
bool __kasan_slab_pre_free(struct kmem_cache *s, void *object, unsigned long ip);/** * kasan_slab_pre_free - Check whether freeing a slab object is safe. * @object: Object to be freed. * * This function checks whether freeing the given object is safe. It may * check for double-free and invalid-free bugs and report them. * * This function is intended only for use by the slab allocator. * * @Return true if freeing the object is unsafe; false otherwise. */static __always_inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object){ if (kasan_enabled()) return __kasan_slab_pre_free(s, object, _RET_IP_); return false;}
bool __kasan_slab_free(struct kmem_cache *s, void *object, bool init, bool still_accessible, bool no_quarantine);/** * kasan_slab_free - Poison, initialize, and quarantine a slab object. * @object: Object to be freed. * @init: Whether to initialize the object. * @still_accessible: Whether the object contents are still accessible. * * This function informs that a slab object has been freed and is not * supposed to be accessed anymore, except when @still_accessible is set * (indicating that the object is in a SLAB_TYPESAFE_BY_RCU cache and an RCU * grace period might not have passed yet). * * For KASAN modes that have integrated memory initialization * (kasan_has_integrated_init() == true), this function also initializes * the object's memory. For other modes, the @init argument is ignored. * * This function might also take ownership of the object to quarantine it. * When this happens, KASAN will defer freeing the object to a later * stage and handle it internally until then. The return value indicates * whether KASAN took ownership of the object. * * This function is intended only for use by the slab allocator. * * @Return true if KASAN took ownership of the object; false otherwise. */static __always_inline bool kasan_slab_free(struct kmem_cache *s, void *object, bool init, bool still_accessible, bool no_quarantine){ if (kasan_enabled()) return __kasan_slab_free(s, object, init, still_accessible, no_quarantine); return false;}
void __kasan_kfree_large(void *ptr, unsigned long ip);static __always_inline void kasan_kfree_large(void *ptr){ if (kasan_enabled()) __kasan_kfree_large(ptr, _RET_IP_);}
void * __must_check __kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags, bool init);static __always_inline void * __must_check kasan_slab_alloc( struct kmem_cache *s, void *object, gfp_t flags, bool init){ if (kasan_enabled()) return __kasan_slab_alloc(s, object, flags, init); return object;}
void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size, gfp_t flags);static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size, gfp_t flags){ if (kasan_enabled()) return __kasan_kmalloc(s, object, size, flags); return (void *)object;}
void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags);static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags){ if (kasan_enabled()) return __kasan_kmalloc_large(ptr, size, flags); return (void *)ptr;}
void * __must_check __kasan_krealloc(const void *object, size_t new_size, gfp_t flags);static __always_inline void * __must_check kasan_krealloc(const void *object, size_t new_size, gfp_t flags){ if (kasan_enabled()) return __kasan_krealloc(object, new_size, flags); return (void *)object;}
bool __kasan_mempool_poison_pages(struct page *page, unsigned int order, unsigned long ip);/** * kasan_mempool_poison_pages - Check and poison a mempool page allocation. * @page: Pointer to the page allocation. * @order: Order of the allocation. * * This function is intended for kernel subsystems that cache page allocations * to reuse them instead of freeing them back to page_alloc (e.g. mempool). * * This function is similar to kasan_mempool_poison_object() but operates on * page allocations. * * Before the poisoned allocation can be reused, it must be unpoisoned via * kasan_mempool_unpoison_pages(). * * Return: true if the allocation can be safely reused; false otherwise. */static __always_inline bool kasan_mempool_poison_pages(struct page *page, unsigned int order){ if (kasan_enabled()) return __kasan_mempool_poison_pages(page, order, _RET_IP_); return true;}
void __kasan_mempool_unpoison_pages(struct page *page, unsigned int order, unsigned long ip);/** * kasan_mempool_unpoison_pages - Unpoison a mempool page allocation. * @page: Pointer to the page allocation. * @order: Order of the allocation. * * This function is intended for kernel subsystems that cache page allocations * to reuse them instead of freeing them back to page_alloc (e.g. mempool). * * This function unpoisons a page allocation that was previously poisoned by * kasan_mempool_poison_pages() without zeroing the allocation's memory. For * the tag-based modes, this function assigns a new tag to the allocation. */static __always_inline void kasan_mempool_unpoison_pages(struct page *page, unsigned int order){ if (kasan_enabled()) __kasan_mempool_unpoison_pages(page, order, _RET_IP_);}
bool __kasan_mempool_poison_object(void *ptr, unsigned long ip);/** * kasan_mempool_poison_object - Check and poison a mempool slab allocation. * @ptr: Pointer to the slab allocation. * * This function is intended for kernel subsystems that cache slab allocations * to reuse them instead of freeing them back to the slab allocator (e.g. * mempool). * * This function poisons a slab allocation and saves a free stack trace for it * without initializing the allocation's memory and without putting it into the * quarantine (for the Generic mode). * * This function also performs checks to detect double-free and invalid-free * bugs and reports them. The caller can use the return value of this function * to find out if the allocation is buggy. * * Before the poisoned allocation can be reused, it must be unpoisoned via * kasan_mempool_unpoison_object(). * * This function operates on all slab allocations including large kmalloc * allocations (i.e. the ones backed directly by the buddy allocator rather * than kmalloc slab caches). * * Return: true if the allocation can be safely reused; false otherwise. */static __always_inline bool kasan_mempool_poison_object(void *ptr){ if (kasan_enabled()) return __kasan_mempool_poison_object(ptr, _RET_IP_); return true;}
void __kasan_mempool_unpoison_object(void *ptr, size_t size, unsigned long ip);/** * kasan_mempool_unpoison_object - Unpoison a mempool slab allocation. * @ptr: Pointer to the slab allocation. * @size: Size to be unpoisoned. * * This function is intended for kernel subsystems that cache slab allocations * to reuse them instead of freeing them back to the slab allocator (e.g. * mempool). * * This function unpoisons a slab allocation that was previously poisoned via * kasan_mempool_poison_object() and saves an alloc stack trace for it without * initializing the allocation's memory. For the tag-based modes, this function * does not assign a new tag to the allocation and instead restores the * original tags based on the pointer value. * * This function operates on all slab allocations including large kmalloc * allocations (i.e. the ones backed directly by the buddy allocator rather * than kmalloc slab caches). */static __always_inline void kasan_mempool_unpoison_object(void *ptr, size_t size){ if (kasan_enabled()) __kasan_mempool_unpoison_object(ptr, size, _RET_IP_);}
/* * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for * the hardware tag-based mode that doesn't rely on compiler instrumentation. */bool __kasan_check_byte(const void *addr, unsigned long ip);static __always_inline bool kasan_check_byte(const void *addr){ if (kasan_enabled()) return __kasan_check_byte(addr, _RET_IP_); return true;}
#else /* CONFIG_KASAN */
static inline void kasan_unpoison_range(const void *address, size_t size) {}static inline void kasan_poison_pages(struct page *page, unsigned int order, bool init) {}static inline bool kasan_unpoison_pages(struct page *page, unsigned int order, bool init){ return false;}static inline void kasan_poison_slab(struct slab *slab) {}static inline void kasan_unpoison_new_object(struct kmem_cache *cache, void *object) {}static inline void kasan_poison_new_object(struct kmem_cache *cache, void *object) {}static inline void *kasan_init_slab_obj(struct kmem_cache *cache, const void *object){ return (void *)object;}
static inline bool kasan_slab_pre_free(struct kmem_cache *s, void *object){ return false;}
static inline bool kasan_slab_free(struct kmem_cache *s, void *object, bool init, bool still_accessible, bool no_quarantine){ return false;}static inline void kasan_kfree_large(void *ptr) {}static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags, bool init){ return object;}static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size, gfp_t flags){ return (void *)object;}static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags){ return (void *)ptr;}static inline void *kasan_krealloc(const void *object, size_t new_size, gfp_t flags){ return (void *)object;}static inline bool kasan_mempool_poison_pages(struct page *page, unsigned int order){ return true;}static inline void kasan_mempool_unpoison_pages(struct page *page, unsigned int order) {}static inline bool kasan_mempool_poison_object(void *ptr){ return true;}static inline void kasan_mempool_unpoison_object(void *ptr, size_t size) {}
static inline bool kasan_check_byte(const void *address){ return true;}
#endif /* CONFIG_KASAN */
#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)void kasan_unpoison_task_stack(struct task_struct *task);asmlinkage void kasan_unpoison_task_stack_below(const void *watermark);#elsestatic inline void kasan_unpoison_task_stack(struct task_struct *task) {}static inline void kasan_unpoison_task_stack_below(const void *watermark) {}#endif
#ifdef CONFIG_KASAN_GENERIC
struct kasan_cache { int alloc_meta_offset; int free_meta_offset;};
size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object);void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags);
void kasan_cache_shrink(struct kmem_cache *cache);void kasan_cache_shutdown(struct kmem_cache *cache);void kasan_record_aux_stack(void *ptr);
#else /* CONFIG_KASAN_GENERIC */
/* Tag-based KASAN modes do not use per-object metadata. */static inline size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object){ return 0;}/* And no cache-related metadata initialization is required. */static inline void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags) {}
static inline void kasan_cache_shrink(struct kmem_cache *cache) {}static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}static inline void kasan_record_aux_stack(void *ptr) {}
#endif /* CONFIG_KASAN_GENERIC */
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
static inline void *kasan_reset_tag(const void *addr){ return (void *)arch_kasan_reset_tag(addr);}
/** * kasan_report - print a report about a bad memory access detected by KASAN * @addr: address of the bad access * @size: size of the bad access * @is_write: whether the bad access is a write or a read * @ip: instruction pointer for the accessibility check or the bad access itself */bool kasan_report(const void *addr, size_t size, bool is_write, unsigned long ip);
#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline void *kasan_reset_tag(const void *addr){ return (void *)addr;}
#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/
#ifdef CONFIG_KASAN_HW_TAGS
void kasan_report_async(void);
#endif /* CONFIG_KASAN_HW_TAGS */
#ifdef CONFIG_KASAN_GENERICvoid __init kasan_init_generic(void);#elsestatic inline void kasan_init_generic(void) { }#endif
#ifdef CONFIG_KASAN_SW_TAGSvoid __init kasan_init_sw_tags(void);#elsestatic inline void kasan_init_sw_tags(void) { }#endif
#ifdef CONFIG_KASAN_HW_TAGSvoid kasan_init_hw_tags_cpu(void);void __init kasan_init_hw_tags(void);#elsestatic inline void kasan_init_hw_tags_cpu(void) { }static inline void kasan_init_hw_tags(void) { }#endif
#ifdef CONFIG_KASAN_VMALLOC
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);int __kasan_populate_vmalloc(unsigned long addr, unsigned long size, gfp_t gfp_mask);static inline int kasan_populate_vmalloc(unsigned long addr, unsigned long size, gfp_t gfp_mask){ if (kasan_enabled()) return __kasan_populate_vmalloc(addr, size, gfp_mask); return 0;}void __kasan_release_vmalloc(unsigned long start, unsigned long end, unsigned long free_region_start, unsigned long free_region_end, unsigned long flags);static inline void kasan_release_vmalloc(unsigned long start, unsigned long end, unsigned long free_region_start, unsigned long free_region_end, unsigned long flags){ if (kasan_enabled()) return __kasan_release_vmalloc(start, end, free_region_start, free_region_end, flags);}
#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
static inline void kasan_populate_early_vm_area_shadow(void *start, unsigned long size){ }static inline int kasan_populate_vmalloc(unsigned long start, unsigned long size, gfp_t gfp_mask){ return 0;}static inline void kasan_release_vmalloc(unsigned long start, unsigned long end, unsigned long free_region_start, unsigned long free_region_end, unsigned long flags) { }
#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, kasan_vmalloc_flags_t flags);static __always_inline void *kasan_unpoison_vmalloc(const void *start, unsigned long size, kasan_vmalloc_flags_t flags){ if (kasan_enabled()) return __kasan_unpoison_vmalloc(start, size, flags); return (void *)start;}
void __kasan_poison_vmalloc(const void *start, unsigned long size);static __always_inline void kasan_poison_vmalloc(const void *start, unsigned long size){ if (kasan_enabled()) __kasan_poison_vmalloc(start, size);}
void __kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, kasan_vmalloc_flags_t flags);static __always_inline voidkasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, kasan_vmalloc_flags_t flags){ if (kasan_enabled()) __kasan_unpoison_vmap_areas(vms, nr_vms, flags);}
void __kasan_vrealloc(const void *start, unsigned long old_size, unsigned long new_size);
static __always_inline void kasan_vrealloc(const void *start, unsigned long old_size, unsigned long new_size){ if (kasan_enabled()) __kasan_vrealloc(start, old_size, new_size);}
#else /* CONFIG_KASAN_VMALLOC */
static inline void kasan_populate_early_vm_area_shadow(void *start, unsigned long size) { }static inline int kasan_populate_vmalloc(unsigned long start, unsigned long size, gfp_t gfp_mask){ return 0;}static inline void kasan_release_vmalloc(unsigned long start, unsigned long end, unsigned long free_region_start, unsigned long free_region_end, unsigned long flags) { }
static inline void *kasan_unpoison_vmalloc(const void *start, unsigned long size, kasan_vmalloc_flags_t flags){ return (void *)start;}static inline void kasan_poison_vmalloc(const void *start, unsigned long size){ }
static __always_inline voidkasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, kasan_vmalloc_flags_t flags){ }
static inline void kasan_vrealloc(const void *start, unsigned long old_size, unsigned long new_size) { }
#endif /* CONFIG_KASAN_VMALLOC */
#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \ !defined(CONFIG_KASAN_VMALLOC)
/* * These functions allocate and free shadow memory for kernel modules. * They are only required when KASAN_VMALLOC is not supported, as otherwise * shadow memory is allocated by the generic vmalloc handlers. */int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask);void kasan_free_module_shadow(const struct vm_struct *vm);
#else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
static inline int kasan_alloc_module_shadow(void *addr, size_t size, gfp_t gfp_mask) { return 0; }static inline void kasan_free_module_shadow(const struct vm_struct *vm) {}
#endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)void kasan_non_canonical_hook(unsigned long addr);#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */static inline void kasan_non_canonical_hook(unsigned long addr) { }#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
#endif /* LINUX_KASAN_H */