Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030// SPDX-License-Identifier: GPL-2.0-only#include <linux/alloc_tag.h>#include <linux/execmem.h>#include <linux/fs.h>#include <linux/gfp.h>#include <linux/kallsyms.h>#include <linux/module.h>#include <linux/page_ext.h>#include <linux/pgalloc_tag.h>#include <linux/proc_fs.h>#include <linux/rcupdate.h>#include <linux/seq_buf.h>#include <linux/seq_file.h>#include <linux/string_choices.h>#include <linux/vmalloc.h>#include <linux/kmemleak.h>
#define ALLOCINFO_FILE_NAME "allocinfo"#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULTstatic bool mem_profiling_support = true;#elsestatic bool mem_profiling_support;#endif
/* * Memory allocation profiling is permanently disabled and cannot be enabled. * Must be called after setup_early_mem_profiling(). */bool mem_alloc_profiling_permanently_disabled(void){ return !mem_profiling_support;}
static struct codetag_type *alloc_tag_cttype;
#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPUDEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);EXPORT_SYMBOL(_shared_alloc_tag);#endif
DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, mem_alloc_profiling_key);EXPORT_SYMBOL(mem_alloc_profiling_key);
DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };unsigned long alloc_tag_ref_mask;int alloc_tag_ref_offs;
struct allocinfo_private { struct codetag_iterator iter; struct codetag_iterator reported_iter; bool print_header;};
static void *allocinfo_start(struct seq_file *m, loff_t *pos){ struct allocinfo_private *priv; loff_t node = *pos;
priv = (struct allocinfo_private *)m->private; codetag_lock_module_list(alloc_tag_cttype); if (node == 0) { priv->print_header = true; priv->iter = codetag_get_ct_iter(alloc_tag_cttype); } else { priv->iter = priv->reported_iter; } codetag_next_ct(&priv->iter); return priv->iter.ct ? priv : NULL;}
static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos){ struct allocinfo_private *priv = (struct allocinfo_private *)arg; struct codetag *ct;
priv->reported_iter = priv->iter; ct = codetag_next_ct(&priv->iter); (*pos)++; if (!ct) return NULL;
return priv;}
static void allocinfo_stop(struct seq_file *m, void *arg){ codetag_unlock_module_list(alloc_tag_cttype);}
static void print_allocinfo_header(struct seq_buf *buf){ /* Output format version, so we can change it. */ seq_buf_printf(buf, "allocinfo - version: 2.0\n"); seq_buf_printf(buf, "# <size> <calls> <tag info>\n");}
static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct){ struct alloc_tag *tag = ct_to_alloc_tag(ct); struct alloc_tag_counters counter = alloc_tag_read(tag); s64 bytes = counter.bytes;
seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls); codetag_to_text(out, ct); if (unlikely(alloc_tag_is_inaccurate(tag))) seq_buf_printf(out, " accurate:no"); seq_buf_putc(out, ' '); seq_buf_putc(out, '\n');}
static int allocinfo_show(struct seq_file *m, void *arg){ struct allocinfo_private *priv = (struct allocinfo_private *)arg; char *bufp; size_t n = seq_get_buf(m, &bufp); struct seq_buf buf;
seq_buf_init(&buf, bufp, n); if (priv->print_header) { print_allocinfo_header(&buf); priv->print_header = false; } alloc_tag_to_text(&buf, priv->iter.ct); seq_commit(m, seq_buf_used(&buf)); return 0;}
static const struct seq_operations allocinfo_seq_op = { .start = allocinfo_start, .next = allocinfo_next, .stop = allocinfo_stop, .show = allocinfo_show,};
size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep){ struct codetag_iterator iter; struct codetag *ct; struct codetag_bytes n; unsigned int i, nr = 0;
if (IS_ERR_OR_NULL(alloc_tag_cttype)) return 0;
if (can_sleep) codetag_lock_module_list(alloc_tag_cttype); else if (!codetag_trylock_module_list(alloc_tag_cttype)) return 0;
iter = codetag_get_ct_iter(alloc_tag_cttype); while ((ct = codetag_next_ct(&iter))) { struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
n.ct = ct; n.bytes = counter.bytes;
for (i = 0; i < nr; i++) if (n.bytes > tags[i].bytes) break;
if (i < count) { nr -= nr == count; memmove(&tags[i + 1], &tags[i], sizeof(tags[0]) * (nr - i)); nr++; tags[i] = n; } }
codetag_unlock_module_list(alloc_tag_cttype);
return nr;}
void pgalloc_tag_split(struct folio *folio, int old_order, int new_order){ int i; struct alloc_tag *tag; unsigned int nr_pages = 1 << new_order;
if (!mem_alloc_profiling_enabled()) return;
tag = __pgalloc_tag_get(&folio->page); if (!tag) return;
for (i = nr_pages; i < (1 << old_order); i += nr_pages) { union pgtag_ref_handle handle; union codetag_ref ref;
if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) { /* Set new reference to point to the original tag */ alloc_tag_ref_set(&ref, tag); update_page_tag_ref(handle, &ref); put_page_tag_ref(handle); } }}
void pgalloc_tag_swap(struct folio *new, struct folio *old){ union pgtag_ref_handle handle_old, handle_new; union codetag_ref ref_old, ref_new; struct alloc_tag *tag_old, *tag_new;
if (!mem_alloc_profiling_enabled()) return;
tag_old = __pgalloc_tag_get(&old->page); if (!tag_old) return; tag_new = __pgalloc_tag_get(&new->page); if (!tag_new) return;
if (!get_page_tag_ref(&old->page, &ref_old, &handle_old)) return; if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) { put_page_tag_ref(handle_old); return; }
/* * Clear tag references to avoid debug warning when using * __alloc_tag_ref_set() with non-empty reference. */ set_codetag_empty(&ref_old); set_codetag_empty(&ref_new);
/* swap tags */ __alloc_tag_ref_set(&ref_old, tag_new); update_page_tag_ref(handle_old, &ref_old); __alloc_tag_ref_set(&ref_new, tag_old); update_page_tag_ref(handle_new, &ref_new);
put_page_tag_ref(handle_old); put_page_tag_ref(handle_new);}
static void shutdown_mem_profiling(bool remove_file){ if (mem_alloc_profiling_enabled()) static_branch_disable(&mem_alloc_profiling_key);
if (!mem_profiling_support) return;
if (remove_file) remove_proc_entry(ALLOCINFO_FILE_NAME, NULL); mem_profiling_support = false;}
void __init alloc_tag_sec_init(void){ struct alloc_tag *last_codetag;
if (!mem_profiling_support) return;
if (!static_key_enabled(&mem_profiling_compressed)) return;
kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name( SECTION_START(ALLOC_TAG_SECTION_NAME)); last_codetag = (struct alloc_tag *)kallsyms_lookup_name( SECTION_STOP(ALLOC_TAG_SECTION_NAME)); kernel_tags.count = last_codetag - kernel_tags.first_tag;
/* Check if kernel tags fit into page flags */ if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); return; }
alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS); alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1); pr_debug("Memory allocation profiling compression is using %d page flag bits!\n", NR_UNUSED_PAGEFLAG_BITS);}
#ifdef CONFIG_MODULES
static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);static struct vm_struct *vm_module_tags;/* A dummy object used to indicate an unloaded module */static struct module unloaded_mod;/* A dummy object used to indicate a module prepended area */static struct module prepend_mod;
struct alloc_tag_module_section module_tags;
static inline unsigned long alloc_tag_align(unsigned long val){ if (!static_key_enabled(&mem_profiling_compressed)) { /* No alignment requirements when we are not indexing the tags */ return val; }
if (val % sizeof(struct alloc_tag) == 0) return val; return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);}
static bool ensure_alignment(unsigned long align, unsigned int *prepend){ if (!static_key_enabled(&mem_profiling_compressed)) { /* No alignment requirements when we are not indexing the tags */ return true; }
/* * If alloc_tag size is not a multiple of required alignment, tag * indexing does not work. */ if (!IS_ALIGNED(sizeof(struct alloc_tag), align)) return false;
/* Ensure prepend consumes multiple of alloc_tag-sized blocks */ if (*prepend) *prepend = alloc_tag_align(*prepend);
return true;}
static inline bool tags_addressable(void){ unsigned long tag_idx_count;
if (!static_key_enabled(&mem_profiling_compressed)) return true; /* with page_ext tags are always addressable */
tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count + module_tags.size / sizeof(struct alloc_tag);
return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);}
static bool needs_section_mem(struct module *mod, unsigned long size){ if (!mem_profiling_support) return false;
return size >= sizeof(struct alloc_tag);}
static bool clean_unused_counters(struct alloc_tag *start_tag, struct alloc_tag *end_tag){ struct alloc_tag *tag; bool ret = true;
for (tag = start_tag; tag <= end_tag; tag++) { struct alloc_tag_counters counter;
if (!tag->counters) continue;
counter = alloc_tag_read(tag); if (!counter.bytes) { free_percpu(tag->counters); tag->counters = NULL; } else { ret = false; } }
return ret;}
/* Called with mod_area_mt locked */static void clean_unused_module_areas_locked(void){ MA_STATE(mas, &mod_area_mt, 0, module_tags.size); struct module *val;
mas_for_each(&mas, val, module_tags.size) { struct alloc_tag *start_tag; struct alloc_tag *end_tag;
if (val != &unloaded_mod) continue;
/* Release area if all tags are unused */ start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); if (clean_unused_counters(start_tag, end_tag)) mas_erase(&mas); }}
/* Called with mod_area_mt locked */static bool find_aligned_area(struct ma_state *mas, unsigned long section_size, unsigned long size, unsigned int prepend, unsigned long align){ bool cleanup_done = false;
repeat: /* Try finding exact size and hope the start is aligned */ if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) { if (IS_ALIGNED(mas->index + prepend, align)) return true;
/* Try finding larger area to align later */ mas_reset(mas); if (!mas_empty_area(mas, 0, section_size - 1, size + prepend + align - 1)) return true; }
/* No free area, try cleanup stale data and repeat the search once */ if (!cleanup_done) { clean_unused_module_areas_locked(); cleanup_done = true; mas_reset(mas); goto repeat; }
return false;}
static int vm_module_tags_populate(void){ unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) + (vm_module_tags->nr_pages << PAGE_SHIFT); unsigned long new_end = module_tags.start_addr + module_tags.size;
if (phys_end < new_end) { struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages; unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN); unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN); unsigned long more_pages; unsigned long nr = 0;
more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT; while (nr < more_pages) { unsigned long allocated;
allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN, NUMA_NO_NODE, more_pages - nr, next_page + nr);
if (!allocated) break; nr += allocated; }
if (nr < more_pages || vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL, next_page, PAGE_SHIFT) < 0) { release_pages_arg arg = { .pages = next_page };
/* Clean up and error out */ release_pages(arg, nr); return -ENOMEM; }
vm_module_tags->nr_pages += nr;
/* * Kasan allocates 1 byte of shadow for every 8 bytes of data. * When kasan_alloc_module_shadow allocates shadow memory, * its unit of allocation is a page. * Therefore, here we need to align to MODULE_ALIGN. */ if (old_shadow_end < new_shadow_end) kasan_alloc_module_shadow((void *)old_shadow_end, new_shadow_end - old_shadow_end, GFP_KERNEL); }
/* * Mark the pages as accessible, now that they are mapped. * With hardware tag-based KASAN, marking is skipped for * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). */ kasan_unpoison_vmalloc((void *)module_tags.start_addr, new_end - module_tags.start_addr, KASAN_VMALLOC_PROT_NORMAL);
return 0;}
static void *reserve_module_tags(struct module *mod, unsigned long size, unsigned int prepend, unsigned long align){ unsigned long section_size = module_tags.end_addr - module_tags.start_addr; MA_STATE(mas, &mod_area_mt, 0, section_size - 1); unsigned long offset; void *ret = NULL;
/* If no tags return error */ if (size < sizeof(struct alloc_tag)) return ERR_PTR(-EINVAL);
/* * align is always power of 2, so we can use IS_ALIGNED and ALIGN. * align 0 or 1 means no alignment, to simplify set to 1. */ if (!align) align = 1;
if (!ensure_alignment(align, &prepend)) { shutdown_mem_profiling(true); pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n", mod->name, align); return ERR_PTR(-EINVAL); }
mas_lock(&mas); if (!find_aligned_area(&mas, section_size, size, prepend, align)) { ret = ERR_PTR(-ENOMEM); goto unlock; }
/* Mark found area as reserved */ offset = mas.index; offset += prepend; offset = ALIGN(offset, align); if (offset != mas.index) { unsigned long pad_start = mas.index;
mas.last = offset - 1; mas_store(&mas, &prepend_mod); if (mas_is_err(&mas)) { ret = ERR_PTR(xa_err(mas.node)); goto unlock; } mas.index = offset; mas.last = offset + size - 1; mas_store(&mas, mod); if (mas_is_err(&mas)) { mas.index = pad_start; mas_erase(&mas); ret = ERR_PTR(xa_err(mas.node)); } } else { mas.last = offset + size - 1; mas_store(&mas, mod); if (mas_is_err(&mas)) ret = ERR_PTR(xa_err(mas.node)); }unlock: mas_unlock(&mas);
if (IS_ERR(ret)) return ret;
if (module_tags.size < offset + size) { int grow_res;
module_tags.size = offset + size; if (mem_alloc_profiling_enabled() && !tags_addressable()) { shutdown_mem_profiling(true); pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n", mod->name, NR_UNUSED_PAGEFLAG_BITS); }
grow_res = vm_module_tags_populate(); if (grow_res) { shutdown_mem_profiling(true); pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n", mod->name); return ERR_PTR(grow_res); } }
return (struct alloc_tag *)(module_tags.start_addr + offset);}
static void release_module_tags(struct module *mod, bool used){ MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size); struct alloc_tag *start_tag; struct alloc_tag *end_tag; struct module *val;
mas_lock(&mas); mas_for_each_rev(&mas, val, 0) if (val == mod) break;
if (!val) /* module not found */ goto out;
if (!used) goto release_area;
start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); if (!clean_unused_counters(start_tag, end_tag)) { struct alloc_tag *tag;
for (tag = start_tag; tag <= end_tag; tag++) { struct alloc_tag_counters counter;
if (!tag->counters) continue;
counter = alloc_tag_read(tag); pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n", tag->ct.filename, tag->ct.lineno, tag->ct.modname, tag->ct.function, counter.bytes); } } else { used = false; }release_area: mas_store(&mas, used ? &unloaded_mod : NULL); val = mas_prev_range(&mas, 0); if (val == &prepend_mod) mas_store(&mas, NULL);out: mas_unlock(&mas);}
static int load_module(struct module *mod, struct codetag *start, struct codetag *stop){ /* Allocate module alloc_tag percpu counters */ struct alloc_tag *start_tag; struct alloc_tag *stop_tag; struct alloc_tag *tag;
/* percpu counters for core allocations are already statically allocated */ if (!mod) return 0;
start_tag = ct_to_alloc_tag(start); stop_tag = ct_to_alloc_tag(stop); for (tag = start_tag; tag < stop_tag; tag++) { WARN_ON(tag->counters); tag->counters = alloc_percpu(struct alloc_tag_counters); if (!tag->counters) { while (--tag >= start_tag) { free_percpu(tag->counters); tag->counters = NULL; } pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n", mod->name); return -ENOMEM; }
/* * Avoid a kmemleak false positive. The pointer to the counters is stored * in the alloc_tag section of the module and cannot be directly accessed. */ kmemleak_ignore_percpu(tag->counters); } return 0;}
static void replace_module(struct module *mod, struct module *new_mod){ MA_STATE(mas, &mod_area_mt, 0, module_tags.size); struct module *val;
mas_lock(&mas); mas_for_each(&mas, val, module_tags.size) { if (val != mod) continue;
mas_store_gfp(&mas, new_mod, GFP_KERNEL); break; } mas_unlock(&mas);}
static int __init alloc_mod_tags_mem(void){ /* Map space to copy allocation tags */ vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE); if (!vm_module_tags) { pr_err("Failed to map %lu bytes for module allocation tags\n", MODULE_ALLOC_TAG_VMAP_SIZE); module_tags.start_addr = 0; return -ENOMEM; }
vm_module_tags->pages = kmalloc_objs(struct page *, get_vm_area_size(vm_module_tags) >> PAGE_SHIFT, GFP_KERNEL | __GFP_ZERO); if (!vm_module_tags->pages) { free_vm_area(vm_module_tags); return -ENOMEM; }
module_tags.start_addr = (unsigned long)vm_module_tags->addr; module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE; /* Ensure the base is alloc_tag aligned when required for indexing */ module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
return 0;}
static void __init free_mod_tags_mem(void){ release_pages_arg arg = { .pages = vm_module_tags->pages };
module_tags.start_addr = 0; release_pages(arg, vm_module_tags->nr_pages); kfree(vm_module_tags->pages); free_vm_area(vm_module_tags);}
#else /* CONFIG_MODULES */
static inline int alloc_mod_tags_mem(void) { return 0; }static inline void free_mod_tags_mem(void) {}
#endif /* CONFIG_MODULES */
/* See: Documentation/mm/allocation-profiling.rst */static int __init setup_early_mem_profiling(char *str){ bool compressed = false; bool enable;
if (!str || !str[0]) return -EINVAL;
if (!strncmp(str, "never", 5)) { enable = false; mem_profiling_support = false; pr_info("Memory allocation profiling is disabled!\n"); } else { char *token = strsep(&str, ",");
if (kstrtobool(token, &enable)) return -EINVAL;
if (str) {
if (strcmp(str, "compressed")) return -EINVAL;
compressed = true; } mem_profiling_support = true; pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n", compressed ? "with" : "without", str_on_off(enable)); }
if (enable != mem_alloc_profiling_enabled()) { if (enable) static_branch_enable(&mem_alloc_profiling_key); else static_branch_disable(&mem_alloc_profiling_key); } if (compressed != static_key_enabled(&mem_profiling_compressed)) { if (compressed) static_branch_enable(&mem_profiling_compressed); else static_branch_disable(&mem_profiling_compressed); }
return 0;}early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
static __init bool need_page_alloc_tagging(void){ if (static_key_enabled(&mem_profiling_compressed)) return false;
return mem_profiling_support;}
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG/* * Track page allocations before page_ext is initialized. * Some pages are allocated before page_ext becomes available, leaving * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. * * Each page is cast to a pfn_pool: the first few bytes hold metadata * (next pointer and slot count), the remainder stores PFNs. */struct pfn_pool { struct pfn_pool *next; atomic_t count; unsigned long pfns[];};
#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ sizeof(unsigned long))
/* * Skip early PFN recording for a page allocation. Reuses the * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid * recursion when allocating pages for the early PFN tracking list * itself. * * Codetags of the pages allocated with __GFP_NO_CODETAG should be * cleared (via clear_page_tag_ref()) before freeing the pages to prevent * alloc_tag_sub_check() from triggering a warning. */#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT
static struct pfn_pool *current_pfn_pool __initdata;
static void __init __alloc_tag_add_early_pfn(unsigned long pfn){ struct pfn_pool *pool; int idx;
do { pool = READ_ONCE(current_pfn_pool); if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG); struct pfn_pool *new;
if (!new_page) { pr_warn_once("early PFN tracking page allocation failed\n"); return; } new = page_address(new_page); new->next = pool; atomic_set(&new->count, 0); if (cmpxchg(¤t_pfn_pool, pool, new) != pool) { clear_page_tag_ref(new_page); __free_page(new_page); continue; } pool = new; } idx = atomic_read(&pool->count); if (idx >= PFN_POOL_SIZE) continue; if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx) break; } while (1);
pool->pfns[idx] = pfn;}
typedef void alloc_tag_add_func(unsigned long pfn);static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = RCU_INITIALIZER(__alloc_tag_add_early_pfn);
void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags){ alloc_tag_add_func *alloc_tag_add;
if (static_key_enabled(&mem_profiling_compressed)) return;
/* Skip allocations for the tracking list itself to avoid recursion. */ if (gfp_flags & __GFP_NO_CODETAG) return;
rcu_read_lock(); alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr); if (alloc_tag_add) alloc_tag_add(pfn); rcu_read_unlock();}
static void __init clear_early_alloc_pfn_tag_refs(void){ struct pfn_pool *pool, *next; struct page *page; int i;
if (static_key_enabled(&mem_profiling_compressed)) return;
rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL); /* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu();
for (pool = current_pfn_pool; pool; pool = next) { int nr_pfns = atomic_read(&pool->count);
for (i = 0; i < nr_pfns; i++) { unsigned long pfn = pool->pfns[i];
if (pfn_valid(pfn)) { union pgtag_ref_handle handle; union codetag_ref ref;
if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) { /* * An early-allocated page could be freed and reallocated * after its page_ext is initialized but before we clear it. * In that case, it already has a valid tag set. * We should not overwrite that valid tag * with CODETAG_EMPTY. * * Note: there is still a small race window between checking * ref.ct and calling set_codetag_empty(). We accept this * race as it's unlikely and the extra complexity of atomic * cmpxchg is not worth it for this debug-only code path. */ if (ref.ct) { put_page_tag_ref(handle); continue; }
set_codetag_empty(&ref); update_page_tag_ref(handle, &ref); put_page_tag_ref(handle); } } }
next = pool->next; page = virt_to_page(pool); clear_page_tag_ref(page); __free_page(page); }}#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */static inline void __init clear_early_alloc_pfn_tag_refs(void) {}#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
static __init void init_page_alloc_tagging(void){ clear_early_alloc_pfn_tag_refs();}
struct page_ext_operations page_alloc_tagging_ops = { .size = sizeof(union codetag_ref), .need = need_page_alloc_tagging, .init = init_page_alloc_tagging,};EXPORT_SYMBOL(page_alloc_tagging_ops);
#ifdef CONFIG_SYSCTL/* * Not using proc_do_static_key() directly to prevent enabling profiling * after it was shut down. */static int proc_mem_profiling_handler(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos){ if (write) { /* * Call from do_sysctl_args() which is a no-op since the same * value was already set by setup_early_mem_profiling. * Return success to avoid warnings from do_sysctl_args(). */ if (!current->mm) return 0;
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG /* User can't toggle profiling while debugging */ return -EACCES;#endif if (!mem_profiling_support) return -EINVAL; }
return proc_do_static_key(table, write, buffer, lenp, ppos);}
static const struct ctl_table memory_allocation_profiling_sysctls[] = { { .procname = "mem_profiling", .data = &mem_alloc_profiling_key, .mode = 0644, .proc_handler = proc_mem_profiling_handler, },};
static void __init sysctl_init(void){ register_sysctl_init("vm", memory_allocation_profiling_sysctls);}#else /* CONFIG_SYSCTL */static inline void sysctl_init(void) {}#endif /* CONFIG_SYSCTL */
static int __init alloc_tag_init(void){ const struct codetag_type_desc desc = { .section = ALLOC_TAG_SECTION_NAME, .tag_size = sizeof(struct alloc_tag),#ifdef CONFIG_MODULES .needs_section_mem = needs_section_mem, .alloc_section_mem = reserve_module_tags, .free_section_mem = release_module_tags, .module_load = load_module, .module_replaced = replace_module,#endif }; int res;
sysctl_init();
if (!mem_profiling_support) { pr_info("Memory allocation profiling is not supported!\n"); return 0; }
if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op, sizeof(struct allocinfo_private), NULL)) { pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); shutdown_mem_profiling(false); return -ENOMEM; }
res = alloc_mod_tags_mem(); if (res) { pr_err("Failed to reserve address space for module tags, errno = %d\n", res); shutdown_mem_profiling(true); return res; }
alloc_tag_cttype = codetag_register_type(&desc); if (IS_ERR(alloc_tag_cttype)) { pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype); free_mod_tags_mem(); shutdown_mem_profiling(true); return PTR_ERR(alloc_tag_cttype); }
return 0;}module_init(alloc_tag_init);