Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
C
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_MMZONE_H#define _LINUX_MMZONE_H
#ifndef __ASSEMBLER__#ifndef __GENERATING_BOUNDS_H
#include <linux/spinlock.h>#include <linux/list.h>#include <linux/list_nulls.h>#include <linux/wait.h>#include <linux/bitops.h>#include <linux/cache.h>#include <linux/threads.h>#include <linux/numa.h>#include <linux/init.h>#include <linux/seqlock.h>#include <linux/nodemask.h>#include <linux/pageblock-flags.h>#include <linux/page-flags-layout.h>#include <linux/atomic.h>#include <linux/mm_types.h>#include <linux/page-flags.h>#include <linux/local_lock.h>#include <linux/zswap.h>#include <linux/sizes.h>#include <asm/page.h>
/* Free memory management - zoned buddy allocator. */#ifndef CONFIG_ARCH_FORCE_MAX_ORDER#define MAX_PAGE_ORDER 10#else#define MAX_PAGE_ORDER CONFIG_ARCH_FORCE_MAX_ORDER#endif#define MAX_ORDER_NR_PAGES (1 << MAX_PAGE_ORDER)
#define IS_MAX_ORDER_ALIGNED(pfn) IS_ALIGNED(pfn, MAX_ORDER_NR_PAGES)
#define NR_PAGE_ORDERS (MAX_PAGE_ORDER + 1)
/* Defines the order for the number of pages that have a migrate type. */#ifndef CONFIG_PAGE_BLOCK_MAX_ORDER#define PAGE_BLOCK_MAX_ORDER MAX_PAGE_ORDER#else#define PAGE_BLOCK_MAX_ORDER CONFIG_PAGE_BLOCK_MAX_ORDER#endif /* CONFIG_PAGE_BLOCK_MAX_ORDER */
/* * The MAX_PAGE_ORDER, which defines the max order of pages to be allocated * by the buddy allocator, has to be larger or equal to the PAGE_BLOCK_MAX_ORDER, * which defines the order for the number of pages that can have a migrate type */#if (PAGE_BLOCK_MAX_ORDER > MAX_PAGE_ORDER)#error MAX_PAGE_ORDER must be >= PAGE_BLOCK_MAX_ORDER#endif
/* * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed * costly to service. That is between allocation orders which should * coalesce naturally under reasonable reclaim pressure and those which * will not. */#define PAGE_ALLOC_COSTLY_ORDER 3
#if !defined(CONFIG_HAVE_GIGANTIC_FOLIOS)/* * We don't expect any folios that exceed buddy sizes (and consequently * memory sections). */#define MAX_FOLIO_ORDER MAX_PAGE_ORDER#elif defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)/* * Only pages within a single memory section are guaranteed to be * contiguous. By limiting folios to a single memory section, all folio * pages are guaranteed to be contiguous. */#define MAX_FOLIO_ORDER PFN_SECTION_SHIFT#elif defined(CONFIG_HUGETLB_PAGE)/* * There is no real limit on the folio size. We limit them to the maximum we * currently expect (see CONFIG_HAVE_GIGANTIC_FOLIOS): with hugetlb, we expect * no folios larger than 16 GiB on 64bit and 1 GiB on 32bit. */#ifdef CONFIG_64BIT#define MAX_FOLIO_ORDER (ilog2(SZ_16G) - PAGE_SHIFT)#else#define MAX_FOLIO_ORDER (ilog2(SZ_1G) - PAGE_SHIFT)#endif#else/* * Without hugetlb, gigantic folios that are bigger than a single PUD are * currently impossible. */#define MAX_FOLIO_ORDER (PUD_SHIFT - PAGE_SHIFT)#endif
#define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER)
/* * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to * be naturally aligned with regard to the folio size. * * HVO which is only active if the size of struct page is a power of 2. */#define MAX_FOLIO_VMEMMAP_ALIGN \ (IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && \ is_power_of_2(sizeof(struct page)) ? \ MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
/* * vmemmap optimization (like HVO) is only possible for page orders that fill * two or more pages with struct pages. */#define VMEMMAP_TAIL_MIN_ORDER (ilog2(2 * PAGE_SIZE / sizeof(struct page)))#define __NR_VMEMMAP_TAILS (MAX_FOLIO_ORDER - VMEMMAP_TAIL_MIN_ORDER + 1)#define NR_VMEMMAP_TAILS (__NR_VMEMMAP_TAILS > 0 ? __NR_VMEMMAP_TAILS : 0)
enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RECLAIMABLE, MIGRATE_PCPTYPES, /* the number of types on the pcp lists */ MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES,#ifdef CONFIG_CMA /* * MIGRATE_CMA migration type is designed to mimic the way * ZONE_MOVABLE works. Only movable pages can be allocated * from MIGRATE_CMA pageblocks and page allocator never * implicitly change migration type of MIGRATE_CMA pageblock. * * The way to use it is to change migratetype of a range of * pageblocks to MIGRATE_CMA which can be done by * __free_pageblock_cma() function. */ MIGRATE_CMA, __MIGRATE_TYPE_END = MIGRATE_CMA,#else __MIGRATE_TYPE_END = MIGRATE_HIGHATOMIC,#endif#ifdef CONFIG_MEMORY_ISOLATION MIGRATE_ISOLATE, /* can't allocate from here */#endif MIGRATE_TYPES};
/* In mm/page_alloc.c; keep in sync also with show_migration_types() there */extern const char * const migratetype_names[MIGRATE_TYPES];
#ifdef CONFIG_CMA# define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)# define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA)/* * __dump_folio() in mm/debug.c passes a folio pointer to on-stack struct folio, * so folio_pfn() cannot be used and pfn is needed. */# define is_migrate_cma_folio(folio, pfn) \ (get_pfnblock_migratetype(&folio->page, pfn) == MIGRATE_CMA)#else# define is_migrate_cma(migratetype) false# define is_migrate_cma_page(_page) false# define is_migrate_cma_folio(folio, pfn) false#endif
static inline bool is_migrate_movable(int mt){ return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;}
/* * Check whether a migratetype can be merged with another migratetype. * * It is only mergeable when it can fall back to other migratetypes for * allocation. See fallbacks[MIGRATE_TYPES][3] in page_alloc.c. */static inline bool migratetype_is_mergeable(int mt){ return mt < MIGRATE_PCPTYPES;}
#define for_each_free_list(list, zone, order) \ for (order = 0; order < NR_PAGE_ORDERS; order++) \ for (unsigned int __type = 0; \ __type < MIGRATE_TYPES && \ (list = &(zone)->free_area[order].free_list[__type], 1); \ __type++)
extern int page_group_by_mobility_disabled;
#define get_pageblock_migratetype(page) \ get_pfnblock_migratetype(page, page_to_pfn(page))
#define folio_migratetype(folio) \ get_pageblock_migratetype(&folio->page)
struct free_area { struct list_head free_list[MIGRATE_TYPES]; unsigned long nr_free;};
struct pglist_data;
#ifdef CONFIG_NUMAenum numa_stat_item { NUMA_HIT, /* allocated in intended node */ NUMA_MISS, /* allocated in non intended node */ NUMA_FOREIGN, /* was intended here, hit elsewhere */ NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */ NUMA_LOCAL, /* allocation from local node */ NUMA_OTHER, /* allocation from other node */ NR_VM_NUMA_EVENT_ITEMS};#else#define NR_VM_NUMA_EVENT_ITEMS 0#endif
enum zone_stat_item { NR_FREE_PAGES, NR_FREE_PAGES_BLOCKS, NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */ NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE, NR_ZONE_ACTIVE_ANON, NR_ZONE_INACTIVE_FILE, NR_ZONE_ACTIVE_FILE, NR_ZONE_UNEVICTABLE, NR_ZONE_WRITE_PENDING, /* Count of dirty, writeback and unstable pages */ NR_MLOCK, /* mlock()ed pages found and moved off LRU */#if IS_ENABLED(CONFIG_ZSMALLOC) NR_ZSPAGES, /* allocated in zsmalloc */#endif NR_FREE_CMA_PAGES,#ifdef CONFIG_UNACCEPTED_MEMORY NR_UNACCEPTED,#endif NR_VM_ZONE_STAT_ITEMS };
enum node_stat_item { NR_LRU_BASE, NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */ NR_ACTIVE_ANON, /* " " " " " */ NR_INACTIVE_FILE, /* " " " " " */ NR_ACTIVE_FILE, /* " " " " " */ NR_UNEVICTABLE, /* " " " " " */ NR_SLAB_RECLAIMABLE_B, NR_SLAB_UNRECLAIMABLE_B, NR_ISOLATED_ANON, /* Temporary isolated pages from anon lru */ NR_ISOLATED_FILE, /* Temporary isolated pages from file lru */ WORKINGSET_NODES, WORKINGSET_REFAULT_BASE, WORKINGSET_REFAULT_ANON = WORKINGSET_REFAULT_BASE, WORKINGSET_REFAULT_FILE, WORKINGSET_ACTIVATE_BASE, WORKINGSET_ACTIVATE_ANON = WORKINGSET_ACTIVATE_BASE, WORKINGSET_ACTIVATE_FILE, WORKINGSET_RESTORE_BASE, WORKINGSET_RESTORE_ANON = WORKINGSET_RESTORE_BASE, WORKINGSET_RESTORE_FILE, WORKINGSET_NODERECLAIM, NR_ANON_MAPPED, /* Mapped anonymous pages */ NR_FILE_MAPPED, /* pagecache pages mapped into pagetables. only modified from process context */ NR_FILE_PAGES, NR_FILE_DIRTY, NR_WRITEBACK, NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */ NR_SHMEM_THPS, NR_SHMEM_PMDMAPPED, NR_FILE_THPS, NR_FILE_PMDMAPPED, NR_ANON_THPS, NR_VMSCAN_WRITE, NR_VMSCAN_IMMEDIATE, /* Prioritise for reclaim when writeback ends */ NR_DIRTIED, /* page dirtyings since bootup */ NR_WRITTEN, /* page writings since bootup */ NR_THROTTLED_WRITTEN, /* NR_WRITTEN while reclaim throttled */ NR_KERNEL_MISC_RECLAIMABLE, /* reclaimable non-slab kernel pages */ NR_FOLL_PIN_ACQUIRED, /* via: pin_user_page(), gup flag: FOLL_PIN */ NR_FOLL_PIN_RELEASED, /* pages returned via unpin_user_page() */ NR_VMALLOC, NR_KERNEL_STACK_KB, /* measured in KiB */#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) NR_KERNEL_SCS_KB, /* measured in KiB */#endif NR_PAGETABLE, /* used for pagetables */ NR_SECONDARY_PAGETABLE, /* secondary pagetables, KVM & IOMMU */#ifdef CONFIG_IOMMU_SUPPORT NR_IOMMU_PAGES, /* # of pages allocated by IOMMU */#endif#ifdef CONFIG_SWAP NR_SWAPCACHE,#endif#ifdef CONFIG_NUMA_BALANCING PGPROMOTE_SUCCESS, /* promote successfully */ /** * Candidate pages for promotion based on hint fault latency. This * counter is used to control the promotion rate and adjust the hot * threshold. */ PGPROMOTE_CANDIDATE, /** * Not rate-limited (NRL) candidate pages for those can be promoted * without considering hot threshold because of enough free pages in * fast-tier node. These promotions bypass the regular hotness checks * and do NOT influence the promotion rate-limiter or * threshold-adjustment logic. * This is for statistics/monitoring purposes. */ PGPROMOTE_CANDIDATE_NRL,#endif /* PGDEMOTE_*: pages demoted */ PGDEMOTE_KSWAPD, PGDEMOTE_DIRECT, PGDEMOTE_KHUGEPAGED, PGDEMOTE_PROACTIVE, PGSTEAL_KSWAPD, PGSTEAL_DIRECT, PGSTEAL_KHUGEPAGED, PGSTEAL_PROACTIVE, PGSTEAL_ANON, PGSTEAL_FILE, PGSCAN_KSWAPD, PGSCAN_DIRECT, PGSCAN_KHUGEPAGED, PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, PGROTATE_ANON, PGROTATE_FILE, PGREFILL,#ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB,#endif NR_BALLOON_PAGES, NR_KERNEL_FILE_PAGES, NR_GPU_ACTIVE, /* Pages assigned to GPU objects */ NR_GPU_RECLAIM, /* Pages in shrinkable GPU pools */ NR_VM_NODE_STAT_ITEMS};
/* * Returns true if the item should be printed in THPs (/proc/vmstat * currently prints number of anon, file and shmem THPs. But the item * is charged in pages). */static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item){ if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return false;
return item == NR_ANON_THPS || item == NR_FILE_THPS || item == NR_SHMEM_THPS || item == NR_SHMEM_PMDMAPPED || item == NR_FILE_PMDMAPPED;}
/* * Returns true if the value is measured in bytes (most vmstat values are * measured in pages). This defines the API part, the internal representation * might be different. */static __always_inline bool vmstat_item_in_bytes(int idx){ /* * Global and per-node slab counters track slab pages. * It's expected that changes are multiples of PAGE_SIZE. * Internally values are stored in pages. * * Per-memcg and per-lruvec counters track memory, consumed * by individual slab objects. These counters are actually * byte-precise. */ return (idx == NR_SLAB_RECLAIMABLE_B || idx == NR_SLAB_UNRECLAIMABLE_B);}
/* * We do arithmetic on the LRU lists in various places in the code, * so it is important to keep the active lists LRU_ACTIVE higher in * the array than the corresponding inactive lists, and to keep * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists. * * This has to be kept in sync with the statistics in zone_stat_item * above and the descriptions in vmstat_text in mm/vmstat.c */#define LRU_BASE 0#define LRU_ACTIVE 1#define LRU_FILE 2
enum lru_list { LRU_INACTIVE_ANON = LRU_BASE, LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE, LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE, LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE, LRU_UNEVICTABLE, NR_LRU_LISTS};
enum vmscan_throttle_state { VMSCAN_THROTTLE_WRITEBACK, VMSCAN_THROTTLE_ISOLATED, VMSCAN_THROTTLE_NOPROGRESS, VMSCAN_THROTTLE_CONGESTED, NR_VMSCAN_THROTTLE,};
#define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
static inline bool is_file_lru(enum lru_list lru){ return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);}
static inline bool is_active_lru(enum lru_list lru){ return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);}
#define WORKINGSET_ANON 0#define WORKINGSET_FILE 1#define ANON_AND_FILE 2
enum lruvec_flags { /* * An lruvec has many dirty pages backed by a congested BDI: * 1. LRUVEC_CGROUP_CONGESTED is set by cgroup-level reclaim. * It can be cleared by cgroup reclaim or kswapd. * 2. LRUVEC_NODE_CONGESTED is set by kswapd node-level reclaim. * It can only be cleared by kswapd. * * Essentially, kswapd can unthrottle an lruvec throttled by cgroup * reclaim, but not vice versa. This only applies to the root cgroup. * The goal is to prevent cgroup reclaim on the root cgroup (e.g. * memory.reclaim) to unthrottle an unbalanced node (that was throttled * by kswapd). */ LRUVEC_CGROUP_CONGESTED, LRUVEC_NODE_CONGESTED,};
#endif /* !__GENERATING_BOUNDS_H */
/* * Evictable folios are divided into multiple generations. The youngest and the * oldest generation numbers, max_seq and min_seq, are monotonically increasing. * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the * corresponding generation. The gen counter in folio->flags stores gen+1 while * a folio is on one of lrugen->folios[]. Otherwise it stores 0. * * After a folio is faulted in, the aging needs to check the accessed bit at * least twice before handing this folio over to the eviction. The first check * clears the accessed bit from the initial fault; the second check makes sure * this folio hasn't been used since then. This process, AKA second chance, * requires a minimum of two generations, hence MIN_NR_GENS. And to maintain ABI * compatibility with the active/inactive LRU, e.g., /proc/vmstat, these two * generations are considered active; the rest of generations, if they exist, * are considered inactive. See lru_gen_is_active(). * * PG_active is always cleared while a folio is on one of lrugen->folios[] so * that the sliding window needs not to worry about it. And it's set again when * a folio considered active is isolated for non-reclaiming purposes, e.g., * migration. See lru_gen_add_folio() and lru_gen_del_folio(). * * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the * number of categories of the active/inactive LRU when keeping track of * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits * in folio->flags, masked by LRU_GEN_MASK. */#define MIN_NR_GENS 2U#define MAX_NR_GENS 4U
/* * Each generation is divided into multiple tiers. A folio accessed N times * through file descriptors is in tier order_base_2(N). A folio in the first * tier (N=0,1) is marked by PG_referenced unless it was faulted in through page * tables or read ahead. A folio in the last tier (MAX_NR_TIERS-1) is marked by * PG_workingset. A folio in any other tier (1<N<5) between the first and last * is marked by additional bits of LRU_REFS_WIDTH in folio->flags. * * In contrast to moving across generations which requires the LRU lock, moving * across tiers only involves atomic operations on folio->flags and therefore * has a negligible cost in the buffered access path. In the eviction path, * comparisons of refaulted/(evicted+protected) from the first tier and the rest * infer whether folios accessed multiple times through file descriptors are * statistically hot and thus worth protecting. * * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the * number of categories of the active/inactive LRU when keeping track of * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in * folio->flags, masked by LRU_REFS_MASK. */#define MAX_NR_TIERS 4U
#ifndef __GENERATING_BOUNDS_H
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
/* * For folios accessed multiple times through file descriptors, * lru_gen_inc_refs() sets additional bits of LRU_REFS_WIDTH in folio->flags * after PG_referenced, then PG_workingset after LRU_REFS_WIDTH. After all its * bits are set, i.e., LRU_REFS_FLAGS|BIT(PG_workingset), a folio is lazily * promoted into the second oldest generation in the eviction path. And when * folio_inc_gen() does that, it clears LRU_REFS_FLAGS so that * lru_gen_inc_refs() can start over. Note that for this case, LRU_REFS_MASK is * only valid when PG_referenced is set. * * For folios accessed multiple times through page tables, folio_update_gen() * from a page table walk or lru_gen_set_refs() from a rmap walk sets * PG_referenced after the accessed bit is cleared for the first time. * Thereafter, those two paths set PG_workingset and promote folios to the * youngest generation. Like folio_inc_gen(), folio_update_gen() also clears * PG_referenced. Note that for this case, LRU_REFS_MASK is not used. * * For both cases above, after PG_workingset is set on a folio, it remains until * this folio is either reclaimed, or "deactivated" by lru_gen_clear_refs(). It * can be set again if lru_gen_test_recent() returns true upon a refault. */#define LRU_REFS_FLAGS (LRU_REFS_MASK | BIT(PG_referenced))
struct lruvec;struct page_vma_mapped_walk;
#ifdef CONFIG_LRU_GEN
enum { LRU_GEN_ANON, LRU_GEN_FILE,};
enum { LRU_GEN_CORE, LRU_GEN_MM_WALK, LRU_GEN_NONLEAF_YOUNG, NR_LRU_GEN_CAPS};
#define MIN_LRU_BATCH BITS_PER_LONG#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64)
/* whether to keep historical stats from evicted generations */#ifdef CONFIG_LRU_GEN_STATS#define NR_HIST_GENS MAX_NR_GENS#else#define NR_HIST_GENS 1U#endif
/* * The youngest generation number is stored in max_seq for both anon and file * types as they are aged on an equal footing. The oldest generation numbers are * stored in min_seq[] separately for anon and file types so that they can be * incremented independently. Ideally min_seq[] are kept in sync when both anon * and file types are evictable. However, to adapt to situations like extreme * swappiness, they are allowed to be out of sync by at most * MAX_NR_GENS-MIN_NR_GENS-1. * * The number of pages in each generation is eventually consistent and therefore * can be transiently negative when reset_batch_size() is pending. */struct lru_gen_folio { /* the aging increments the youngest generation number */ unsigned long max_seq; /* the eviction increments the oldest generation numbers */ unsigned long min_seq[ANON_AND_FILE]; /* the birth time of each generation in jiffies */ unsigned long timestamps[MAX_NR_GENS]; /* the multi-gen LRU lists, lazily sorted on eviction */ struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the multi-gen LRU sizes, eventually consistent */ long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the exponential moving average of refaulted */ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; /* the exponential moving average of evicted+protected */ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; /* can only be modified under the LRU lock */ unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; /* can be modified without holding the LRU lock */ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; /* whether the multi-gen LRU is enabled */ bool enabled; /* the memcg generation this lru_gen_folio belongs to */ u8 gen; /* the list segment this lru_gen_folio belongs to */ u8 seg; /* per-node lru_gen_folio list for global reclaim */ struct hlist_nulls_node list;};
enum { MM_LEAF_TOTAL, /* total leaf entries */ MM_LEAF_YOUNG, /* young leaf entries */ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ NR_MM_STATS};
/* double-buffering Bloom filters */#define NR_BLOOM_FILTERS 2
struct lru_gen_mm_state { /* synced with max_seq after each iteration */ unsigned long seq; /* where the current iteration continues after */ struct list_head *head; /* where the last iteration ended before */ struct list_head *tail; /* Bloom filters flip after each iteration */ unsigned long *filters[NR_BLOOM_FILTERS]; /* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS][NR_MM_STATS];};
struct lru_gen_mm_walk { /* the lruvec under reclaim */ struct lruvec *lruvec; /* max_seq from lru_gen_folio: can be out of date */ unsigned long seq; /* the next address within an mm to scan */ unsigned long next_addr; /* to batch promoted pages */ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* to batch the mm stats */ int mm_stats[NR_MM_STATS]; /* total batched items */ int batched; int swappiness; bool force_scan;};
/* * For each node, memcgs are divided into two generations: the old and the * young. For each generation, memcgs are randomly sharded into multiple bins * to improve scalability. For each bin, the hlist_nulls is virtually divided * into three segments: the head, the tail and the default. * * An onlining memcg is added to the tail of a random bin in the old generation. * The eviction starts at the head of a random bin in the old generation. The * per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes * the old generation, is incremented when all its bins become empty. * * There are four operations: * 1. MEMCG_LRU_HEAD, which moves a memcg to the head of a random bin in its * current generation (old or young) and updates its "seg" to "head"; * 2. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its * current generation (old or young) and updates its "seg" to "tail"; * 3. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old * generation, updates its "gen" to "old" and resets its "seg" to "default"; * 4. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the * young generation, updates its "gen" to "young" and resets its "seg" to * "default". * * The events that trigger the above operations are: * 1. Exceeding the soft limit, which triggers MEMCG_LRU_HEAD; * 2. The first attempt to reclaim a memcg below low, which triggers * MEMCG_LRU_TAIL; * 3. The first attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_TAIL; * 4. The second attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_YOUNG; * 5. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG; * 6. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG; * 7. Offlining a memcg, which triggers MEMCG_LRU_OLD. * * Notes: * 1. Memcg LRU only applies to global reclaim, and the round-robin incrementing * of their max_seq counters ensures the eventual fairness to all eligible * memcgs. For memcg reclaim, it still relies on mem_cgroup_iter(). * 2. There are only two valid generations: old (seq) and young (seq+1). * MEMCG_NR_GENS is set to three so that when reading the generation counter * locklessly, a stale value (seq-1) does not wraparound to young. */#define MEMCG_NR_GENS 3#define MEMCG_NR_BINS 8
struct lru_gen_memcg { /* the per-node memcg generation counter */ unsigned long seq; /* each memcg has one lru_gen_folio per node */ unsigned long nr_memcgs[MEMCG_NR_GENS]; /* per-node lru_gen_folio list for global reclaim */ struct hlist_nulls_head fifo[MEMCG_NR_GENS][MEMCG_NR_BINS]; /* protects the above */ spinlock_t lock;};
void lru_gen_init_pgdat(struct pglist_data *pgdat);void lru_gen_init_lruvec(struct lruvec *lruvec);bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr);
void lru_gen_init_memcg(struct mem_cgroup *memcg);void lru_gen_exit_memcg(struct mem_cgroup *memcg);void lru_gen_online_memcg(struct mem_cgroup *memcg);void lru_gen_offline_memcg(struct mem_cgroup *memcg);void lru_gen_release_memcg(struct mem_cgroup *memcg);void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid);void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid);bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid);void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
#else /* !CONFIG_LRU_GEN */
static inline void lru_gen_init_pgdat(struct pglist_data *pgdat){}
static inline void lru_gen_init_lruvec(struct lruvec *lruvec){}
static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr){ return false;}
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_online_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_offline_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_release_memcg(struct mem_cgroup *memcg){}
static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid){}
static inline void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid){}
static inline bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid){ return true;}
static inlinevoid lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid){}
#endif /* CONFIG_LRU_GEN */
struct lru_cost { unsigned long count; unsigned long last_rotated; unsigned long last_io;};
struct lruvec { struct list_head lists[NR_LRU_LISTS]; /* per lruvec lru_lock for memcg */ spinlock_t lru_lock; /* * These track the cost of reclaiming one LRU - file or anon - * over the other. As the observed cost of reclaiming one LRU * increases, the reclaim scan balance tips toward the other. * Updated and decayed at prepare_scan_control() time; cost_lock * serialises that update. */ struct lru_cost cost[ANON_AND_FILE]; /* Protects cost[]. */ spinlock_t cost_lock; /* Non-resident age, driven by LRU movement */ atomic_long_t nonresident_age; /* Refaults at the time of last reclaim cycle */ unsigned long refaults[ANON_AND_FILE]; /* Various lruvec state flags (enum lruvec_flags) */ unsigned long flags;#ifdef CONFIG_LRU_GEN /* evictable pages divided into generations */ struct lru_gen_folio lrugen;#ifdef CONFIG_LRU_GEN_WALKS_MMU /* to concurrently iterate lru_gen_mm_list */ struct lru_gen_mm_state mm_state;#endif#endif /* CONFIG_LRU_GEN */#ifdef CONFIG_MEMCG struct pglist_data *pgdat;#endif struct zswap_lruvec_state zswap_lruvec_state;};
/* Isolate for asynchronous migration */#define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)/* Isolate unevictable pages */#define ISOLATE_UNEVICTABLE ((__force isolate_mode_t)0x8)
/* LRU Isolation modes. */typedef unsigned __bitwise isolate_mode_t;
enum zone_watermarks { WMARK_MIN, WMARK_LOW, WMARK_HIGH, WMARK_PROMO, NR_WMARK};
/* * One per migratetype for each PAGE_ALLOC_COSTLY_ORDER. Two additional lists * are added for THP. One PCP list is used by GPF_MOVABLE, and the other PCP list * is used by GFP_UNMOVABLE and GFP_RECLAIMABLE. */#ifdef CONFIG_TRANSPARENT_HUGEPAGE#define NR_PCP_THP 2#else#define NR_PCP_THP 0#endif#define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1))#define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP)
/* * Flags used in pcp->flags field. * * PCPF_PREV_FREE_HIGH_ORDER: a high-order page is freed in the * previous page freeing. To avoid to drain PCP for an accident * high-order page freeing. * * PCPF_FREE_HIGH_BATCH: preserve "pcp->batch" pages in PCP before * draining PCP for consecutive high-order pages freeing without * allocation if data cache slice of CPU is large enough. To reduce * zone lock contention and keep cache-hot pages reusing. */#define PCPF_PREV_FREE_HIGH_ORDER BIT(0)#define PCPF_FREE_HIGH_BATCH BIT(1)
struct per_cpu_pages { spinlock_t lock; /* Protects lists field */ int count; /* number of pages in the list */ int high; /* high watermark, emptying needed */ int high_min; /* min high watermark */ int high_max; /* max high watermark */ int batch; /* chunk size for buddy add/remove */ u8 flags; /* protected by pcp->lock */ u8 alloc_factor; /* batch scaling factor during allocate */#ifdef CONFIG_NUMA u8 expire; /* When 0, remote pagesets are drained */#endif short free_count; /* consecutive free count */
/* Lists of pages, one per migrate type stored on the pcp-lists */ struct list_head lists[NR_PCP_LISTS];} ____cacheline_aligned_in_smp;
struct per_cpu_zonestat {#ifdef CONFIG_SMP s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; s8 stat_threshold;#endif#ifdef CONFIG_NUMA /* * Low priority inaccurate counters that are only folded * on demand. Use a large type to avoid the overhead of * folding during refresh_cpu_vm_stats. */ unsigned long vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];#endif};
struct per_cpu_nodestat { s8 stat_threshold; s8 vm_node_stat_diff[NR_VM_NODE_STAT_ITEMS];};
#endif /* !__GENERATING_BOUNDS.H */
enum zone_type { /* * ZONE_DMA and ZONE_DMA32 are used when there are peripherals not able * to DMA to all of the addressable memory (ZONE_NORMAL). * On architectures where this area covers the whole 32 bit address * space ZONE_DMA32 is used. ZONE_DMA is left for the ones with smaller * DMA addressing constraints. This distinction is important as a 32bit * DMA mask is assumed when ZONE_DMA32 is defined. Some 64-bit * platforms may need both zones as they support peripherals with * different DMA addressing limitations. */#ifdef CONFIG_ZONE_DMA ZONE_DMA,#endif#ifdef CONFIG_ZONE_DMA32 ZONE_DMA32,#endif /* * Normal addressable memory is in ZONE_NORMAL. DMA operations can be * performed on pages in ZONE_NORMAL if the DMA devices support * transfers to all addressable memory. */ ZONE_NORMAL,#ifdef CONFIG_HIGHMEM /* * A memory area that is only addressable by the kernel through * mapping portions into its own address space. This is for example * used by i386 to allow the kernel to address the memory beyond * 900MB. The kernel will set up special mappings (page * table entries on i386) for each page that the kernel needs to * access. */ ZONE_HIGHMEM,#endif /* * ZONE_MOVABLE is similar to ZONE_NORMAL, except that it contains * movable pages with few exceptional cases described below. Main use * cases for ZONE_MOVABLE are to make memory offlining/unplug more * likely to succeed, and to locally limit unmovable allocations - e.g., * to increase the number of THP/huge pages. Notable special cases are: * * 1. Pinned pages: (long-term) pinning of movable pages might * essentially turn such pages unmovable. Therefore, we do not allow * pinning long-term pages in ZONE_MOVABLE. When pages are pinned and * faulted, they come from the right zone right away. However, it is * still possible that address space already has pages in * ZONE_MOVABLE at the time when pages are pinned (i.e. user has * touches that memory before pinning). In such case we migrate them * to a different zone. When migration fails - pinning fails. * 2. memblock allocations: kernelcore/movablecore setups might create * situations where ZONE_MOVABLE contains unmovable allocations * after boot. Memory offlining and allocations fail early. * 3. Memory holes: kernelcore/movablecore setups might create very rare * situations where ZONE_MOVABLE contains memory holes after boot, * for example, if we have sections that are only partially * populated. Memory offlining and allocations fail early. * 4. PG_hwpoison pages: while poisoned pages can be skipped during * memory offlining, such pages cannot be allocated. * 5. Unmovable PG_offline pages: in paravirtualized environments, * hotplugged memory blocks might only partially be managed by the * buddy (e.g., via XEN-balloon, Hyper-V balloon, virtio-mem). The * parts not manged by the buddy are unmovable PG_offline pages. In * some cases (virtio-mem), such pages can be skipped during * memory offlining, however, cannot be moved/allocated. These * techniques might use alloc_contig_range() to hide previously * exposed pages from the buddy again (e.g., to implement some sort * of memory unplug in virtio-mem). * 6. ZERO_PAGE(0), kernelcore/movablecore setups might create * situations where ZERO_PAGE(0) which is allocated differently * on different platforms may end up in a movable zone. ZERO_PAGE(0) * cannot be migrated. * 7. Memory-hotplug: when using memmap_on_memory and onlining the * memory to the MOVABLE zone, the vmemmap pages are also placed in * such zone. Such pages cannot be really moved around as they are * self-stored in the range, but they are treated as movable when * the range they describe is about to be offlined. * * In general, no unmovable allocations that degrade memory offlining * should end up in ZONE_MOVABLE. Allocators (like alloc_contig_range()) * have to expect that migrating pages in ZONE_MOVABLE can fail (even * if has_unmovable_pages() states that there are no unmovable pages, * there can be false negatives). */ ZONE_MOVABLE,#ifdef CONFIG_ZONE_DEVICE ZONE_DEVICE,#endif __MAX_NR_ZONES
};
#ifndef __GENERATING_BOUNDS_H
#define ASYNC_AND_SYNC 2
struct zone { /* Read-mostly fields */
/* zone watermarks, access with *_wmark_pages(zone) macros */ unsigned long _watermark[NR_WMARK]; unsigned long watermark_boost;
unsigned long nr_reserved_highatomic; unsigned long nr_free_highatomic;
/* * We don't know if the memory that we're going to allocate will be * freeable or/and it will be released eventually, so to avoid totally * wasting several GB of ram we must reserve some of the lower zone * memory (otherwise we risk to run OOM on the lower zones despite * there being tons of freeable ram on the higher zones). This array is * recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl * changes. */ long lowmem_reserve[MAX_NR_ZONES];
#ifdef CONFIG_NUMA int node;#endif struct pglist_data *zone_pgdat; struct per_cpu_pages __percpu *per_cpu_pageset; struct per_cpu_zonestat __percpu *per_cpu_zonestats; /* * the high and batch values are copied to individual pagesets for * faster access */ int pageset_high_min; int pageset_high_max; int pageset_batch;
#ifndef CONFIG_SPARSEMEM /* * Flags for a pageblock_nr_pages block. See pageblock-flags.h. * In SPARSEMEM, this map is stored in struct mem_section */ unsigned long *pageblock_flags;#endif /* CONFIG_SPARSEMEM */
/* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */ unsigned long zone_start_pfn;
/* * spanned_pages is the total pages spanned by the zone, including * holes, which is calculated as: * spanned_pages = zone_end_pfn - zone_start_pfn; * * present_pages is physical pages existing within the zone, which * is calculated as: * present_pages = spanned_pages - absent_pages(pages in holes); * * present_early_pages is present pages existing within the zone * located on memory available since early boot, excluding hotplugged * memory. * * managed_pages is present pages managed by the buddy system, which * is calculated as (reserved_pages includes pages allocated by the * bootmem allocator): * managed_pages = present_pages - reserved_pages; * * cma pages is present pages that are assigned for CMA use * (MIGRATE_CMA). * * So present_pages may be used by memory hotplug or memory power * management logic to figure out unmanaged pages by checking * (present_pages - managed_pages). And managed_pages should be used * by page allocator and vm scanner to calculate all kinds of watermarks * and thresholds. * * Locking rules: * * zone_start_pfn and spanned_pages are protected by span_seqlock. * It is a seqlock because it has to be read outside of zone->lock, * and it is done in the main allocator path. But, it is written * quite infrequently. * * The span_seq lock is declared along with zone->lock because it is * frequently read in proximity to zone->lock. It's good to * give them a chance of being in the same cacheline. * * Write access to present_pages at runtime should be protected by * mem_hotplug_begin/done(). Any reader who can't tolerant drift of * present_pages should use get_online_mems() to get a stable value. */ atomic_long_t managed_pages; unsigned long spanned_pages; unsigned long present_pages;#if defined(CONFIG_MEMORY_HOTPLUG) unsigned long present_early_pages;#endif#ifdef CONFIG_CMA unsigned long cma_pages;#endif
const char *name;
#ifdef CONFIG_MEMORY_ISOLATION /* * Number of isolated pageblock. It is used to solve incorrect * freepage counting problem due to racy retrieving migratetype * of pageblock. Protected by zone->lock. */ unsigned long nr_isolate_pageblock;#endif
#ifdef CONFIG_MEMORY_HOTPLUG /* see spanned/present_pages for more description */ seqlock_t span_seqlock;#endif
int initialized;
/* Write-intensive fields used from the page allocator */ CACHELINE_PADDING(_pad1_);
/* free areas of different sizes */ struct free_area free_area[NR_PAGE_ORDERS];
#ifdef CONFIG_UNACCEPTED_MEMORY /* Pages to be accepted. All pages on the list are MAX_PAGE_ORDER */ struct list_head unaccepted_pages;
/* To be called once the last page in the zone is accepted */ struct work_struct unaccepted_cleanup;#endif
/* zone flags, see below */ unsigned long flags;
/* Primarily protects free_area */ spinlock_t lock;
/* Pages to be freed when next trylock succeeds */ struct llist_head trylock_free_pages;
/* Write-intensive fields used by compaction and vmstats. */ CACHELINE_PADDING(_pad2_);
/* * When free pages are below this point, additional steps are taken * when reading the number of free pages to avoid per-cpu counter * drift allowing watermarks to be breached */ unsigned long percpu_drift_mark;
#if defined CONFIG_COMPACTION || defined CONFIG_CMA /* pfn where compaction free scanner should start */ unsigned long compact_cached_free_pfn; /* pfn where compaction migration scanner should start */ unsigned long compact_cached_migrate_pfn[ASYNC_AND_SYNC]; unsigned long compact_init_migrate_pfn; unsigned long compact_init_free_pfn;#endif
#ifdef CONFIG_COMPACTION /* * On compaction failure, 1<<compact_defer_shift compactions * are skipped before trying again. The number attempted since * last failure is tracked with compact_considered. * compact_order_failed is the minimum compaction failed order. */ unsigned int compact_considered; unsigned int compact_defer_shift; int compact_order_failed;#endif
#if defined CONFIG_COMPACTION || defined CONFIG_CMA /* Set to true when the PG_migrate_skip bits should be cleared */ bool compact_blockskip_flush;#endif
bool contiguous;
CACHELINE_PADDING(_pad3_); /* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP struct page *vmemmap_tails[NR_VMEMMAP_TAILS];#endif} ____cacheline_internodealigned_in_smp;
enum pgdat_flags { PGDAT_WRITEBACK, /* reclaim scanning has recently found * many pages under writeback */ PGDAT_RECLAIM_LOCKED, /* prevents concurrent reclaim */};
enum zone_flags { ZONE_BOOSTED_WATERMARK, /* zone recently boosted watermarks. * Cleared when kswapd is woken. */ ZONE_RECLAIM_ACTIVE, /* kswapd may be scanning the zone. */ ZONE_BELOW_HIGH, /* zone is below high watermark. */};
static inline unsigned long wmark_pages(const struct zone *z, enum zone_watermarks w){ return z->_watermark[w] + z->watermark_boost;}
static inline unsigned long min_wmark_pages(const struct zone *z){ return wmark_pages(z, WMARK_MIN);}
static inline unsigned long low_wmark_pages(const struct zone *z){ return wmark_pages(z, WMARK_LOW);}
static inline unsigned long high_wmark_pages(const struct zone *z){ return wmark_pages(z, WMARK_HIGH);}
static inline unsigned long promo_wmark_pages(const struct zone *z){ return wmark_pages(z, WMARK_PROMO);}
static inline unsigned long zone_managed_pages(const struct zone *zone){ return (unsigned long)atomic_long_read(&zone->managed_pages);}
static inline unsigned long zone_cma_pages(struct zone *zone){#ifdef CONFIG_CMA return zone->cma_pages;#else return 0;#endif}
static inline unsigned long zone_end_pfn(const struct zone *zone){ return zone->zone_start_pfn + zone->spanned_pages;}
static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn){ return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);}
static inline bool zone_is_initialized(const struct zone *zone){ return zone->initialized;}
static inline bool zone_is_empty(const struct zone *zone){ return zone->spanned_pages == 0;}
#ifndef BUILD_VDSO32_64/* * The zone field is never updated after free_area_init_core() * sets it, so none of the operations on it need to be atomic. */
/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)#define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH)#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH)
/* * Define the bit shifts to access each section. For non-existent * sections we define the shift as 0; that plus a 0 mask ensures * the compiler will optimise away reference to them. */#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))#define KASAN_TAG_PGSHIFT (KASAN_TAG_PGOFF * (KASAN_TAG_WIDTH != 0))
/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */#ifdef NODE_NOT_IN_PAGE_FLAGS#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF) ? \ SECTIONS_PGOFF : ZONES_PGOFF)#else#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF) ? \ NODES_PGOFF : ZONES_PGOFF)#endif
#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)#define NODES_MASK ((1UL << NODES_WIDTH) - 1)#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)#define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1)#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
static inline enum zone_type memdesc_zonenum(const memdesc_flags_t *flags){#if ZONES_WIDTH != 0 ASSERT_EXCLUSIVE_BITS(flags->f, ZONES_MASK << ZONES_PGSHIFT);#endif return (flags->f >> ZONES_PGSHIFT) & ZONES_MASK;}
static inline enum zone_type page_zonenum(const struct page *page){ return memdesc_zonenum(&page->flags);}
static inline enum zone_type folio_zonenum(const struct folio *folio){ return memdesc_zonenum(&folio->flags);}
#ifdef CONFIG_ZONE_DEVICEstatic inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf){ return memdesc_zonenum(mdf) == ZONE_DEVICE;}
static inline struct dev_pagemap *page_pgmap(const struct page *page){ VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(&page->flags), page); return page_folio(page)->pgmap;}
/* * Consecutive zone device pages should not be merged into the same sgl * or bvec segment with other types of pages or if they belong to different * pgmaps. Otherwise getting the pgmap of a given segment is not possible * without scanning the entire segment. This helper returns true either if * both pages are not zone device pages or both pages are zone device pages * with the same pgmap. */static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b){ if (memdesc_is_zone_device(&a->flags) != memdesc_is_zone_device(&b->flags)) return false; if (!memdesc_is_zone_device(&a->flags)) return true; return page_pgmap(a) == page_pgmap(b);}
extern void memmap_init_zone_device(struct zone *, unsigned long, unsigned long, struct dev_pagemap *);#elsestatic inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf){ return false;}static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b){ return true;}static inline struct dev_pagemap *page_pgmap(const struct page *page){ return NULL;}#endif
static inline bool is_zone_device_page(const struct page *page){ return memdesc_is_zone_device(&page->flags);}
static inline bool folio_is_zone_device(const struct folio *folio){ return memdesc_is_zone_device(&folio->flags);}
static inline bool is_zone_movable_page(const struct page *page){ return page_zonenum(page) == ZONE_MOVABLE;}
static inline bool folio_is_zone_movable(const struct folio *folio){ return folio_zonenum(folio) == ZONE_MOVABLE;}#endif
/* * Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty * intersection with the given zone */static inline bool zone_intersects(const struct zone *zone, unsigned long start_pfn, unsigned long nr_pages){ if (zone_is_empty(zone)) return false; if (start_pfn >= zone_end_pfn(zone) || start_pfn + nr_pages <= zone->zone_start_pfn) return false;
return true;}
/* * The "priority" of VM scanning is how much of the queues we will scan in one * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the * queues ("queue_length >> 12") during an aging round. */#define DEF_PRIORITY 12
/* Maximum number of zones on a zonelist */#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
enum { ZONELIST_FALLBACK, /* zonelist with fallback */#ifdef CONFIG_NUMA /* * The NUMA zonelists are doubled because we need zonelists that * restrict the allocations to a single node for __GFP_THISNODE. */ ZONELIST_NOFALLBACK, /* zonelist without fallback (__GFP_THISNODE) */#endif MAX_ZONELISTS};
/* * This struct contains information about a zone in a zonelist. It is stored * here to avoid dereferences into large structures and lookups of tables */struct zoneref { struct zone *zone; /* Pointer to actual zone */ int zone_idx; /* zone_idx(zoneref->zone) */};
/* * One allocation request operates on a zonelist. A zonelist * is a list of zones, the first one is the 'goal' of the * allocation, the other zones are fallback zones, in decreasing * priority. * * To speed the reading of the zonelist, the zonerefs contain the zone index * of the entry being read. Helper functions to access information given * a struct zoneref are * * zonelist_zone() - Return the struct zone * for an entry in _zonerefs * zonelist_zone_idx() - Return the index of the zone for an entry * zonelist_node_idx() - Return the index of the node for an entry */struct zonelist { struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];};
/* * The array of struct pages for flatmem. * It must be declared for SPARSEMEM as well because there are configurations * that rely on that. */extern struct page *mem_map;
#ifdef CONFIG_MEMORY_FAILURE/* * Per NUMA node memory failure handling statistics. */struct memory_failure_stats { /* * Number of raw pages poisoned. * Cases not accounted: memory outside kernel control, offline page, * arch-specific memory_failure (SGX), hwpoison_filter() filtered * error events, and unpoison actions from hwpoison_unpoison. */ unsigned long total; /* * Recovery results of poisoned raw pages handled by memory_failure, * in sync with mf_result. * total = ignored + failed + delayed + recovered. * total * PAGE_SIZE * #nodes = /proc/meminfo/HardwareCorrupted. */ unsigned long ignored; unsigned long failed; unsigned long delayed; unsigned long recovered;};#endif
/* * On NUMA machines, each NUMA node would have a pg_data_t to describe * it's memory layout. On UMA machines there is a single pglist_data which * describes the whole memory. * * Memory statistics and page replacement data structures are maintained on a * per-zone basis. */typedef struct pglist_data { /* * node_zones contains just the zones for THIS node. Not all of the * zones may be populated, but it is the full list. It is referenced by * this node's node_zonelists as well as other node's node_zonelists. */ struct zone node_zones[MAX_NR_ZONES];
/* * node_zonelists contains references to all zones in all nodes. * Generally the first zones will be references to this node's * node_zones. */ struct zonelist node_zonelists[MAX_ZONELISTS];
int nr_zones; /* number of populated zones in this node */#ifdef CONFIG_FLATMEM /* means !SPARSEMEM */ struct page *node_mem_map;#ifdef CONFIG_PAGE_EXTENSION struct page_ext *node_page_ext;#endif#endif#if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) /* * Must be held any time you expect node_start_pfn, * node_present_pages, node_spanned_pages or nr_zones to stay constant. * Also synchronizes pgdat->first_deferred_pfn during deferred page * init. * * pgdat_resize_lock() and pgdat_resize_unlock() are provided to * manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG * or CONFIG_DEFERRED_STRUCT_PAGE_INIT. * * Nests above zone->lock and zone->span_seqlock */ spinlock_t node_size_lock;#endif unsigned long node_start_pfn; unsigned long node_present_pages; /* total number of physical pages */ unsigned long node_spanned_pages; /* total size of physical page range, including holes */ int node_id; wait_queue_head_t kswapd_wait; wait_queue_head_t pfmemalloc_wait;
/* workqueues for throttling reclaim for different reasons. */ wait_queue_head_t reclaim_wait[NR_VMSCAN_THROTTLE];
atomic_t nr_writeback_throttled;/* nr of writeback-throttled tasks */ unsigned long nr_reclaim_start; /* nr pages written while throttled * when throttling started. */#ifdef CONFIG_MEMORY_HOTPLUG struct mutex kswapd_lock;#endif struct task_struct *kswapd; /* Protected by kswapd_lock */ int kswapd_order; enum zone_type kswapd_highest_zoneidx;
atomic_t kswapd_failures; /* Number of 'reclaimed == 0' runs */
#ifdef CONFIG_COMPACTION int kcompactd_max_order; enum zone_type kcompactd_highest_zoneidx; wait_queue_head_t kcompactd_wait; struct task_struct *kcompactd; bool proactive_compact_trigger;#endif /* * This is a per-node reserve of pages that are not available * to userspace allocations. */ unsigned long totalreserve_pages;
#ifdef CONFIG_NUMA /* * node reclaim becomes active if more unmapped pages exist. */ unsigned long min_unmapped_pages; unsigned long min_slab_pages;#endif /* CONFIG_NUMA */
/* Write-intensive fields used by page reclaim */ CACHELINE_PADDING(_pad1_);
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT /* * If memory initialisation on large machines is deferred then this * is the first PFN that needs to be initialised. */ unsigned long first_deferred_pfn;#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
#ifdef CONFIG_NUMA_BALANCING /* start time in ms of current promote rate limit period */ unsigned int nbp_rl_start; /* number of promote candidate pages at start time of current rate limit period */ unsigned long nbp_rl_nr_cand; /* promote threshold in ms */ unsigned int nbp_threshold; /* start time in ms of current promote threshold adjustment period */ unsigned int nbp_th_start; /* * number of promote candidate pages at start time of current promote * threshold adjustment period */ unsigned long nbp_th_nr_cand;#endif /* Fields commonly accessed by the page reclaim scanner */
/* * NOTE: THIS IS UNUSED IF MEMCG IS ENABLED. * * Use mem_cgroup_lruvec() to look up lruvecs. */ struct lruvec __lruvec;
unsigned long flags;
#ifdef CONFIG_LRU_GEN /* kswap mm walk data */ struct lru_gen_mm_walk mm_walk; /* lru_gen_folio list */ struct lru_gen_memcg memcg_lru;#endif
CACHELINE_PADDING(_pad2_);
/* Per-node vmstats */ struct per_cpu_nodestat __percpu *per_cpu_nodestats; atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];#ifdef CONFIG_NUMA struct memory_tier __rcu *memtier;#endif#ifdef CONFIG_MEMORY_FAILURE struct memory_failure_stats mf_stats;#endif} pg_data_t;
#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)#define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat){ return pgdat->node_start_pfn + pgdat->node_spanned_pages;}
#include <linux/memory_hotplug.h>
void build_all_zonelists(pg_data_t *pgdat);bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, int highest_zoneidx, unsigned int alloc_flags, long free_pages);bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, int highest_zoneidx, unsigned int alloc_flags);
enum kswapd_clear_hopeless_reason { KSWAPD_CLEAR_HOPELESS_OTHER = 0, KSWAPD_CLEAR_HOPELESS_KSWAPD, KSWAPD_CLEAR_HOPELESS_DIRECT, KSWAPD_CLEAR_HOPELESS_PCP,};
void wakeup_kswapd(struct zone *zone, gfp_t gfp_mask, int order, enum zone_type highest_zoneidx);void kswapd_try_clear_hopeless(struct pglist_data *pgdat, unsigned int order, int highest_zoneidx);void kswapd_clear_hopeless(pg_data_t *pgdat, enum kswapd_clear_hopeless_reason reason);bool kswapd_test_hopeless(pg_data_t *pgdat);
/* * Memory initialization context, use to differentiate memory added by * the platform statically or via memory hotplug interface. */enum meminit_context { MEMINIT_EARLY, MEMINIT_HOTPLUG,};
extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn, unsigned long size);
extern void lruvec_init(struct lruvec *lruvec);
static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec){#ifdef CONFIG_MEMCG return lruvec->pgdat;#else return container_of(lruvec, struct pglist_data, __lruvec);#endif}
#ifdef CONFIG_HAVE_MEMORYLESS_NODESint local_memory_node(int node_id);#elsestatic inline int local_memory_node(int node_id) { return node_id; };#endif
/* * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc. */#define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones)
#ifdef CONFIG_ZONE_DEVICEstatic inline bool zone_is_zone_device(const struct zone *zone){ return zone_idx(zone) == ZONE_DEVICE;}#elsestatic inline bool zone_is_zone_device(const struct zone *zone){ return false;}#endif
/* * Returns true if a zone has pages managed by the buddy allocator. * All the reclaim decisions have to use this function rather than * populated_zone(). If the whole zone is reserved then we can easily * end up with populated_zone() && !managed_zone(). */static inline bool managed_zone(const struct zone *zone){ return zone_managed_pages(zone);}
/* Returns true if a zone has memory */static inline bool populated_zone(const struct zone *zone){ return zone->present_pages;}
#ifdef CONFIG_NUMAstatic inline int zone_to_nid(const struct zone *zone){ return zone->node;}
static inline void zone_set_nid(struct zone *zone, int nid){ zone->node = nid;}#elsestatic inline int zone_to_nid(const struct zone *zone){ return 0;}
static inline void zone_set_nid(struct zone *zone, int nid) {}#endif
extern int movable_zone;
static inline int is_highmem_idx(enum zone_type idx){#ifdef CONFIG_HIGHMEM return (idx == ZONE_HIGHMEM || (idx == ZONE_MOVABLE && movable_zone == ZONE_HIGHMEM));#else return 0;#endif}
/** * is_highmem - helper function to quickly check if a struct zone is a * highmem zone or not. This is an attempt to keep references * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum. * @zone: pointer to struct zone variable * Return: 1 for a highmem zone, 0 otherwise */static inline int is_highmem(const struct zone *zone){ return is_highmem_idx(zone_idx(zone));}
bool has_managed_zone(enum zone_type zone);static inline bool has_managed_dma(void){#ifdef CONFIG_ZONE_DMA return has_managed_zone(ZONE_DMA);#else return false;#endif}
#ifndef CONFIG_NUMA
extern struct pglist_data contig_page_data;static inline struct pglist_data *NODE_DATA(int nid){ return &contig_page_data;}
#else /* CONFIG_NUMA */
#include <asm/mmzone.h>
#endif /* !CONFIG_NUMA */
extern struct pglist_data *first_online_pgdat(void);extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);extern struct zone *next_zone(struct zone *zone);
/** * for_each_online_pgdat - helper macro to iterate over all online nodes * @pgdat: pointer to a pg_data_t variable */#define for_each_online_pgdat(pgdat) \ for (pgdat = first_online_pgdat(); \ pgdat; \ pgdat = next_online_pgdat(pgdat))/** * for_each_zone - helper macro to iterate over all memory zones * @zone: pointer to struct zone variable * * The user only needs to declare the zone variable, for_each_zone * fills it in. */#define for_each_zone(zone) \ for (zone = (first_online_pgdat())->node_zones; \ zone; \ zone = next_zone(zone))
#define for_each_populated_zone(zone) \ for (zone = (first_online_pgdat())->node_zones; \ zone; \ zone = next_zone(zone)) \ if (!populated_zone(zone)) \ ; /* do nothing */ \ else
static inline struct zone *zonelist_zone(struct zoneref *zoneref){ return zoneref->zone;}
static inline int zonelist_zone_idx(const struct zoneref *zoneref){ return zoneref->zone_idx;}
static inline int zonelist_node_idx(const struct zoneref *zoneref){ return zone_to_nid(zoneref->zone);}
struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, const nodemask_t *nodes);
/** * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point * @z: The cursor used as a starting point for the search * @highest_zoneidx: The zone index of the highest zone to return * @nodes: An optional nodemask to filter the zonelist with * * This function returns the next zone at or below a given zone index that is * within the allowed nodemask using a cursor as the starting point for the * search. The zoneref returned is a cursor that represents the current zone * being examined. It should be advanced by one before calling * next_zones_zonelist again. * * Return: the next zone at or below highest_zoneidx within the allowed * nodemask using a cursor within a zonelist as a starting point */static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, const nodemask_t *nodes){ if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) return z; return __next_zones_zonelist(z, highest_zoneidx, nodes);}
/** * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist * @zonelist: The zonelist to search for a suitable zone * @highest_zoneidx: The zone index of the highest zone to return * @nodes: An optional nodemask to filter the zonelist with * * This function returns the first zone at or below a given zone index that is * within the allowed nodemask. The zoneref returned is a cursor that can be * used to iterate the zonelist with next_zones_zonelist by advancing it by * one before calling. * * When no eligible zone is found, zoneref->zone is NULL (zoneref itself is * never NULL). This may happen either genuinely, or due to concurrent nodemask * update due to cpuset modification. * * Return: Zoneref pointer for the first suitable zone found */static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, const nodemask_t *nodes){ return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes);}
/** * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask * @zone: The current zone in the iterator * @z: The current pointer within zonelist->_zonerefs being iterated * @zlist: The zonelist being iterated * @highidx: The zone index of the highest zone to return * @nodemask: Nodemask allowed by the allocator * * This iterator iterates though all zones at or below a given zone index and * within a given nodemask */#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \ for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z); \ zone; \ z = next_zones_zonelist(++z, highidx, nodemask), \ zone = zonelist_zone(z))
#define for_next_zone_zonelist_nodemask(zone, z, highidx, nodemask) \ for (zone = zonelist_zone(z); \ zone; \ z = next_zones_zonelist(++z, highidx, nodemask), \ zone = zonelist_zone(z))
/** * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index * @zone: The current zone in the iterator * @z: The current pointer within zonelist->zones being iterated * @zlist: The zonelist being iterated * @highidx: The zone index of the highest zone to return * * This iterator iterates though all zones at or below a given zone index. */#define for_each_zone_zonelist(zone, z, zlist, highidx) \ for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
/* Whether the 'nodes' are all movable nodes */static inline bool movable_only_nodes(nodemask_t *nodes){ struct zonelist *zonelist; struct zoneref *z; int nid;
if (nodes_empty(*nodes)) return false;
/* * We can chose arbitrary node from the nodemask to get a * zonelist as they are interlinked. We just need to find * at least one zone that can satisfy kernel allocations. */ nid = first_node(*nodes); zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK]; z = first_zones_zonelist(zonelist, ZONE_NORMAL, nodes); return (!zonelist_zone(z)) ? true : false;}
#ifdef CONFIG_SPARSEMEM#include <asm/sparsemem.h>#endif
#ifdef CONFIG_FLATMEM#define pfn_to_nid(pfn) (0)#endif
#ifdef CONFIG_SPARSEMEM
/* * PA_SECTION_SHIFT physical address to/from section number * PFN_SECTION_SHIFT pfn to/from section number */#define PA_SECTION_SHIFT (SECTION_SIZE_BITS)#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
#define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
#define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT)#define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1))
#define SECTION_BLOCKFLAGS_BITS \ ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
#if (MAX_PAGE_ORDER + PAGE_SHIFT) > SECTION_SIZE_BITS#error Allocator MAX_PAGE_ORDER exceeds SECTION_SIZE#endif
static inline unsigned long pfn_to_section_nr(unsigned long pfn){ return pfn >> PFN_SECTION_SHIFT;}static inline unsigned long section_nr_to_pfn(unsigned long sec){ return sec << PFN_SECTION_SHIFT;}
#define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)#define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK)
#define SUBSECTION_SHIFT 21#define SUBSECTION_SIZE (1UL << SUBSECTION_SHIFT)
#define PFN_SUBSECTION_SHIFT (SUBSECTION_SHIFT - PAGE_SHIFT)#define PAGES_PER_SUBSECTION (1UL << PFN_SUBSECTION_SHIFT)#define PAGE_SUBSECTION_MASK (~(PAGES_PER_SUBSECTION-1))
#if SUBSECTION_SHIFT > SECTION_SIZE_BITS#error Subsection size exceeds section size#else#define SUBSECTIONS_PER_SECTION (1UL << (SECTION_SIZE_BITS - SUBSECTION_SHIFT))#endif
#define SUBSECTION_ALIGN_UP(pfn) ALIGN((pfn), PAGES_PER_SUBSECTION)#define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK)
struct mem_section_usage { struct rcu_head rcu;#ifdef CONFIG_SPARSEMEM_VMEMMAP DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION);#endif /* See declaration of similar field in struct zone */ unsigned long pageblock_flags[0];};
struct page;struct page_ext;struct mem_section { /* * This is, logically, a pointer to an array of struct * pages. However, it is stored with some other magic. * (see sparse_init_one_section()) * * Additionally during early boot we encode node id of * the location of the section here to guide allocation. * (see sparse.c::memory_present()) * * Making it a UL at least makes someone do a cast * before using it wrong. */ unsigned long section_mem_map;
struct mem_section_usage *usage;#ifdef CONFIG_PAGE_EXTENSION /* * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use * section. (see page_ext.h about this.) */ struct page_ext *page_ext; unsigned long pad;#endif /* * WARNING: mem_section must be a power-of-2 in size for the * calculation and use of SECTION_ROOT_MASK to make sense. */};
#ifdef CONFIG_SPARSEMEM_EXTREME#define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))#else#define SECTIONS_PER_ROOT 1#endif
#define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)#define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
#ifdef CONFIG_SPARSEMEM_EXTREMEextern struct mem_section **mem_section;#elseextern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];#endif
static inline unsigned long *section_to_usemap(struct mem_section *ms){ return ms->usage->pageblock_flags;}
static inline struct mem_section *__nr_to_section(unsigned long nr){ unsigned long root = SECTION_NR_TO_ROOT(nr);
if (unlikely(root >= NR_SECTION_ROOTS)) return NULL;
#ifdef CONFIG_SPARSEMEM_EXTREME if (!mem_section || !mem_section[root]) return NULL;#endif return &mem_section[root][nr & SECTION_ROOT_MASK];}
/* * We use the lower bits of the mem_map pointer to store a little bit of * information. The pointer is calculated as mem_map - section_nr_to_pfn(). * The result is aligned to the minimum alignment of the two values: * * 1. All mem_map arrays are page-aligned. * 2. section_nr_to_pfn() always clears PFN_SECTION_SHIFT lowest bits. * * We always expect a single section to cover full pages. Therefore, * we can safely assume that PFN_SECTION_SHIFT is large enough to * accommodate SECTION_MAP_LAST_BIT. We use BUILD_BUG_ON() to ensure this. */enum { SECTION_MARKED_PRESENT_BIT, SECTION_HAS_MEM_MAP_BIT, SECTION_IS_ONLINE_BIT, SECTION_IS_EARLY_BIT,#ifdef CONFIG_ZONE_DEVICE SECTION_TAINT_ZONE_DEVICE_BIT,#endif#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT SECTION_IS_VMEMMAP_PREINIT_BIT,#endif SECTION_MAP_LAST_BIT,};
#define SECTION_MARKED_PRESENT BIT(SECTION_MARKED_PRESENT_BIT)#define SECTION_HAS_MEM_MAP BIT(SECTION_HAS_MEM_MAP_BIT)#define SECTION_IS_ONLINE BIT(SECTION_IS_ONLINE_BIT)#define SECTION_IS_EARLY BIT(SECTION_IS_EARLY_BIT)#ifdef CONFIG_ZONE_DEVICE#define SECTION_TAINT_ZONE_DEVICE BIT(SECTION_TAINT_ZONE_DEVICE_BIT)#endif#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT#define SECTION_IS_VMEMMAP_PREINIT BIT(SECTION_IS_VMEMMAP_PREINIT_BIT)#endif#define SECTION_MAP_MASK (~(BIT(SECTION_MAP_LAST_BIT) - 1))#define SECTION_NID_SHIFT SECTION_MAP_LAST_BIT
static inline struct page *__section_mem_map_addr(struct mem_section *section){ unsigned long map = section->section_mem_map; map &= SECTION_MAP_MASK; return (struct page *)map;}
static inline int present_section(const struct mem_section *section){ return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));}
static inline int present_section_nr(unsigned long nr){ return present_section(__nr_to_section(nr));}
static inline int valid_section(const struct mem_section *section){ return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));}
static inline int early_section(const struct mem_section *section){ return (section && (section->section_mem_map & SECTION_IS_EARLY));}
static inline int valid_section_nr(unsigned long nr){ return valid_section(__nr_to_section(nr));}
static inline int online_section(const struct mem_section *section){ return (section && (section->section_mem_map & SECTION_IS_ONLINE));}
#ifdef CONFIG_ZONE_DEVICEstatic inline int online_device_section(const struct mem_section *section){ unsigned long flags = SECTION_IS_ONLINE | SECTION_TAINT_ZONE_DEVICE;
return section && ((section->section_mem_map & flags) == flags);}#elsestatic inline int online_device_section(const struct mem_section *section){ return 0;}#endif
#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINITstatic inline int preinited_vmemmap_section(const struct mem_section *section){ return (section && (section->section_mem_map & SECTION_IS_VMEMMAP_PREINIT));}
void sparse_vmemmap_init_nid_early(int nid);#elsestatic inline int preinited_vmemmap_section(const struct mem_section *section){ return 0;}static inline void sparse_vmemmap_init_nid_early(int nid){}#endif
static inline int online_section_nr(unsigned long nr){ return online_section(__nr_to_section(nr));}
#ifdef CONFIG_MEMORY_HOTPLUGvoid online_mem_sections(unsigned long start_pfn, unsigned long end_pfn);void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn);#endif
static inline struct mem_section *__pfn_to_section(unsigned long pfn){ return __nr_to_section(pfn_to_section_nr(pfn));}
extern unsigned long __highest_present_section_nr;
static inline int subsection_map_index(unsigned long pfn){ return (pfn & ~(PAGE_SECTION_MASK)) / PAGES_PER_SUBSECTION;}
#ifdef CONFIG_SPARSEMEM_VMEMMAPstatic inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn){ int idx = subsection_map_index(pfn); struct mem_section_usage *usage = READ_ONCE(ms->usage);
return usage ? test_bit(idx, usage->subsection_map) : 0;}
static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn){ struct mem_section_usage *usage = READ_ONCE(ms->usage); int idx = subsection_map_index(*pfn); unsigned long bit;
if (!usage) return false;
if (test_bit(idx, usage->subsection_map)) return true;
/* Find the next subsection that exists */ bit = find_next_bit(usage->subsection_map, SUBSECTIONS_PER_SECTION, idx); if (bit == SUBSECTIONS_PER_SECTION) return false;
*pfn = (*pfn & PAGE_SECTION_MASK) + (bit * PAGES_PER_SUBSECTION); return true;}#elsestatic inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn){ return 1;}
static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn){ return true;}#endif
void sparse_init_early_section(int nid, struct page *map, unsigned long pnum, unsigned long flags);
#ifndef CONFIG_HAVE_ARCH_PFN_VALID/** * pfn_valid - check if there is a valid memory map entry for a PFN * @pfn: the page frame number to check * * Check if there is a valid memory map entry aka struct page for the @pfn. * Note, that availability of the memory map entry does not imply that * there is actual usable memory at that @pfn. The struct page may * represent a hole or an unusable page frame. * * Return: 1 for PFNs that have memory map entries and 0 otherwise */static inline int pfn_valid(unsigned long pfn){ struct mem_section *ms; int ret;
/* * Ensure the upper PAGE_SHIFT bits are clear in the * pfn. Else it might lead to false positives when * some of the upper bits are set, but the lower bits * match a valid pfn. */ if (PHYS_PFN(PFN_PHYS(pfn)) != pfn) return 0;
if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) return 0; ms = __pfn_to_section(pfn); rcu_read_lock_sched(); if (!valid_section(ms)) { rcu_read_unlock_sched(); return 0; } /* * Traditionally early sections always returned pfn_valid() for * the entire section-sized span. */ ret = early_section(ms) || pfn_section_valid(ms, pfn); rcu_read_unlock_sched();
return ret;}
/* Returns end_pfn or higher if no valid PFN remaining in range */static inline unsigned long first_valid_pfn(unsigned long pfn, unsigned long end_pfn){ unsigned long nr = pfn_to_section_nr(pfn);
rcu_read_lock_sched();
while (nr <= __highest_present_section_nr && pfn < end_pfn) { struct mem_section *ms = __pfn_to_section(pfn);
if (valid_section(ms) && (early_section(ms) || pfn_section_first_valid(ms, &pfn))) { rcu_read_unlock_sched(); return pfn; }
/* Nothing left in this section? Skip to next section */ nr++; pfn = section_nr_to_pfn(nr); }
rcu_read_unlock_sched(); return end_pfn;}
static inline unsigned long next_valid_pfn(unsigned long pfn, unsigned long end_pfn){ pfn++;
if (pfn >= end_pfn) return end_pfn;
/* * Either every PFN within the section (or subsection for VMEMMAP) is * valid, or none of them are. So there's no point repeating the check * for every PFN; only call first_valid_pfn() again when crossing a * (sub)section boundary (i.e. !(pfn & ~PAGE_{SUB,}SECTION_MASK)). */ if (pfn & ~(IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP) ? PAGE_SUBSECTION_MASK : PAGE_SECTION_MASK)) return pfn;
return first_valid_pfn(pfn, end_pfn);}
#define for_each_valid_pfn(_pfn, _start_pfn, _end_pfn) \ for ((_pfn) = first_valid_pfn((_start_pfn), (_end_pfn)); \ (_pfn) < (_end_pfn); \ (_pfn) = next_valid_pfn((_pfn), (_end_pfn)))
#endif
static inline int pfn_in_present_section(unsigned long pfn){ if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) return 0; return present_section(__pfn_to_section(pfn));}
static inline unsigned long next_present_section_nr(unsigned long section_nr){ while (++section_nr <= __highest_present_section_nr) { if (present_section_nr(section_nr)) return section_nr; }
return -1;}
#define for_each_present_section_nr(start, section_nr) \ for (section_nr = next_present_section_nr(start - 1); \ section_nr != -1; \ section_nr = next_present_section_nr(section_nr))
/* * These are _only_ used during initialisation, therefore they * can use __initdata ... They could have names to indicate * this restriction. */#ifdef CONFIG_NUMA#define pfn_to_nid(pfn) \({ \ unsigned long __pfn_to_nid_pfn = (pfn); \ page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \})#else#define pfn_to_nid(pfn) (0)#endif
#else#define sparse_vmemmap_init_nid_early(_nid) do {} while (0)#define pfn_in_present_section pfn_valid#endif /* CONFIG_SPARSEMEM */
/* * Fallback case for when the architecture provides its own pfn_valid() but * not a corresponding for_each_valid_pfn(). */#ifndef for_each_valid_pfn#define for_each_valid_pfn(_pfn, _start_pfn, _end_pfn) \ for ((_pfn) = (_start_pfn); (_pfn) < (_end_pfn); (_pfn)++) \ if (pfn_valid(_pfn))#endif
#endif /* !__GENERATING_BOUNDS.H */#endif /* !__ASSEMBLER__ */#endif /* _LINUX_MMZONE_H */