Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
C
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_MM_H#define _LINUX_MM_H
#include <linux/args.h>#include <linux/errno.h>#include <linux/mmdebug.h>#include <linux/gfp.h>#include <linux/pgalloc_tag.h>#include <linux/bug.h>#include <linux/list.h>#include <linux/mmzone.h>#include <linux/rbtree.h>#include <linux/atomic.h>#include <linux/debug_locks.h>#include <linux/compiler.h>#include <linux/mm_types.h>#include <linux/mmap_lock.h>#include <linux/range.h>#include <linux/pfn.h>#include <linux/percpu-refcount.h>#include <linux/bit_spinlock.h>#include <linux/shrinker.h>#include <linux/resource.h>#include <linux/page_ext.h>#include <linux/err.h>#include <linux/page-flags.h>#include <linux/page_ref.h>#include <linux/overflow.h>#include <linux/sched.h>#include <linux/pgtable.h>#include <linux/kasan.h>#include <linux/memremap.h>#include <linux/slab.h>#include <linux/cacheinfo.h>#include <linux/rcuwait.h>#include <linux/bitmap.h>#include <linux/bitops.h>#include <linux/iommu-debug-pagealloc.h>#include <linux/kcsan-checks.h>
struct mempolicy;struct anon_vma;struct anon_vma_chain;struct user_struct;struct pt_regs;struct folio_batch;
void arch_mm_preinit(void);void mm_core_init_early(void);void mm_core_init(void);void init_mm_internals(void);
extern atomic_long_t _totalram_pages;static inline unsigned long totalram_pages(void){ return (unsigned long)atomic_long_read(&_totalram_pages);}
static inline void totalram_pages_inc(void){ atomic_long_inc(&_totalram_pages);}
static inline void totalram_pages_dec(void){ atomic_long_dec(&_totalram_pages);}
static inline void totalram_pages_add(long count){ atomic_long_add(count, &_totalram_pages);}
extern void * high_memory;
/* * Convert between pages and MB * 20 is the shift for 1MB (2^20 = 1MB) * PAGE_SHIFT is the shift for page size (e.g., 12 for 4KB pages) * So (20 - PAGE_SHIFT) converts between pages and MB */#define PAGES_TO_MB(pages) ((pages) >> (20 - PAGE_SHIFT))#define MB_TO_PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
#ifdef CONFIG_SYSCTLextern int sysctl_legacy_va_layout;#else#define sysctl_legacy_va_layout 0#endif
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITSextern const int mmap_rnd_bits_min;extern int mmap_rnd_bits_max __ro_after_init;extern int mmap_rnd_bits __read_mostly;#endif#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITSextern const int mmap_rnd_compat_bits_min;extern const int mmap_rnd_compat_bits_max;extern int mmap_rnd_compat_bits __read_mostly;#endif
#ifndef DIRECT_MAP_PHYSMEM_END# ifdef MAX_PHYSMEM_BITS# define DIRECT_MAP_PHYSMEM_END ((1ULL << MAX_PHYSMEM_BITS) - 1)# else# define DIRECT_MAP_PHYSMEM_END (((phys_addr_t)-1)&~(1ULL<<63))# endif#endif
#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
#include <asm/page.h>#include <asm/processor.h>
#ifndef __pa_symbol#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))#endif
#ifndef page_to_virt#define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))#endif
#ifndef lm_alias#define lm_alias(x) __va(__pa_symbol(x))#endif
/* * To prevent common memory management code establishing * a zero page mapping on a read fault. * This macro should be defined within <asm/pgtable.h>. * s390 does this to prevent multiplexing of hardware bits * related to the physical page in case of virtualization. */#ifndef mm_forbids_zeropage#define mm_forbids_zeropage(X) (0)#endif
/* * On some architectures it is expensive to call memset() for small sizes. * If an architecture decides to implement their own version of * mm_zero_struct_page they should wrap the defines below in a #ifndef and * define their own version of this macro in <asm/pgtable.h> */#if BITS_PER_LONG == 64/* This function must be updated when the size of struct page grows above 96 * or reduces below 56. The idea that compiler optimizes out switch() * statement, and only leaves move/store instructions. Also the compiler can * combine write statements if they are both assignments and can be reordered, * this can result in several of the writes here being dropped. */#define mm_zero_struct_page(pp) __mm_zero_struct_page(pp)static inline void __mm_zero_struct_page(struct page *page){ unsigned long *_pp = (void *)page;
/* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */ BUILD_BUG_ON(sizeof(struct page) & 7); BUILD_BUG_ON(sizeof(struct page) < 56); BUILD_BUG_ON(sizeof(struct page) > 96);
switch (sizeof(struct page)) { case 96: _pp[11] = 0; fallthrough; case 88: _pp[10] = 0; fallthrough; case 80: _pp[9] = 0; fallthrough; case 72: _pp[8] = 0; fallthrough; case 64: _pp[7] = 0; fallthrough; case 56: _pp[6] = 0; _pp[5] = 0; _pp[4] = 0; _pp[3] = 0; _pp[2] = 0; _pp[1] = 0; _pp[0] = 0; }}#else#define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))#endif
/* * Default maximum number of active map areas, this limits the number of vmas * per mm struct. Users can overwrite this number by sysctl but there is a * problem. * * When a program's coredump is generated as ELF format, a section is created * per a vma. In ELF, the number of sections is represented in unsigned short. * This means the number of sections should be smaller than 65535 at coredump. * Because the kernel adds some informative sections to a image of program at * generating coredump, we need some margin. The number of extra sections is * 1-3 now and depends on arch. We use "5" as safe margin, here. * * ELF extended numbering allows more than 65535 sections, so 16-bit bound is * not a hard limit any more. Although some userspace tools can be surprised by * that. */#define MAPCOUNT_ELF_CORE_MARGIN (5)#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
extern unsigned long sysctl_user_reserve_kbytes;extern unsigned long sysctl_admin_reserve_kbytes;
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)bool page_range_contiguous(const struct page *page, unsigned long nr_pages);#elsestatic inline bool page_range_contiguous(const struct page *page, unsigned long nr_pages){ return true;}#endif
/* to align the pointer to the (next) page boundary */#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
/* to align the pointer to the (prev) page boundary */#define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE)
/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
/** * folio_page_idx - Return the number of a page in a folio. * @folio: The folio. * @page: The folio page. * * This function expects that the page is actually part of the folio. * The returned number is relative to the start of the folio. */static inline unsigned long folio_page_idx(const struct folio *folio, const struct page *page){ return page - &folio->page;}
static inline struct folio *lru_to_folio(struct list_head *head){ return list_entry((head)->prev, struct folio, lru);}
void setup_initial_init_mm(void *start_code, void *end_code, void *end_data, void *brk);
/* * Linux kernel virtual memory manager primitives. * The idea being to have a "virtual" mm in the same way * we have a virtual fs - giving a cleaner interface to the * mm details, and allowing different kinds of memory mappings * (from shared memory to executable loading to arbitrary * mmap() functions). */
struct vm_area_struct *vm_area_alloc(struct mm_struct *);struct vm_area_struct *vm_area_dup(struct vm_area_struct *);void vm_area_free(struct vm_area_struct *);
#ifndef CONFIG_MMUextern struct rb_root nommu_region_tree;extern struct rw_semaphore nommu_region_sem;
extern unsigned int kobjsize(const void *objp);#endif
/* * vm_flags in vm_area_struct, see mm_types.h. * When changing, update also include/trace/events/mmflags.h */
#define VM_NONE 0x00000000
/** * typedef vma_flag_t - specifies an individual VMA flag by bit number. * * This value is made type safe by sparse to avoid passing invalid flag values * around. */typedef int __bitwise vma_flag_t;
#define DECLARE_VMA_BIT(name, bitnum) \ VMA_ ## name ## _BIT = ((__force vma_flag_t)bitnum)#define DECLARE_VMA_BIT_ALIAS(name, aliased) \ VMA_ ## name ## _BIT = (VMA_ ## aliased ## _BIT)enum { DECLARE_VMA_BIT(READ, 0), DECLARE_VMA_BIT(WRITE, 1), DECLARE_VMA_BIT(EXEC, 2), DECLARE_VMA_BIT(SHARED, 3), /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ DECLARE_VMA_BIT(MAYREAD, 4), /* limits for mprotect() etc. */ DECLARE_VMA_BIT(MAYWRITE, 5), DECLARE_VMA_BIT(MAYEXEC, 6), DECLARE_VMA_BIT(MAYSHARE, 7), DECLARE_VMA_BIT(GROWSDOWN, 8), /* general info on the segment */#ifdef CONFIG_MMU DECLARE_VMA_BIT(UFFD_MISSING, 9),/* missing pages tracking */#else /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */ DECLARE_VMA_BIT(MAYOVERLAY, 9),#endif /* CONFIG_MMU */ /* Page-ranges managed without "struct page", just pure PFN */ DECLARE_VMA_BIT(PFNMAP, 10), DECLARE_VMA_BIT(MAYBE_GUARD, 11), DECLARE_VMA_BIT(UFFD_WP, 12), /* wrprotect pages tracking */ DECLARE_VMA_BIT(LOCKED, 13), DECLARE_VMA_BIT(IO, 14), /* Memory mapped I/O or similar */ DECLARE_VMA_BIT(SEQ_READ, 15), /* App will access data sequentially */ DECLARE_VMA_BIT(RAND_READ, 16), /* App will not benefit from clustered reads */ DECLARE_VMA_BIT(DONTCOPY, 17), /* Do not copy this vma on fork */ DECLARE_VMA_BIT(DONTEXPAND, 18),/* Cannot expand with mremap() */ DECLARE_VMA_BIT(LOCKONFAULT, 19),/* Lock pages covered when faulted in */ DECLARE_VMA_BIT(ACCOUNT, 20), /* Is a VM accounted object */ DECLARE_VMA_BIT(NORESERVE, 21), /* should the VM suppress accounting */ DECLARE_VMA_BIT(HUGETLB, 22), /* Huge TLB Page VM */ DECLARE_VMA_BIT(SYNC, 23), /* Synchronous page faults */ DECLARE_VMA_BIT(ARCH_1, 24), /* Architecture-specific flag */ DECLARE_VMA_BIT(WIPEONFORK, 25),/* Wipe VMA contents in child. */ DECLARE_VMA_BIT(DONTDUMP, 26), /* Do not include in the core dump */ DECLARE_VMA_BIT(SOFTDIRTY, 27), /* NOT soft dirty clean area */ DECLARE_VMA_BIT(MIXEDMAP, 28), /* Can contain struct page and pure PFN pages */ DECLARE_VMA_BIT(HUGEPAGE, 29), /* MADV_HUGEPAGE marked this vma */ DECLARE_VMA_BIT(NOHUGEPAGE, 30),/* MADV_NOHUGEPAGE marked this vma */ DECLARE_VMA_BIT(MERGEABLE, 31), /* KSM may merge identical pages */ /* These bits are reused, we define specific uses below. */ DECLARE_VMA_BIT(HIGH_ARCH_0, 32), DECLARE_VMA_BIT(HIGH_ARCH_1, 33), DECLARE_VMA_BIT(HIGH_ARCH_2, 34), DECLARE_VMA_BIT(HIGH_ARCH_3, 35), DECLARE_VMA_BIT(HIGH_ARCH_4, 36), DECLARE_VMA_BIT(HIGH_ARCH_5, 37), DECLARE_VMA_BIT(HIGH_ARCH_6, 38), /* * This flag is used to connect VFIO to arch specific KVM code. It * indicates that the memory under this VMA is safe for use with any * non-cachable memory type inside KVM. Some VFIO devices, on some * platforms, are thought to be unsafe and can cause machine crashes * if KVM does not lock down the memory type. */ DECLARE_VMA_BIT(ALLOW_ANY_UNCACHED, 39),#if defined(CONFIG_PPC32) DECLARE_VMA_BIT_ALIAS(DROPPABLE, ARCH_1),#elif defined(CONFIG_64BIT) DECLARE_VMA_BIT(DROPPABLE, 40),#endif DECLARE_VMA_BIT(UFFD_MINOR, 41), DECLARE_VMA_BIT(SEALED, 42), DECLARE_VMA_BIT(UFFD_RWP, 43), /* Flags that reuse flags above. */ DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), DECLARE_VMA_BIT_ALIAS(PKEY_BIT2, HIGH_ARCH_2), DECLARE_VMA_BIT_ALIAS(PKEY_BIT3, HIGH_ARCH_3), DECLARE_VMA_BIT_ALIAS(PKEY_BIT4, HIGH_ARCH_4),#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_RISCV_USER_CFI) /* * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of * support core mm. * * These VMAs will get a single end guard page. This helps userspace * protect itself from attacks. A single page is enough for current * shadow stack archs (x86). See the comments near alloc_shstk() in * arch/x86/kernel/shstk.c for more details on the guard size. */ DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_5),#elif defined(CONFIG_ARM64_GCS) /* * arm64's Guarded Control Stack implements similar functionality and * has similar constraints to shadow stacks. */ DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_6),#endif DECLARE_VMA_BIT_ALIAS(SAO, ARCH_1), /* Strong Access Ordering (powerpc) */ DECLARE_VMA_BIT_ALIAS(GROWSUP, ARCH_1), /* parisc */ DECLARE_VMA_BIT_ALIAS(SPARC_ADI, ARCH_1), /* sparc64 */ DECLARE_VMA_BIT_ALIAS(ARM64_BTI, ARCH_1), /* arm64 */ DECLARE_VMA_BIT_ALIAS(ARCH_CLEAR, ARCH_1), /* sparc64, arm64 */ DECLARE_VMA_BIT_ALIAS(MAPPED_COPY, ARCH_1), /* !CONFIG_MMU */ DECLARE_VMA_BIT_ALIAS(MTE, HIGH_ARCH_4), /* arm64 */ DECLARE_VMA_BIT_ALIAS(MTE_ALLOWED, HIGH_ARCH_5),/* arm64 */#ifdef CONFIG_STACK_GROWSUP DECLARE_VMA_BIT_ALIAS(STACK, GROWSUP), DECLARE_VMA_BIT_ALIAS(STACK_EARLY, GROWSDOWN),#else DECLARE_VMA_BIT_ALIAS(STACK, GROWSDOWN),#endif};#undef DECLARE_VMA_BIT#undef DECLARE_VMA_BIT_ALIAS
#define INIT_VM_FLAG(name) BIT((__force int) VMA_ ## name ## _BIT)#define VM_READ INIT_VM_FLAG(READ)#define VM_WRITE INIT_VM_FLAG(WRITE)#define VM_EXEC INIT_VM_FLAG(EXEC)#define VM_SHARED INIT_VM_FLAG(SHARED)#define VM_MAYREAD INIT_VM_FLAG(MAYREAD)#define VM_MAYWRITE INIT_VM_FLAG(MAYWRITE)#define VM_MAYEXEC INIT_VM_FLAG(MAYEXEC)#define VM_MAYSHARE INIT_VM_FLAG(MAYSHARE)#define VM_GROWSDOWN INIT_VM_FLAG(GROWSDOWN)#ifdef CONFIG_MMU#define VM_UFFD_MISSING INIT_VM_FLAG(UFFD_MISSING)#else#define VM_UFFD_MISSING VM_NONE#define VM_MAYOVERLAY INIT_VM_FLAG(MAYOVERLAY)#endif#define VM_PFNMAP INIT_VM_FLAG(PFNMAP)#define VM_MAYBE_GUARD INIT_VM_FLAG(MAYBE_GUARD)#define VM_UFFD_WP INIT_VM_FLAG(UFFD_WP)#define VM_LOCKED INIT_VM_FLAG(LOCKED)#define VM_IO INIT_VM_FLAG(IO)#define VM_SEQ_READ INIT_VM_FLAG(SEQ_READ)#define VM_RAND_READ INIT_VM_FLAG(RAND_READ)#define VM_DONTCOPY INIT_VM_FLAG(DONTCOPY)#define VM_DONTEXPAND INIT_VM_FLAG(DONTEXPAND)#define VM_LOCKONFAULT INIT_VM_FLAG(LOCKONFAULT)#define VM_ACCOUNT INIT_VM_FLAG(ACCOUNT)#define VM_NORESERVE INIT_VM_FLAG(NORESERVE)#define VM_HUGETLB INIT_VM_FLAG(HUGETLB)#define VM_SYNC INIT_VM_FLAG(SYNC)#define VM_ARCH_1 INIT_VM_FLAG(ARCH_1)#define VM_WIPEONFORK INIT_VM_FLAG(WIPEONFORK)#define VM_DONTDUMP INIT_VM_FLAG(DONTDUMP)#ifdef CONFIG_MEM_SOFT_DIRTY#define VM_SOFTDIRTY INIT_VM_FLAG(SOFTDIRTY)#else#define VM_SOFTDIRTY VM_NONE#endif#define VM_MIXEDMAP INIT_VM_FLAG(MIXEDMAP)#define VM_HUGEPAGE INIT_VM_FLAG(HUGEPAGE)#define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE)#define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE)#define VM_STACK INIT_VM_FLAG(STACK)#ifdef CONFIG_STACK_GROWSUP#define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY)#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT)#else#define VM_STACK_EARLY VM_NONE#define VMA_STACK_EARLY EMPTY_VMA_FLAGS#endif#ifdef CONFIG_ARCH_HAS_PKEYS#define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT)/* Despite the naming, these are FLAGS not bits. */#define VM_PKEY_BIT0 INIT_VM_FLAG(PKEY_BIT0)#define VM_PKEY_BIT1 INIT_VM_FLAG(PKEY_BIT1)#define VM_PKEY_BIT2 INIT_VM_FLAG(PKEY_BIT2)#if CONFIG_ARCH_PKEY_BITS > 3#define VM_PKEY_BIT3 INIT_VM_FLAG(PKEY_BIT3)#else#define VM_PKEY_BIT3 VM_NONE#endif /* CONFIG_ARCH_PKEY_BITS > 3 */#if CONFIG_ARCH_PKEY_BITS > 4#define VM_PKEY_BIT4 INIT_VM_FLAG(PKEY_BIT4)#else#define VM_PKEY_BIT4 VM_NONE#endif /* CONFIG_ARCH_PKEY_BITS > 4 */#endif /* CONFIG_ARCH_HAS_PKEYS */#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ defined(CONFIG_RISCV_USER_CFI)#define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK)#define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT)#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT)#else#define VM_SHADOW_STACK VM_NONE#define VMA_SHADOW_STACK EMPTY_VMA_FLAGS#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT)#endif#if defined(CONFIG_PPC64)#define VM_SAO INIT_VM_FLAG(SAO)#elif defined(CONFIG_PARISC)#define VM_GROWSUP INIT_VM_FLAG(GROWSUP)#define VMA_GROWSUP mk_vma_flags(VMA_GROWSUP_BIT)#elif defined(CONFIG_SPARC64)#define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI)#define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR)#elif defined(CONFIG_ARM64)#define VM_ARM64_BTI INIT_VM_FLAG(ARM64_BTI)#define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR)#elif !defined(CONFIG_MMU)#define VM_MAPPED_COPY INIT_VM_FLAG(MAPPED_COPY)#endif#ifndef VM_GROWSUP#define VM_GROWSUP VM_NONE#define VMA_GROWSUP EMPTY_VMA_FLAGS#endif#ifdef CONFIG_ARM64_MTE#define VM_MTE INIT_VM_FLAG(MTE)#define VM_MTE_ALLOWED INIT_VM_FLAG(MTE_ALLOWED)#else#define VM_MTE VM_NONE#define VM_MTE_ALLOWED VM_NONE#endif#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR#define VM_UFFD_MINOR INIT_VM_FLAG(UFFD_MINOR)#else#define VM_UFFD_MINOR VM_NONE#endif#ifdef CONFIG_USERFAULTFD_RWP#define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP)#else#define VM_UFFD_RWP VM_NONE#endif
/* * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are * gated on the same configs as their VM_* flags above -- both of which imply * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build * whose bitmap cannot hold it. */#define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT)#define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT)#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR#define VMA_UFFD_MINOR mk_vma_flags(VMA_UFFD_MINOR_BIT)#else#define VMA_UFFD_MINOR EMPTY_VMA_FLAGS#endif#ifdef CONFIG_USERFAULTFD_RWP#define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT)#else#define VMA_UFFD_RWP EMPTY_VMA_FLAGS#endif
#ifdef CONFIG_64BIT#define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED)#define VM_SEALED INIT_VM_FLAG(SEALED)#else#define VM_ALLOW_ANY_UNCACHED VM_NONE#define VM_SEALED VM_NONE#endif#if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)#define VM_DROPPABLE INIT_VM_FLAG(DROPPABLE)#define VMA_DROPPABLE mk_vma_flags(VMA_DROPPABLE_BIT)#else#define VM_DROPPABLE VM_NONE#define VMA_DROPPABLE EMPTY_VMA_FLAGS#endif
/* Bits set in the VMA until the stack is in its final location */#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY)#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT)
#define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ VMA_EXEC_BIT : VMA_READ_BIT)
/* Common data flag combinations */#define VMA_DATA_FLAGS_TSK_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ TASK_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ VMA_MAYEXEC_BIT)#define VMA_DATA_FLAGS_NON_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, VMA_MAYEXEC_BIT)#define VMA_DATA_FLAGS_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ VMA_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ VMA_MAYEXEC_BIT)
#ifndef VMA_DATA_DEFAULT_FLAGS /* arch can override this */#define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_EXEC#endif
#ifndef VMA_STACK_DEFAULT_FLAGS /* arch can override this */#define VMA_STACK_DEFAULT_FLAGS VMA_DATA_DEFAULT_FLAGS#endif
#define VMA_STACK_FLAGS append_vma_flags(VMA_STACK_DEFAULT_FLAGS, \ VMA_STACK_BIT, VMA_ACCOUNT_BIT)
/* Temporary until VMA flags conversion complete. */#define VM_STACK_FLAGS vma_flags_to_legacy(VMA_STACK_FLAGS)
#ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS#define VM_SEALED_SYSMAP VM_SEALED#else#define VM_SEALED_SYSMAP VM_NONE#endif
/* VMA basic access permission flags */#define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)#define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT)
/* * Special vmas that are non-mergable, non-mlock()able. */
#define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \ VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT)#define VM_SPECIAL vma_flags_to_legacy(VMA_SPECIAL_FLAGS)
/* * Physically remapped pages are special. Tell the * rest of the world about it: * IO tells people not to look at these pages * (accesses can have side effects). * PFNMAP tells the core MM that the base pages are just * raw PFN mappings, and do not have a "struct page" associated * with them. * DONTEXPAND * Disable vma merging and expanding with mremap(). * DONTDUMP * Omit vma from core dump, even when VM_IO turned off. */#define VMA_REMAP_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, \ VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)
/* This mask prevents VMA from being scanned with khugepaged */#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
/* This mask defines which mm->def_flags a process can inherit its parent */#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
/* This mask represents all the VMA flag bits used by mlock */#define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT)
#define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT)
/* These flags can be updated atomically via VMA/mmap read lock. */#define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD
/* Arch-specific flags to clear when updating VM flags on protection change */#ifndef VM_ARCH_CLEAR#define VM_ARCH_CLEAR VM_NONE#endif#define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR)
/* * Flags which should be 'sticky' on merge - that is, flags which, when one VMA * possesses it but the other does not, the merged VMA should nonetheless have * applied to it: * * VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its * references cleared via /proc/$pid/clear_refs, any * merged VMA should be considered soft-dirty also as it * operates at a VMA granularity. * * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies * that mapped page tables may contain metadata not * described by the VMA and thus any merged VMA may also * contain this metadata, and thus we must make this flag * sticky. */#ifdef CONFIG_MEM_SOFT_DIRTY#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)#else#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)#endif
/* * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one * of these flags and the other not does not preclude a merge. * * VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they * are 'sticky'. If any sticky flags exist in either VMA, * we simply set all of them on the merged VMA. */#define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS
/* * Flags which should result in page tables being copied on fork. These are * flags which indicate that the VMA maps page tables which cannot be * reconsistuted upon page fault, so necessitate page table copying upon fork. * * Note that these flags should be compared with the DESTINATION VMA not the * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate. * * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be * reasonably reconstructed on page fault. * * VM_UFFD_WP - Encodes metadata about an installed uffd * VM_UFFD_RWP write- or read-write-protect handler, which * cannot be reconstructed on page fault. * * We always copy pgtables when dst_vma has the * uffd PTE bit in use even if it's file-backed * (e.g. shmem). Because when the uffd bit is * in use, the pgtable contains the protection * information, that's something we can't * retrieve from page cache, and skip copying * will lose those info. * * VM_MAYBE_GUARD - Could contain page guard region markers which * by design are a property of the page tables * only and thus cannot be reconstructed on page * fault. */#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \ VM_MAYBE_GUARD)
/* * mapping from the currently active vm_flags protection bits (the * low four bits) to a page protection mask.. */
/* * The default fault flags that should be used by most of the * arch-specific page fault handlers. */#define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \ FAULT_FLAG_KILLABLE | \ FAULT_FLAG_INTERRUPTIBLE)
/** * fault_flag_allow_retry_first - check ALLOW_RETRY the first time * @flags: Fault flags. * * This is mostly used for places where we want to try to avoid taking * the mmap_lock for too long a time when waiting for another condition * to change, in which case we can try to be polite to release the * mmap_lock in the first round to avoid potential starvation of other * processes that would also want the mmap_lock. * * Return: true if the page fault allows retry and this is the first * attempt of the fault handling; false otherwise. */static inline bool fault_flag_allow_retry_first(enum fault_flag flags){ return (flags & FAULT_FLAG_ALLOW_RETRY) && (!(flags & FAULT_FLAG_TRIED));}
#define FAULT_FLAG_TRACE \ { FAULT_FLAG_WRITE, "WRITE" }, \ { FAULT_FLAG_MKWRITE, "MKWRITE" }, \ { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \ { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \ { FAULT_FLAG_KILLABLE, "KILLABLE" }, \ { FAULT_FLAG_TRIED, "TRIED" }, \ { FAULT_FLAG_USER, "USER" }, \ { FAULT_FLAG_REMOTE, "REMOTE" }, \ { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" }
/* * vm_fault is filled by the pagefault handler and passed to the vma's * ->fault function. The vma's ->fault is responsible for returning a bitmask * of VM_FAULT_xxx flags that give details about how the fault was handled. * * MM layer fills up gfp_mask for page allocations but fault handler might * alter it if its implementation requires a different allocation context. * * pgoff should be used in favour of virtual_address, if possible. */struct vm_fault { const struct { struct vm_area_struct *vma; /* Target VMA */ gfp_t gfp_mask; /* gfp mask to be used for allocations */ pgoff_t pgoff; /* Logical page offset based on vma */ unsigned long address; /* Faulting virtual address - masked */ unsigned long real_address; /* Faulting virtual address - unmasked */ }; enum fault_flag flags; /* FAULT_FLAG_xxx flags * XXX: should really be 'const' */ pmd_t *pmd; /* Pointer to pmd entry matching * the 'address' */ pud_t *pud; /* Pointer to pud entry matching * the 'address' */ union { pte_t orig_pte; /* Value of PTE at the time of fault */ pmd_t orig_pmd; /* Value of PMD at the time of fault, * used by PMD fault only. */ };
struct page *cow_page; /* Page handler may use for COW fault */ struct page *page; /* ->fault handlers should return a * page here, unless VM_FAULT_NOPAGE * is set (which is also implied by * VM_FAULT_ERROR). */ /* These three entries are valid only while holding ptl lock */ pte_t *pte; /* Pointer to pte entry matching * the 'address'. NULL if the page * table hasn't been allocated. */ spinlock_t *ptl; /* Page table lock. * Protects pte page table if 'pte' * is not NULL, otherwise pmd. */ pgtable_t prealloc_pte; /* Pre-allocated pte page table. * vm_ops->map_pages() sets up a page * table from atomic context. * do_fault_around() pre-allocates * page table to avoid allocation from * atomic context. */};
struct vm_uffd_ops;
/* * These are the virtual MM functions - opening of an area, closing and * unmapping it (needed to keep files on disk up-to-date etc), pointer * to the functions called when a no-page or a wp-page exception occurs. */struct vm_operations_struct { /** * @open: Called when a VMA is remapped, split or forked. Not called * upon first mapping a VMA. * Context: User context. May sleep. Caller holds mmap_lock. */ void (*open)(struct vm_area_struct *vma); /** * @close: Called when the VMA is being removed from the MM. * Context: User context. May sleep. Caller holds mmap_lock. */ void (*close)(struct vm_area_struct *vma); /** * @mapped: Called when the VMA is first mapped in the MM. Not called if * the new VMA is merged with an adjacent VMA. * * The @vm_private_data field is an output field allowing the user to * modify vma->vm_private_data as necessary. * * ONLY valid if set from f_op->mmap_prepare. Will result in an error if * set from f_op->mmap. * * Returns %0 on success, or an error otherwise. On error, the VMA will * be unmapped. * * Context: User context. May sleep. Caller holds mmap_lock. */ int (*mapped)(unsigned long start, unsigned long end, pgoff_t pgoff, const struct file *file, void **vm_private_data); /* Called any time before splitting to check if it's allowed */ int (*may_split)(struct vm_area_struct *vma, unsigned long addr); int (*mremap)(struct vm_area_struct *vma); /* * Called by mprotect() to make driver-specific permission * checks before mprotect() is finalised. The VMA must not * be modified. Returns 0 if mprotect() can proceed. */ int (*mprotect)(struct vm_area_struct *vma, unsigned long start, unsigned long end, unsigned long newflags); vm_fault_t (*fault)(struct vm_fault *vmf); vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order); vm_fault_t (*map_pages)(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff); unsigned long (*pagesize)(struct vm_area_struct *vma);
/* notification that a previously read-only page is about to become * writable, if an error is returned it will cause a SIGBUS */ vm_fault_t (*page_mkwrite)(struct vm_fault *vmf);
/* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */ vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf);
/* called by access_process_vm when get_user_pages() fails, typically * for use by special VMAs. See also generic_access_phys() for a generic * implementation useful for any iomem mapping. */ int (*access)(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write);
/* Called by the /proc/PID/maps code to ask the vma whether it * has a special name. Returning non-NULL will also cause this * vma to be dumped unconditionally. */ const char *(*name)(struct vm_area_struct *vma);
#ifdef CONFIG_NUMA /* * set_policy() op must add a reference to any non-NULL @new mempolicy * to hold the policy upon return. Caller should pass NULL @new to * remove a policy and fall back to surrounding context--i.e. do not * install a MPOL_DEFAULT policy, nor the task or system default * mempolicy. */ int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
/* * get_policy() op must add reference [mpol_get()] to any policy at * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure * in mm/mempolicy.c will do this automatically. * get_policy() must NOT add a ref if the policy at (vma,addr) is not * marked as MPOL_SHARED. vma policies are protected by the mmap_lock. * If no [shared/vma] mempolicy exists at the addr, get_policy() op * must return NULL--i.e., do not "fallback" to task or system default * policy. */ struct mempolicy *(*get_policy)(struct vm_area_struct *vma, unsigned long addr, pgoff_t *ilx);#endif#ifdef CONFIG_FIND_NORMAL_PAGE /* * Called by vm_normal_page() for special PTEs in @vma at @addr. This * allows for returning a "normal" page from vm_normal_page() even * though the PTE indicates that the "struct page" either does not exist * or should not be touched: "special". * * Do not add new users: this really only works when a "normal" page * was mapped, but then the PTE got changed to something weird (+ * marked special) that would not make pte_pfn() identify the originally * inserted page. */ struct page *(*find_normal_page)(struct vm_area_struct *vma, unsigned long addr);#endif /* CONFIG_FIND_NORMAL_PAGE */#ifdef CONFIG_USERFAULTFD const struct vm_uffd_ops *uffd_ops;#endif};
#ifdef CONFIG_NUMA_BALANCINGstatic inline void vma_numab_state_init(struct vm_area_struct *vma){ vma->numab_state = NULL;}static inline void vma_numab_state_free(struct vm_area_struct *vma){ kfree(vma->numab_state);}#elsestatic inline void vma_numab_state_init(struct vm_area_struct *vma) {}static inline void vma_numab_state_free(struct vm_area_struct *vma) {}#endif /* CONFIG_NUMA_BALANCING */
/* * These must be here rather than mmap_lock.h as dependent on vm_fault type, * declared in this header. */#ifdef CONFIG_PER_VMA_LOCKstatic inline void release_fault_lock(struct vm_fault *vmf){ if (vmf->flags & FAULT_FLAG_VMA_LOCK) vma_end_read(vmf->vma); else mmap_read_unlock(vmf->vma->vm_mm);}
static inline void assert_fault_locked(const struct vm_fault *vmf){ if (vmf->flags & FAULT_FLAG_VMA_LOCK) vma_assert_locked(vmf->vma); else mmap_assert_locked(vmf->vma->vm_mm);}#elsestatic inline void release_fault_lock(struct vm_fault *vmf){ mmap_read_unlock(vmf->vma->vm_mm);}
static inline void assert_fault_locked(const struct vm_fault *vmf){ mmap_assert_locked(vmf->vma->vm_mm);}#endif /* CONFIG_PER_VMA_LOCK */
static inline bool mm_flags_test(int flag, const struct mm_struct *mm){ return test_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline bool mm_flags_test_and_set(int flag, struct mm_struct *mm){ return test_and_set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline bool mm_flags_test_and_clear(int flag, struct mm_struct *mm){ return test_and_clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline void mm_flags_set(int flag, struct mm_struct *mm){ set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline void mm_flags_clear(int flag, struct mm_struct *mm){ clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));}
static inline void mm_flags_clear_all(struct mm_struct *mm){ bitmap_zero(ACCESS_PRIVATE(&mm->flags, __mm_flags), NUM_MM_FLAG_BITS);}
extern const struct vm_operations_struct vma_dummy_vm_ops;
static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm){ memset(vma, 0, sizeof(*vma)); vma->vm_mm = mm; vma->vm_ops = &vma_dummy_vm_ops; INIT_LIST_HEAD(&vma->anon_vma_chain); vma_lock_init(vma, false);}
/* Use when VMA is not part of the VMA tree and needs no locking */static inline void vm_flags_init(struct vm_area_struct *vma, vm_flags_t flags){ VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); vma_flags_clear_all(&vma->flags); vma_flags_overwrite_word(&vma->flags, flags);}
/* * Use when VMA is part of the VMA tree and modifications need coordination * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and * it should be locked explicitly beforehand. */static inline void vm_flags_reset(struct vm_area_struct *vma, vm_flags_t flags){ VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); vma_assert_write_locked(vma); vm_flags_init(vma, flags);}
static inline void vma_flags_reset_once(struct vm_area_struct *vma, vma_flags_t *flags){ const unsigned long word = flags->__vma_flags[0];
/* It is assumed only the first system word must be written once. */ vma_flags_overwrite_word_once(&vma->flags, word); /* The remainder can be copied normally. */ if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { unsigned long *dst = &vma->flags.__vma_flags[1]; const unsigned long *src = &flags->__vma_flags[1];
bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); }}
static inline void vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags){ vma_start_write(vma); vma_flags_set_word(&vma->flags, flags);}
static inline void vm_flags_clear(struct vm_area_struct *vma, vm_flags_t flags){ VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY)); vma_start_write(vma); vma_flags_clear_word(&vma->flags, flags);}
/* * Use only if VMA is not part of the VMA tree or has no other users and * therefore needs no locking. */static inline void __vm_flags_mod(struct vm_area_struct *vma, vm_flags_t set, vm_flags_t clear){ vm_flags_init(vma, (vma->vm_flags | set) & ~clear);}
/* * Use only when the order of set/clear operations is unimportant, otherwise * use vm_flags_{set|clear} explicitly. */static inline void vm_flags_mod(struct vm_area_struct *vma, vm_flags_t set, vm_flags_t clear){ vma_start_write(vma); __vm_flags_mod(vma, set, clear);}
static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, vma_flag_t bit){ const vm_flags_t mask = BIT((__force int)bit);
/* Only specific flags are permitted */ if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED))) return false;
return true;}
/* * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific * valid flags are allowed to do this. */static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit){ unsigned long *bitmap = vma->flags.__vma_flags;
vma_assert_stabilised(vma); if (__vma_atomic_valid_flag(vma, bit)) set_bit((__force int)bit, bitmap);}
/* * Test for VMA flag atomically. Requires no locks. Only specific valid flags * are allowed to do this. * * This is necessarily racey, so callers must ensure that serialisation is * achieved through some other means, or that races are permissible. */static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit){ if (__vma_atomic_valid_flag(vma, bit)) return test_bit((__force int)bit, &vma->vm_flags);
return false;}
/* Set an individual VMA flag in flags, non-atomically. */static __always_inline void vma_flags_set_flag(vma_flags_t *flags, vma_flag_t bit){ unsigned long *bitmap = flags->__vma_flags;
__set_bit((__force int)bit, bitmap);}
static __always_inline vma_flags_t __mk_vma_flags(vma_flags_t flags, size_t count, const vma_flag_t *bits){ int i;
for (i = 0; i < count; i++) vma_flags_set_flag(&flags, bits[i]); return flags;}
/* * Helper macro which bitwise-or combines the specified input flags into a * vma_flags_t bitmap value. E.g.: * * vma_flags_t flags = mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT); * * The compiler cleverly optimises away all of the work and this ends up being * equivalent to aggregating the values manually. */#define mk_vma_flags(...) __mk_vma_flags(EMPTY_VMA_FLAGS, \ COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__})
/* * Helper macro which acts like mk_vma_flags, only appending to a copy of the * specified flags rather than establishing new flags. E.g.: * * vma_flags_t flags = append_vma_flags(VMA_STACK_DEFAULT_FLAGS, VMA_STACK_BIT, * VMA_ACCOUNT_BIT); */#define append_vma_flags(flags, ...) __mk_vma_flags(flags, \ COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__})
/* Calculates the number of set bits in the specified VMA flags. */static __always_inline int vma_flags_count(const vma_flags_t *flags){ const unsigned long *bitmap = flags->__vma_flags;
return bitmap_weight(bitmap, NUM_VMA_FLAG_BITS);}
/* * Test whether a specific VMA flag is set, e.g.: * * if (vma_flags_test(flags, VMA_READ_BIT)) { ... } */static __always_inline bool vma_flags_test(const vma_flags_t *flags, vma_flag_t bit){ const unsigned long *bitmap = flags->__vma_flags;
return test_bit((__force int)bit, bitmap);}
/* * Obtain a set of VMA flags which contain the overlapping flags contained * within flags and to_and. */static __always_inline vma_flags_t vma_flags_and_mask(const vma_flags_t *flags, vma_flags_t to_and){ vma_flags_t dst; unsigned long *bitmap_dst = dst.__vma_flags; const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_and = to_and.__vma_flags;
bitmap_and(bitmap_dst, bitmap, bitmap_to_and, NUM_VMA_FLAG_BITS); return dst;}
/* * Obtain a set of VMA flags which contains the specified overlapping flags, * e.g.: * * vma_flags_t read_flags = vma_flags_and(&flags, VMA_READ_BIT, * VMA_MAY_READ_BIT); */#define vma_flags_and(flags, ...) \ vma_flags_and_mask(flags, mk_vma_flags(__VA_ARGS__))
/* Test each of to_test flags in flags, non-atomically. */static __always_inline bool vma_flags_test_any_mask(const vma_flags_t *flags, vma_flags_t to_test){ const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_test = to_test.__vma_flags;
return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);}
/* * Test whether any specified VMA flag is set, e.g.: * * if (vma_flags_test_any(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } */#define vma_flags_test_any(flags, ...) \ vma_flags_test_any_mask(flags, mk_vma_flags(__VA_ARGS__))
/* Test that ALL of the to_test flags are set, non-atomically. */static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags, vma_flags_t to_test){ const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_test = to_test.__vma_flags;
return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);}
/* * Test whether ALL specified VMA flags are set, e.g.: * * if (vma_flags_test_all(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } */#define vma_flags_test_all(flags, ...) \ vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__))
/* * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set * (returning false if flagmask has no flags set). * * This is defined to make the semantics clearer when testing an optionally * defined VMA flags mask, e.g.: * * if (vma_flags_test_single_mask(&flags, VMA_DROPPABLE)) { ... } * * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS * otherwise. */static __always_inline bool vma_flags_test_single_mask(const vma_flags_t *flags, vma_flags_t flagmask){ VM_WARN_ON_ONCE(vma_flags_count(&flagmask) > 1);
return vma_flags_test_any_mask(flags, flagmask);}
/* Set each of the to_set flags in flags, non-atomically. */static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set){ unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_set = to_set.__vma_flags;
bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS);}
/* * Set all specified VMA flags, e.g.: * * vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT); */#define vma_flags_set(flags, ...) \ vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__))
static __always_inline vma_flags_t __mk_vma_flags_from_masks(size_t count, const vma_flags_t *masks){ vma_flags_t flags = EMPTY_VMA_FLAGS; size_t i;
for (i = 0; i < count; i++) vma_flags_set_mask(&flags, masks[i]); return flags;}
/* * Combine pre-computed vma_flags_t masks into one value, e.g.: * * vma_flags_t flags = mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR); * * Unlike mk_vma_flags(), which takes bit numbers, this takes whole masks -- * each of which may be EMPTY_VMA_FLAGS when its feature is unavailable -- so a * bit that does not exist on the current build is never materialised. */#define mk_vma_flags_from_masks(...) \ __mk_vma_flags_from_masks(COUNT_ARGS(__VA_ARGS__), \ (const vma_flags_t []){__VA_ARGS__})
/* Clear all of the to-clear flags in flags, non-atomically. */static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear){ unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_clear = to_clear.__vma_flags;
bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS);}
/* * Clear all specified individual flags, e.g.: * * vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT); */#define vma_flags_clear(flags, ...) \ vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__))
/* * Obtain a VMA flags value containing those flags that are present in flags or * flags_other but not in both. */static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, const vma_flags_t *flags_other){ vma_flags_t dst; const unsigned long *bitmap_other = flags_other->__vma_flags; const unsigned long *bitmap = flags->__vma_flags; unsigned long *bitmap_dst = dst.__vma_flags;
bitmap_xor(bitmap_dst, bitmap, bitmap_other, NUM_VMA_FLAG_BITS); return dst;}
/* Determine if flags and flags_other have precisely the same flags set. */static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, const vma_flags_t *flags_other){ const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_other = flags_other->__vma_flags;
return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS);}
/* Determine if flags and flags_other have precisely the same flags set. */static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, vma_flags_t flags_other){ const unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_other = flags_other.__vma_flags;
return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS);}
/* * Helper macro to determine if only the specific flags are set, e.g.: * * if (vma_flags_same(&flags, VMA_WRITE_BIT) { ... } */#define vma_flags_same(flags, ...) \ vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__))
/* * Test whether a specific flag in the VMA is set, e.g.: * * if (vma_test(vma, VMA_READ_BIT)) { ... } */static __always_inline bool vma_test(const struct vm_area_struct *vma, vma_flag_t bit){ return vma_flags_test(&vma->flags, bit);}
/* Helper to test any VMA flags in a VMA . */static __always_inline bool vma_test_any_mask(const struct vm_area_struct *vma, vma_flags_t flags){ return vma_flags_test_any_mask(&vma->flags, flags);}
/* * Helper macro for testing whether any VMA flags are set in a VMA, * e.g.: * * if (vma_test_any(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } */#define vma_test_any(vma, ...) \ vma_test_any_mask(vma, mk_vma_flags(__VA_ARGS__))
/* * Helper to test that ALL specified flags are set in a VMA. * * Note: appropriate locks must be held, this function does not acquire them for * you. */static __always_inline bool vma_test_all_mask(const struct vm_area_struct *vma, vma_flags_t flags){ return vma_flags_test_all_mask(&vma->flags, flags);}
/* * Helper macro for checking that ALL specified flags are set in a VMA, e.g.: * * if (vma_test_all(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... } */#define vma_test_all(vma, ...) \ vma_test_all_mask(vma, mk_vma_flags(__VA_ARGS__))
/* * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set * (returning false if flagmask has no flags set). * * This is useful when a flag needs to be either defined or not depending upon * kernel configuration, e.g.: * * if (vma_test_single_mask(vma, VMA_DROPPABLE)) { ... } * * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS * otherwise. */static __always_inline boolvma_test_single_mask(const struct vm_area_struct *vma, vma_flags_t flagmask){ return vma_flags_test_single_mask(&vma->flags, flagmask);}
/* * Helper to set all VMA flags in a VMA. * * Note: appropriate locks must be held, this function does not acquire them for * you. */static __always_inline void vma_set_flags_mask(struct vm_area_struct *vma, vma_flags_t flags){ vma_flags_set_mask(&vma->flags, flags);}
/* * Helper macro for specifying VMA flags in a VMA, e.g.: * * vma_set_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, * VMA_DONTDUMP_BIT); * * Note: appropriate locks must be held, this function does not acquire them for * you. */#define vma_set_flags(vma, ...) \ vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
/* Helper to clear all VMA flags in a VMA. */static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, vma_flags_t flags){ vma_flags_clear_mask(&vma->flags, flags);}
/* * Helper macro for clearing VMA flags, e.g.: * * vma_clear_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, * VMA_DONTDUMP_BIT); */#define vma_clear_flags(vma, ...) \ vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
/* * Test whether a specific VMA flag is set in a VMA descriptor, e.g.: * * if (vma_desc_test(desc, VMA_READ_BIT)) { ... } */static __always_inline bool vma_desc_test(const struct vm_area_desc *desc, vma_flag_t bit){ return vma_flags_test(&desc->vma_flags, bit);}
/* Helper to test any VMA flags in a VMA descriptor. */static __always_inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc, vma_flags_t flags){ return vma_flags_test_any_mask(&desc->vma_flags, flags);}
/* * Helper macro for testing whether any VMA flags are set in a VMA descriptor, * e.g.: * * if (vma_desc_test_any(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } */#define vma_desc_test_any(desc, ...) \ vma_desc_test_any_mask(desc, mk_vma_flags(__VA_ARGS__))
/* Helper to test all VMA flags in a VMA descriptor. */static __always_inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc, vma_flags_t flags){ return vma_flags_test_all_mask(&desc->vma_flags, flags);}
/* * Helper macro for testing whether ALL VMA flags are set in a VMA descriptor, * e.g.: * * if (vma_desc_test_all(desc, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } */#define vma_desc_test_all(desc, ...) \ vma_desc_test_all_mask(desc, mk_vma_flags(__VA_ARGS__))
/* Helper to set all VMA flags in a VMA descriptor. */static __always_inline void vma_desc_set_flags_mask(struct vm_area_desc *desc, vma_flags_t flags){ vma_flags_set_mask(&desc->vma_flags, flags);}
/* * Helper macro for specifying VMA flags for an input pointer to a struct * vm_area_desc object describing a proposed VMA, e.g.: * * vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, * VMA_DONTDUMP_BIT); */#define vma_desc_set_flags(desc, ...) \ vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
/* Helper to clear all VMA flags in a VMA descriptor. */static __always_inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc, vma_flags_t flags){ vma_flags_clear_mask(&desc->vma_flags, flags);}
/* * Helper macro for clearing VMA flags for an input pointer to a struct * vm_area_desc object describing a proposed VMA, e.g.: * * vma_desc_clear_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, * VMA_DONTDUMP_BIT); */#define vma_desc_clear_flags(desc, ...) \ vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
static inline void vma_set_anonymous(struct vm_area_struct *vma){ vma->vm_ops = NULL;}
static inline void vma_desc_set_anonymous(struct vm_area_desc *desc){ desc->vm_ops = NULL;}
static inline bool vma_is_anonymous(const struct vm_area_struct *vma){ return !vma->vm_ops;}
/* * Indicate if the VMA is a heap for the given task; for * /proc/PID/maps that is the heap of the main task. */static inline bool vma_is_initial_heap(const struct vm_area_struct *vma){ return vma->vm_start < vma->vm_mm->brk && vma->vm_end > vma->vm_mm->start_brk;}
/* * Indicate if the VMA is a stack for the given task; for * /proc/PID/maps that is the stack of the main task. */static inline bool vma_is_initial_stack(const struct vm_area_struct *vma){ /* * We make no effort to guess what a given thread considers to be * its "stack". It's not even well-defined for programs written * languages like Go. */ return vma->vm_start <= vma->vm_mm->start_stack && vma->vm_end >= vma->vm_mm->start_stack;}
static inline bool vma_flags_can_grow(const vma_flags_t *flags){ if (vma_flags_test_single_mask(flags, VMA_GROWSUP)) return true; if (vma_flags_test(flags, VMA_GROWSDOWN_BIT)) return true;
return false;}
static inline bool vma_can_grow(const struct vm_area_struct *vma){ return vma_flags_can_grow(&vma->flags);}
static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma){ if (!vma_can_grow(vma)) return false;
if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == VM_STACK_INCOMPLETE_SETUP) return true;
return false;}
static inline bool vma_is_foreign(const struct vm_area_struct *vma){ if (!current->mm) return true;
if (current->mm != vma->vm_mm) return true;
return false;}
static inline bool vma_is_accessible(const struct vm_area_struct *vma){ return vma->vm_flags & VM_ACCESS_FLAGS;}
static inline bool is_shared_maywrite(const vma_flags_t *flags){ return vma_flags_test_all(flags, VMA_SHARED_BIT, VMA_MAYWRITE_BIT);}
static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma){ return is_shared_maywrite(&vma->flags);}
/** * vma_kernel_pagesize - Default page size granularity for this VMA. * @vma: The user mapping. * * The kernel page size specifies in which granularity VMA modifications * can be performed. Folios in this VMA will be aligned to, and at least * the size of the number of bytes returned by this function. * * The default kernel page size is not affected by Transparent Huge Pages * being in effect. * * Return: The default page size granularity for this VMA. */static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma){ if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) return vma->vm_ops->pagesize(vma); return PAGE_SIZE;}
unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
static inlinestruct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max){ return mas_find(&vmi->mas, max - 1);}
static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi){ /* * Uses mas_find() to get the first VMA when the iterator starts. * Calling mas_next() could skip the first entry. */ return mas_find(&vmi->mas, ULONG_MAX);}
static inlinestruct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi){ return mas_next_range(&vmi->mas, ULONG_MAX);}
static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi){ return mas_prev(&vmi->mas, 0);}
static inline int vma_iter_clear_gfp(struct vma_iterator *vmi, unsigned long start, unsigned long end, gfp_t gfp){ __mas_set_range(&vmi->mas, start, end - 1); mas_store_gfp(&vmi->mas, NULL, gfp); if (unlikely(mas_is_err(&vmi->mas))) return -ENOMEM;
return 0;}
/* Free any unused preallocations */static inline void vma_iter_free(struct vma_iterator *vmi){ mas_destroy(&vmi->mas);}
static inline int vma_iter_bulk_store(struct vma_iterator *vmi, struct vm_area_struct *vma){ vmi->mas.index = vma->vm_start; vmi->mas.last = vma->vm_end - 1; mas_store(&vmi->mas, vma); if (unlikely(mas_is_err(&vmi->mas))) return -ENOMEM;
vma_mark_attached(vma); return 0;}
static inline void vma_iter_invalidate(struct vma_iterator *vmi){ mas_pause(&vmi->mas);}
static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr){ mas_set(&vmi->mas, addr);}
#define for_each_vma(__vmi, __vma) \ while (((__vma) = vma_next(&(__vmi))) != NULL)
/* The MM code likes to work with exclusive end addresses */#define for_each_vma_range(__vmi, __vma, __end) \ while (((__vma) = vma_find(&(__vmi), (__end))) != NULL)
#ifdef CONFIG_SHMEM/* * The vma_is_shmem is not inline because it is used only by slow * paths in userfault. */bool vma_is_shmem(const struct vm_area_struct *vma);bool vma_is_anon_shmem(const struct vm_area_struct *vma);#elsestatic inline bool vma_is_shmem(const struct vm_area_struct *vma) { return false; }static inline bool vma_is_anon_shmem(const struct vm_area_struct *vma) { return false; }#endif
int vma_is_stack_for_current(const struct vm_area_struct *vma);
/* flush_tlb_range() takes a vma, not a mm, and can care about flags */#define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) }
struct mmu_gather;struct inode;
extern void prep_compound_page(struct page *page, unsigned int order);
static inline unsigned int folio_large_order(const struct folio *folio){ return folio->_flags_1 & 0xff;}
#ifdef NR_PAGES_IN_LARGE_FOLIOstatic inline unsigned long folio_large_nr_pages(const struct folio *folio){ return folio->_nr_pages;}#elsestatic inline unsigned long folio_large_nr_pages(const struct folio *folio){ return 1L << folio_large_order(folio);}#endif
/* * compound_order() can be called without holding a reference, which means * that niceties like page_folio() don't work. These callers should be * prepared to handle wild return values. For example, PG_head may be * set before the order is initialised, or this may be a tail page. * See compaction.c for some good examples. */static inline unsigned int compound_order(const struct page *page){ const struct folio *folio = (struct folio *)page;
if (!test_bit(PG_head, &folio->flags.f)) return 0; return folio_large_order(folio);}
/** * folio_order - The allocation order of a folio. * @folio: The folio. * * A folio is composed of 2^order pages. See get_order() for the definition * of order. * * Return: The order of the folio. */static inline unsigned int folio_order(const struct folio *folio){ if (!folio_test_large(folio)) return 0; return folio_large_order(folio);}
/** * folio_reset_order - Reset the folio order and derived _nr_pages * @folio: The folio. * * Reset the order and derived _nr_pages to 0. Must only be used in the * process of splitting large folios. */static inline void folio_reset_order(struct folio *folio){ if (WARN_ON_ONCE(!folio_test_large(folio))) return; folio->_flags_1 &= ~0xffUL;#ifdef NR_PAGES_IN_LARGE_FOLIO folio->_nr_pages = 0;#endif}
#include <linux/huge_mm.h>
/* * Methods to modify the page usage count. * * What counts for a page usage: * - cache mapping (page->mapping) * - private data (page->private) * - page mapped in a task's page tables, each mapping * is counted separately * * Also, many kernel routines increase the page count before a critical * routine so they can be sure the page doesn't go away from under them. */
/* * Drop a ref, return true if the refcount fell to zero (the page has no users) */static inline int put_page_testzero(struct page *page){ VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); return page_ref_dec_and_test(page);}
static inline int folio_put_testzero(struct folio *folio){ return put_page_testzero(&folio->page);}
/* * Try to grab a ref unless the page has a refcount of zero, return false if * that is the case. * This can be called when MMU is off so it must not access * any of the virtual mappings. */static inline bool get_page_unless_zero(struct page *page){ return page_ref_add_unless_zero(page, 1);}
static inline struct folio *folio_get_nontail_page(struct page *page){ if (unlikely(!get_page_unless_zero(page))) return NULL; return (struct folio *)page;}
extern int page_is_ram(unsigned long pfn);
enum { REGION_INTERSECTS, REGION_DISJOINT, REGION_MIXED,};
int region_intersects(resource_size_t offset, size_t size, unsigned long flags, unsigned long desc);
/* Support for virtually mapped pages */struct page *vmalloc_to_page(const void *addr);unsigned long vmalloc_to_pfn(const void *addr);
/* * Determine if an address is within the vmalloc range * * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there * is no special casing required. */#ifdef CONFIG_MMUextern bool is_vmalloc_addr(const void *x);extern int is_vmalloc_or_module_addr(const void *x);#elsestatic inline bool is_vmalloc_addr(const void *x){ return false;}static inline int is_vmalloc_or_module_addr(const void *x){ return 0;}#endif
/* * How many times the entire folio is mapped as a single unit (eg by a * PMD or PUD entry). This is probably not what you want, except for * debugging purposes or implementation of other core folio_*() primitives. */static inline int folio_entire_mapcount(const struct folio *folio){ VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); if (!IS_ENABLED(CONFIG_64BIT) && unlikely(folio_large_order(folio) == 1)) return 0; return atomic_read(&folio->_entire_mapcount) + 1;}
static inline int folio_large_mapcount(const struct folio *folio){ VM_WARN_ON_FOLIO(!folio_test_large(folio), folio); return atomic_read(&folio->_large_mapcount) + 1;}
/** * folio_mapcount() - Number of mappings of this folio. * @folio: The folio. * * The folio mapcount corresponds to the number of present user page table * entries that reference any part of a folio. Each such present user page * table entry must be paired with exactly on folio reference. * * For ordindary folios, each user page table entry (PTE/PMD/PUD/...) counts * exactly once. * * For hugetlb folios, each abstracted "hugetlb" user page table entry that * references the entire folio counts exactly once, even when such special * page table entries are comprised of multiple ordinary page table entries. * * Will report 0 for pages which cannot be mapped into userspace, such as * slab, page tables and similar. * * Return: The number of times this folio is mapped. */static inline int folio_mapcount(const struct folio *folio){ int mapcount;
if (likely(!folio_test_large(folio))) { mapcount = atomic_read(&folio->_mapcount) + 1; if (page_mapcount_is_type(mapcount)) mapcount = 0; return mapcount; } return folio_large_mapcount(folio);}
/** * folio_mapped - Is this folio mapped into userspace? * @folio: The folio. * * Return: True if any page in this folio is referenced by user page tables. */static inline bool folio_mapped(const struct folio *folio){ return folio_mapcount(folio) >= 1;}
static inline struct page *virt_to_head_page(const void *x){ struct page *page = virt_to_page(x);
return compound_head(page);}
static inline struct folio *virt_to_folio(const void *x){ struct page *page = virt_to_page(x);
return page_folio(page);}
void __folio_put(struct folio *folio);
void split_page(struct page *page, unsigned int order);void folio_copy(struct folio *dst, struct folio *src);int folio_mc_copy(struct folio *dst, struct folio *src);
unsigned long nr_free_buffer_pages(void);
/* Returns the number of bytes in this potentially compound page. */static inline unsigned long page_size(const struct page *page){ return PAGE_SIZE << compound_order(page);}
/* Returns the number of bits needed for the number of bytes in a page */static inline unsigned int page_shift(struct page *page){ return PAGE_SHIFT + compound_order(page);}
/** * thp_order - Order of a transparent huge page. * @page: Head page of a transparent huge page. */static inline unsigned int thp_order(struct page *page){ VM_BUG_ON_PGFLAGS(PageTail(page), page); return compound_order(page);}
/** * thp_size - Size of a transparent huge page. * @page: Head page of a transparent huge page. * * Return: Number of bytes in this page. */static inline unsigned long thp_size(struct page *page){ return PAGE_SIZE << thp_order(page);}
#ifdef CONFIG_MMU/* * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when * servicing faults for write access. In the normal case, do always want * pte_mkwrite. But get_user_pages can cause write faults for mappings * that do not have writing enabled, when used by access_process_vm. */static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma){ if (likely(vma->vm_flags & VM_WRITE)) pte = pte_mkwrite(pte, vma); return pte;}
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page);void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr);
vm_fault_t finish_fault(struct vm_fault *vmf);#endif
/* * Multiple processes may "see" the same page. E.g. for untouched * mappings of /dev/null, all processes see the same page full of * zeroes, and text pages of executables and shared libraries have * only one copy in memory, at most, normally. * * For the non-reserved pages, page_count(page) denotes a reference count. * page_count() == 0 means the page is free. page->lru is then used for * freelist management in the buddy allocator. * page_count() > 0 means the page has been allocated. * * Pages are allocated by the slab allocator in order to provide memory * to kmalloc and kmem_cache_alloc. In this case, the management of the * page, and the fields in 'struct page' are the responsibility of mm/slab.c * unless a particular usage is carefully commented. (the responsibility of * freeing the kmalloc memory is the caller's, of course). * * A page may be used by anyone else who does a __get_free_page(). * In this case, page_count still tracks the references, and should only * be used through the normal accessor functions. The top bits of page->flags * and page->virtual store page management information, but all other fields * are unused and could be used privately, carefully. The management of this * page is the responsibility of the one who allocated it, and those who have * subsequently been given references to it. * * The other pages (we may call them "pagecache pages") are completely * managed by the Linux memory manager: I/O, buffers, swapping etc. * The following discussion applies only to them. * * A pagecache page contains an opaque `private' member, which belongs to the * page's address_space. Usually, this is the address of a circular list of * the page's disk buffers. PG_private must be set to tell the VM to call * into the filesystem to release these pages. * * A folio may belong to an inode's memory mapping. In this case, * folio->mapping points to the inode, and folio->index is the file * offset of the folio, in units of PAGE_SIZE. * * If pagecache pages are not associated with an inode, they are said to be * anonymous pages. These may become associated with the swapcache, and in that * case PG_swapcache is set, and page->private is an offset into the swapcache. * * In either case (swapcache or inode backed), the pagecache itself holds one * reference to the page. Setting PG_private should also increment the * refcount. The each user mapping also has a reference to the page. * * The pagecache pages are stored in a per-mapping radix tree, which is * rooted at mapping->i_pages, and indexed by offset. * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space * lists, we instead now tag pages as dirty/writeback in the radix tree. * * All pagecache pages may be subject to I/O: * - inode pages may need to be read from disk, * - inode pages which have been modified and are MAP_SHARED may need * to be written back to the inode on disk, * - anonymous pages (including MAP_PRIVATE file mappings) which have been * modified may need to be swapped out to swap space and (later) to be read * back into memory. */
/* 127: arbitrary random number, small enough to assemble well */#define folio_ref_zero_or_close_to_overflow(folio) \ ((unsigned int) folio_ref_count(folio) + 127u <= 127u)
/** * folio_get - Increment the reference count on a folio. * @folio: The folio. * * Context: May be called in any context, as long as you know that * you have a refcount on the folio. If you do not already have one, * folio_try_get() may be the right interface for you to use. */static inline void folio_get(struct folio *folio){ VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio); folio_ref_inc(folio);}
static inline void get_page(struct page *page){ struct folio *folio = page_folio(page); if (WARN_ON_ONCE(folio_test_slab(folio))) return; if (WARN_ON_ONCE(folio_test_large_kmalloc(folio))) return; folio_get(folio);}
static inline __must_check bool try_get_page(struct page *page){ page = compound_head(page); if (WARN_ON_ONCE(page_ref_count(page) <= 0)) return false; page_ref_inc(page); return true;}
/** * folio_put - Decrement the reference count on a folio. * @folio: The folio. * * If the folio's reference count reaches zero, the memory will be * released back to the page allocator and may be used by another * allocation immediately. Do not access the memory or the struct folio * after calling folio_put() unless you can be sure that it wasn't the * last reference. * * Context: May be called in process or interrupt context, but not in NMI * context. May be called while holding a spinlock. */static inline void folio_put(struct folio *folio){ if (folio_put_testzero(folio)) __folio_put(folio);}
/** * folio_put_refs - Reduce the reference count on a folio. * @folio: The folio. * @refs: The amount to subtract from the folio's reference count. * * If the folio's reference count reaches zero, the memory will be * released back to the page allocator and may be used by another * allocation immediately. Do not access the memory or the struct folio * after calling folio_put_refs() unless you can be sure that these weren't * the last references. * * Context: May be called in process or interrupt context, but not in NMI * context. May be called while holding a spinlock. */static inline void folio_put_refs(struct folio *folio, int refs){ if (folio_ref_sub_and_test(folio, refs)) __folio_put(folio);}
void folios_put_refs(struct folio_batch *folios, unsigned int *refs);
/* * union release_pages_arg - an array of pages or folios * * release_pages() releases a simple array of multiple pages, and * accepts various different forms of said page array: either * a regular old boring array of pages, an array of folios, or * an array of encoded page pointers. * * The transparent union syntax for this kind of "any of these * argument types" is all kinds of ugly, so look away. */typedef union { struct page **pages; struct folio **folios; struct encoded_page **encoded_pages;} release_pages_arg __attribute__ ((__transparent_union__));
void release_pages(release_pages_arg, int nr);
/** * folios_put - Decrement the reference count on an array of folios. * @folios: The folios. * * Like folio_put(), but for a batch of folios. This is more efficient * than writing the loop yourself as it will optimise the locks which need * to be taken if the folios are freed. The folios batch is returned * empty and ready to be reused for another batch; there is no need to * reinitialise it. * * Context: May be called in process or interrupt context, but not in NMI * context. May be called while holding a spinlock. */static inline void folios_put(struct folio_batch *folios){ folios_put_refs(folios, NULL);}
static inline void put_page(struct page *page){ struct folio *folio = page_folio(page);
if (folio_test_slab(folio) || folio_test_large_kmalloc(folio)) return;
folio_put(folio);}
/* * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload * the page's refcount so that two separate items are tracked: the original page * reference count, and also a new count of how many pin_user_pages() calls were * made against the page. ("gup-pinned" is another term for the latter). * * With this scheme, pin_user_pages() becomes special: such pages are marked as * distinct from normal pages. As such, the unpin_user_page() call (and its * variants) must be used in order to release gup-pinned pages. * * Choice of value: * * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference * counts with respect to pin_user_pages() and unpin_user_page() becomes * simpler, due to the fact that adding an even power of two to the page * refcount has the effect of using only the upper N bits, for the code that * counts up using the bias value. This means that the lower bits are left for * the exclusive use of the original code that increments and decrements by one * (or at least, by much smaller values than the bias value). * * Of course, once the lower bits overflow into the upper bits (and this is * OK, because subtraction recovers the original values), then visual inspection * no longer suffices to directly view the separate counts. However, for normal * applications that don't have huge page reference counts, this won't be an * issue. * * Locking: the lockless algorithm described in folio_try_get_rcu() * provides safe operation for get_user_pages(), folio_mkclean() and * other calls that race to set up page table entries. */#define GUP_PIN_COUNTING_BIAS (1U << 10)
void unpin_user_page(struct page *page);void unpin_folio(struct folio *folio);void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, bool make_dirty);void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, bool make_dirty);void unpin_user_pages(struct page **pages, unsigned long npages);void unpin_user_folio(struct folio *folio, unsigned long npages);void unpin_folios(struct folio **folios, unsigned long nfolios);
/** * vma_flags_is_cow_mapping() - Do these VMA flags imply a CoW mapping? * @flags: The VMA flags to check. * * Mappings which could be CoW'd (subject to Copy-On-Write faults) are * described as CoW mappings. * * All mappings backed by anonymous folios (all anonymous mappings and most * MAP_PRIVATE-file backed ranges) are CoW mappings. * * All other mappings (including all MAP_SHARED mappings) are non-CoW. * * The criteria are !VMA_SHARED_BIT, VMA_MAYWRITE_BIT. * * VMA_MAYWRITE_BIT is checked instead of VMA_WRITE_BIT to account for both * future mprotect() calls which can render a read-only mapping writable, and * GUP with FOLL_FORCE (e.g. ptrace) which can CoW a read-only mapping. * * - No anonymous mapping can ever clear VMA_MAYWRITE_BIT. * * - Writes to anonymous mappings do not immediately result in CoW faults but * may do so after the process is forked or if a read is followed by a * write. * * - Writes to MAP_PRIVATE file-backed mappings result in CoW faults and may * do so again after fork. * * - MAP_SHARED mappings of a file opened read-only are transformed into * VMA_MAYSHARE_BIT, !VMA_SHARED_BIT, !VMA_MAYWRITE_BIT mappings, so remain * non-CoW. * * - Drivers may clear VMA_MAYWRITE_BIT but do so at mmap() time and cannot * mark themselves anonymous. Having cleared this flag it is not valid for * them to leave the VMA_WRITE_BIT flag set. * * As a consequence, the anonymous reverse mapping only tracks CoW mappings. * * Returns: true if the flags indicate a CoW mapping, otherwise false. */static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags){ return vma_flags_test(flags, VMA_MAYWRITE_BIT) && !vma_flags_test(flags, VMA_SHARED_BIT);}
/** * vma_is_cow_mapping() - Is this VMA a CoW mapping? * @vma: The VMA to check. * * See vma_flags_is_cow_mapping() for details. * * Returns: true if the VMA is a CoW mapping, otherwise false. */static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma){ return vma_flags_is_cow_mapping(&vma->flags);}
/** * vma_desc_is_cow_mapping() - Is this VMA descriptor a CoW mapping? * @desc: The VMA descriptor to check. * * See vma_flags_is_cow_mapping() for details. * * Returns: true if the VMA descriptor describes a CoW mapping, otherwise * false. */static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc){ return vma_flags_is_cow_mapping(&desc->vma_flags);}
#ifndef CONFIG_MMUstatic inline bool is_nommu_shared_mapping(vm_flags_t flags){ /* * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of * a file mapping. R/O MAP_PRIVATE mappings might still modify * underlying memory if ptrace is active, so this is only possible if * ptrace does not apply. Note that there is no mprotect() to upgrade * write permissions later. */ return flags & (VM_MAYSHARE | VM_MAYOVERLAY);}
static inline bool is_nommu_shared_vma_flags(const vma_flags_t *flags){ return vma_flags_test_any(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT);}#endif
#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)#define SECTION_IN_PAGE_FLAGS#endif
/* * The identification function is mainly used by the buddy allocator for * determining if two pages could be buddies. We are not really identifying * the zone since we could be using the section number id if we do not have * node id available in page flags. * We only guarantee that it will return the same value for two combinable * pages in a zone. */static inline int page_zone_id(struct page *page){ return (page->flags.f >> ZONEID_PGSHIFT) & ZONEID_MASK;}
#ifdef NODE_NOT_IN_PAGE_FLAGSint memdesc_nid(const memdesc_flags_t *mdf);#else#ifdef CONFIG_NUMAstatic inline int memdesc_nid(const memdesc_flags_t *mdf){ ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT); return (mdf->f >> NODES_PGSHIFT) & NODES_MASK;}#elsestatic inline int memdesc_nid(const memdesc_flags_t *mdf){ return 0;}#endif#endif
static inline int page_to_nid(const struct page *page){ return memdesc_nid(&(PF_POISONED_CHECK(page)->flags));}
static inline int folio_nid(const struct folio *folio){ return memdesc_nid(&folio->flags);}
#ifdef CONFIG_NUMA_BALANCING/* page access time bits needs to hold at least 4 seconds */#define PAGE_ACCESS_TIME_MIN_BITS 12#if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS#define PAGE_ACCESS_TIME_BUCKETS \ (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT)#else#define PAGE_ACCESS_TIME_BUCKETS 0#endif
#define PAGE_ACCESS_TIME_MASK \ (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS)
static inline int cpu_pid_to_cpupid(int cpu, int pid){ return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);}
static inline int cpupid_to_pid(int cpupid){ return cpupid & LAST__PID_MASK;}
static inline int cpupid_to_cpu(int cpupid){ return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;}
static inline int cpupid_to_nid(int cpupid){ return cpu_to_node(cpupid_to_cpu(cpupid));}
static inline bool cpupid_pid_unset(int cpupid){ return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);}
static inline bool cpupid_cpu_unset(int cpupid){ return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);}
static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid){ return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);}
#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGSstatic inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid){ return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK);}
static inline int folio_last_cpupid(struct folio *folio){ return folio->_last_cpupid;}static inline void page_cpupid_reset_last(struct page *page){ page->_last_cpupid = -1 & LAST_CPUPID_MASK;}#elsestatic inline int folio_last_cpupid(struct folio *folio){ return (folio->flags.f >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;}
int folio_xchg_last_cpupid(struct folio *folio, int cpupid);
static inline void page_cpupid_reset_last(struct page *page){ page->flags.f |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;}#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
static inline int folio_xchg_access_time(struct folio *folio, int time){ int last_time;
last_time = folio_xchg_last_cpupid(folio, time >> PAGE_ACCESS_TIME_BUCKETS); return last_time << PAGE_ACCESS_TIME_BUCKETS;}
static inline void vma_set_access_pid_bit(struct vm_area_struct *vma){ unsigned int pid_bit;
pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG)); if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) { __set_bit(pid_bit, &vma->numab_state->pids_active[1]); }}
bool folio_use_access_time(struct folio *folio);#else /* !CONFIG_NUMA_BALANCING */static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid){ return folio_nid(folio); /* XXX */}
static inline int folio_xchg_access_time(struct folio *folio, int time){ return 0;}
static inline int folio_last_cpupid(struct folio *folio){ return folio_nid(folio); /* XXX */}
static inline int cpupid_to_nid(int cpupid){ return -1;}
static inline int cpupid_to_pid(int cpupid){ return -1;}
static inline int cpupid_to_cpu(int cpupid){ return -1;}
static inline int cpu_pid_to_cpupid(int nid, int pid){ return -1;}
static inline bool cpupid_pid_unset(int cpupid){ return true;}
static inline void page_cpupid_reset_last(struct page *page){}
static inline bool cpupid_match_pid(struct task_struct *task, int cpupid){ return false;}
static inline void vma_set_access_pid_bit(struct vm_area_struct *vma){}static inline bool folio_use_access_time(struct folio *folio){ return false;}#endif /* CONFIG_NUMA_BALANCING */
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
/* * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid * setting tags for all pages to native kernel tag value 0xff, as the default * value 0x00 maps to 0xff. */
static inline u8 page_kasan_tag(const struct page *page){ u8 tag = KASAN_TAG_KERNEL;
if (kasan_enabled()) { tag = (page->flags.f >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; tag ^= 0xff; }
return tag;}
static inline void page_kasan_tag_set(struct page *page, u8 tag){ unsigned long old_flags, flags;
if (!kasan_enabled()) return;
tag ^= 0xff; old_flags = READ_ONCE(page->flags.f); do { flags = old_flags; flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; } while (unlikely(!try_cmpxchg(&page->flags.f, &old_flags, flags)));}
static inline void page_kasan_tag_reset(struct page *page){ if (kasan_enabled()) page_kasan_tag_set(page, KASAN_TAG_KERNEL);}
#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline u8 page_kasan_tag(const struct page *page){ return 0xff;}
static inline void page_kasan_tag_set(struct page *page, u8 tag) { }static inline void page_kasan_tag_reset(struct page *page) { }
#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
static inline struct zone *page_zone(const struct page *page){ return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];}
static inline pg_data_t *page_pgdat(const struct page *page){ return NODE_DATA(page_to_nid(page));}
static inline pg_data_t *folio_pgdat(const struct folio *folio){ return NODE_DATA(folio_nid(folio));}
static inline struct zone *folio_zone(const struct folio *folio){ return &folio_pgdat(folio)->node_zones[folio_zonenum(folio)];}
#ifdef SECTION_IN_PAGE_FLAGSstatic inline void set_page_section(struct page *page, unsigned long section){ page->flags.f &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;}
static inline unsigned long memdesc_section(const memdesc_flags_t *mdf){ ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK;}#else /* !SECTION_IN_PAGE_FLAGS */static inline unsigned long memdesc_section(const memdesc_flags_t *mdf){ return 0;}#endif /* SECTION_IN_PAGE_FLAGS */
/** * folio_pfn - Return the Page Frame Number of a folio. * @folio: The folio. * * A folio may contain multiple pages. The pages have consecutive * Page Frame Numbers. * * Return: The Page Frame Number of the first page in the folio. */static inline unsigned long folio_pfn(const struct folio *folio){ return page_to_pfn(&folio->page);}
static inline struct folio *pfn_folio(unsigned long pfn){ return page_folio(pfn_to_page(pfn));}
#ifdef CONFIG_MMUstatic inline pte_t mk_pte(const struct page *page, pgprot_t pgprot){ return pfn_pte(page_to_pfn(page), pgprot);}
/** * folio_mk_pte - Create a PTE for this folio * @folio: The folio to create a PTE for * @pgprot: The page protection bits to use * * Create a page table entry for the first page of this folio. * This is suitable for passing to set_ptes(). * * Return: A page table entry suitable for mapping this folio. */static inline pte_t folio_mk_pte(const struct folio *folio, pgprot_t pgprot){ return pfn_pte(folio_pfn(folio), pgprot);}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE/** * folio_mk_pmd - Create a PMD for this folio * @folio: The folio to create a PMD for * @pgprot: The page protection bits to use * * Create a page table entry for the first page of this folio. * This is suitable for passing to set_pmd_at(). * * Return: A page table entry suitable for mapping this folio. */static inline pmd_t folio_mk_pmd(const struct folio *folio, pgprot_t pgprot){ return pmd_mkhuge(pfn_pmd(folio_pfn(folio), pgprot));}
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD/** * folio_mk_pud - Create a PUD for this folio * @folio: The folio to create a PUD for * @pgprot: The page protection bits to use * * Create a page table entry for the first page of this folio. * This is suitable for passing to set_pud_at(). * * Return: A page table entry suitable for mapping this folio. */static inline pud_t folio_mk_pud(const struct folio *folio, pgprot_t pgprot){ return pud_mkhuge(pfn_pud(folio_pfn(folio), pgprot));}#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */#endif /* CONFIG_TRANSPARENT_HUGEPAGE */#endif /* CONFIG_MMU */
static inline bool folio_has_pincount(const struct folio *folio){ if (IS_ENABLED(CONFIG_64BIT)) return folio_test_large(folio); return folio_order(folio) > 1;}
/** * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA. * @folio: The folio. * * This function checks if a folio has been pinned via a call to * a function in the pin_user_pages() family. * * For small folios, the return value is partially fuzzy: false is not fuzzy, * because it means "definitely not pinned for DMA", but true means "probably * pinned for DMA, but possibly a false positive due to having at least * GUP_PIN_COUNTING_BIAS worth of normal folio references". * * False positives are OK, because: a) it's unlikely for a folio to * get that many refcounts, and b) all the callers of this routine are * expected to be able to deal gracefully with a false positive. * * For most large folios, the result will be exactly correct. That's because * we have more tracking data available: the _pincount field is used * instead of the GUP_PIN_COUNTING_BIAS scheme. * * For more information, please see Documentation/core-api/pin_user_pages.rst. * * Return: True, if it is likely that the folio has been "dma-pinned". * False, if the folio is definitely not dma-pinned. */static inline bool folio_maybe_dma_pinned(struct folio *folio){ if (folio_has_pincount(folio)) return atomic_read(&folio->_pincount) > 0;
/* * folio_ref_count() is signed. If that refcount overflows, then * folio_ref_count() returns a negative value, and callers will avoid * further incrementing the refcount. * * Here, for that overflow case, use the sign bit to count a little * bit higher via unsigned math, and thus still get an accurate result. */ return ((unsigned int)folio_ref_count(folio)) >= GUP_PIN_COUNTING_BIAS;}
/* * This should most likely only be called during fork() to see whether we * should break the cow immediately for an anon page on the src mm. * * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq. */static inline bool folio_needs_cow_for_dma(struct vm_area_struct *vma, struct folio *folio){ VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1));
if (!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm)) return false;
return folio_maybe_dma_pinned(folio);}
/** * is_zero_page - Query if a page is a zero page * @page: The page to query * * This returns true if @page is one of the permanent zero pages. */static inline bool is_zero_page(const struct page *page){ return is_zero_pfn(page_to_pfn(page));}
/** * is_zero_folio - Query if a folio is a zero page * @folio: The folio to query * * This returns true if @folio is one of the permanent zero pages. */static inline bool is_zero_folio(const struct folio *folio){ return is_zero_page(&folio->page);}
/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */#ifdef CONFIG_MIGRATIONstatic inline bool folio_is_longterm_pinnable(struct folio *folio){#ifdef CONFIG_CMA int mt = folio_migratetype(folio);
if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) return false;#endif /* The zero page can be "pinned" but gets special handling. */ if (is_zero_folio(folio)) return true;
/* Coherent device memory must always allow eviction. */ if (folio_is_device_coherent(folio)) return false;
/* * Filesystems can only tolerate transient delays to truncate and * hole-punch operations */ if (folio_is_fsdax(folio)) return false;
/* Otherwise, non-movable zone folios can be pinned. */ return !folio_is_zone_movable(folio);
}#elsestatic inline bool folio_is_longterm_pinnable(struct folio *folio){ return true;}#endif
static inline void set_page_zone(struct page *page, enum zone_type zone){ page->flags.f &= ~(ZONES_MASK << ZONES_PGSHIFT); page->flags.f |= (zone & ZONES_MASK) << ZONES_PGSHIFT;}
static inline void set_page_node(struct page *page, unsigned long node){ page->flags.f &= ~(NODES_MASK << NODES_PGSHIFT); page->flags.f |= (node & NODES_MASK) << NODES_PGSHIFT;}
static inline void set_page_links(struct page *page, enum zone_type zone, unsigned long node, unsigned long pfn){ set_page_zone(page, zone); set_page_node(page, node);#ifdef SECTION_IN_PAGE_FLAGS set_page_section(page, pfn_to_section_nr(pfn));#endif}
/** * folio_nr_pages - The number of pages in the folio. * @folio: The folio. * * Return: A positive power of two. */static inline unsigned long folio_nr_pages(const struct folio *folio){ if (!folio_test_large(folio)) return 1; return folio_large_nr_pages(folio);}
/* * compound_nr() returns the number of pages in this potentially compound * page. compound_nr() can be called on a tail page, and is defined to * return 1 in that case. */static inline unsigned long compound_nr(const struct page *page){ const struct folio *folio = (struct folio *)page;
if (!test_bit(PG_head, &folio->flags.f)) return 1; return folio_large_nr_pages(folio);}
/** * folio_next - Move to the next physical folio. * @folio: The folio we're currently operating on. * * If you have physically contiguous memory which may span more than * one folio (eg a &struct bio_vec), use this function to move from one * folio to the next. Do not use it if the memory is only virtually * contiguous as the folios are almost certainly not adjacent to each * other. This is the folio equivalent to writing ``page++``. * * Context: We assume that the folios are refcounted and/or locked at a * higher level and do not adjust the reference counts. * Return: The next struct folio. */static inline struct folio *folio_next(struct folio *folio){ return (struct folio *)folio_page(folio, folio_nr_pages(folio));}
/** * folio_shift - The size of the memory described by this folio. * @folio: The folio. * * A folio represents a number of bytes which is a power-of-two in size. * This function tells you which power-of-two the folio is. See also * folio_size() and folio_order(). * * Context: The caller should have a reference on the folio to prevent * it from being split. It is not necessary for the folio to be locked. * Return: The base-2 logarithm of the size of this folio. */static inline unsigned int folio_shift(const struct folio *folio){ return PAGE_SHIFT + folio_order(folio);}
/** * folio_size - The number of bytes in a folio. * @folio: The folio. * * Context: The caller should have a reference on the folio to prevent * it from being split. It is not necessary for the folio to be locked. * Return: The number of bytes in this folio. */static inline size_t folio_size(const struct folio *folio){ return PAGE_SIZE << folio_order(folio);}
/** * folio_maybe_mapped_shared - Whether the folio is mapped into the page * tables of more than one MM * @folio: The folio. * * This function checks if the folio maybe currently mapped into more than one * MM ("maybe mapped shared"), or if the folio is certainly mapped into a single * MM ("mapped exclusively"). * * For KSM folios, this function also returns "mapped shared" when a folio is * mapped multiple times into the same MM, because the individual page mappings * are independent. * * For small anonymous folios and anonymous hugetlb folios, the return * value will be exactly correct: non-KSM folios can only be mapped at most once * into an MM, and they cannot be partially mapped. KSM folios are * considered shared even if mapped multiple times into the same MM. * * For other folios, the result can be fuzzy: * #. For partially-mappable large folios (THP), the return value can wrongly * indicate "mapped shared" (false positive) if a folio was mapped by * more than two MMs at one point in time. * #. For pagecache folios (including hugetlb), the return value can wrongly * indicate "mapped shared" (false positive) when two VMAs in the same MM * cover the same file range. * * Further, this function only considers current page table mappings that * are tracked using the folio mapcount(s). * * This function does not consider: * #. If the folio might get mapped in the (near) future (e.g., swapcache, * pagecache, temporary unmapping for migration). * #. If the folio is mapped differently (VM_PFNMAP). * #. If hugetlb page table sharing applies. Callers might want to check * hugetlb_pmd_shared(). * * Return: Whether the folio is estimated to be mapped into more than one MM. */static inline bool folio_maybe_mapped_shared(struct folio *folio){ int mapcount = folio_mapcount(folio);
/* Only partially-mappable folios require more care. */ if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio))) return mapcount > 1;
/* * vm_insert_page() without CONFIG_TRANSPARENT_HUGEPAGE ... * simply assume "mapped shared", nobody should really care * about this for arbitrary kernel allocations. */ if (!IS_ENABLED(CONFIG_MM_ID)) return true;
/* * A single mapping implies "mapped exclusively", even if the * folio flag says something different: it's easier to handle this * case here instead of on the RMAP hot path. */ if (mapcount <= 1) return false; return test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids);}
/** * folio_expected_ref_count - calculate the expected folio refcount * @folio: the folio * * Calculate the expected folio refcount, taking references from the pagecache, * swapcache, PG_private and page table mappings into account. Useful in * combination with folio_ref_count() to detect unexpected references (e.g., * GUP or other temporary references). * * Does currently not consider references from the LRU cache. If the folio * was isolated from the LRU (which is the case during migration or split), * the LRU cache does not apply. * * Calling this function on an unmapped folio -- !folio_mapped() -- that is * locked will return a stable result. * * Calling this function on a mapped folio will not result in a stable result, * because nothing stops additional page table mappings from coming (e.g., * fork()) or going (e.g., munmap()). * * Calling this function without the folio lock will also not result in a * stable result: for example, the folio might get dropped from the swapcache * concurrently. * * However, even when called without the folio lock or on a mapped folio, * this function can be used to detect unexpected references early (for example, * if it makes sense to even lock the folio and unmap it). * * The caller must add any reference (e.g., from folio_try_get()) it might be * holding itself to the result. * * Returns: the expected folio refcount. */static inline int folio_expected_ref_count(const struct folio *folio){ const int order = folio_order(folio); int ref_count = 0;
if (WARN_ON_ONCE(page_has_type(&folio->page) && !folio_test_hugetlb(folio))) return 0;
/* One reference per page from the swapcache. */ ref_count += folio_test_swapcache(folio) << order;
if (!folio_test_anon(folio)) { /* One reference per page from the pagecache. */ ref_count += !!folio->mapping << order; /* One reference from PG_private. */ ref_count += folio_test_private(folio); }
/* One reference per page table mapping. */ return ref_count + folio_mapcount(folio);}
#ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLEstatic inline int arch_make_folio_accessible(struct folio *folio){ return 0;}#endif
/* * Some inline functions in vmstat.h depend on page_zone() */#include <linux/vmstat.h>
#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)#define HASHED_PAGE_VIRTUAL#endif
#if defined(WANT_PAGE_VIRTUAL)static inline void *page_address(const struct page *page){ return page->virtual;}static inline void set_page_address(struct page *page, void *address){ page->virtual = address;}#define page_address_init() do { } while(0)#endif
#if defined(HASHED_PAGE_VIRTUAL)void *page_address(const struct page *page);void set_page_address(struct page *page, void *virtual);void page_address_init(void);#endif
static __always_inline void *lowmem_page_address(const struct page *page){ return page_to_virt(page);}
#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)#define page_address(page) lowmem_page_address(page)#define set_page_address(page, address) do { } while(0)#define page_address_init() do { } while(0)#endif
static inline void *folio_address(const struct folio *folio){ return page_address(&folio->page);}
/* * Return true only if the page has been allocated with * ALLOC_NO_WATERMARKS and the low watermark was not * met implying that the system is under some pressure. */static inline bool page_is_pfmemalloc(const struct page *page){ /* * lru.next has bit 1 set if the page is allocated from the * pfmemalloc reserves. Callers may simply overwrite it if * they do not need to preserve that information. */ return (uintptr_t)page->lru.next & BIT(1);}
/* * Return true only if the folio has been allocated with * ALLOC_NO_WATERMARKS and the low watermark was not * met implying that the system is under some pressure. */static inline bool folio_is_pfmemalloc(const struct folio *folio){ /* * lru.next has bit 1 set if the page is allocated from the * pfmemalloc reserves. Callers may simply overwrite it if * they do not need to preserve that information. */ return (uintptr_t)folio->lru.next & BIT(1);}
/* * Only to be called by the page allocator on a freshly allocated * page. */static inline void set_page_pfmemalloc(struct page *page){ page->lru.next = (void *)BIT(1);}
static inline void clear_page_pfmemalloc(struct page *page){ page->lru.next = NULL;}
/* * Can be called by the pagefault handler when it gets a VM_FAULT_OOM. */extern void pagefault_out_of_memory(void);
#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)#define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1))
/* * Parameter block passed down to zap_pte_range in exceptional cases. */struct zap_details { struct folio *single_folio; /* Locked folio to be unmapped */ bool skip_cows; /* Do not zap COWed private pages */ bool reclaim_pt; /* Need reclaim page tables? */ bool reaping; /* Reaping, do not block. */ zap_flags_t zap_flags; /* Extra flags for zapping */};
/* * Whether to drop the pte markers, for example, the uffd-wp information for * file-backed memory. This should only be specified when we will completely * drop the page in the mm, either by truncation or unmapping of the vma. By * default, the flag is not set. */#define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0))/* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */#define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1))
#ifdef CONFIG_MMUextern bool can_do_mlock(void);#elsestatic inline bool can_do_mlock(void) { return false; }#endifextern int user_shm_lock(size_t, struct ucounts *);extern void user_shm_unlock(size_t, struct ucounts *);
struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr, pte_t pte);struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte);struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd);struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t pmd);struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, pud_t pud);
void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address, unsigned long size);void zap_vma_range(struct vm_area_struct *vma, unsigned long address, unsigned long size);/** * zap_vma - zap all page table entries in a vma * @vma: The vma to zap. */static inline void zap_vma(struct vm_area_struct *vma){ zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start);}struct mmu_notifier_range;
void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling);intcopy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write);
struct follow_pfnmap_args { /** * Inputs: * @vma: Pointer to @vm_area_struct struct * @address: the virtual address to walk */ struct vm_area_struct *vma; unsigned long address; /** * Internals: * * The caller shouldn't touch any of these. */ spinlock_t *lock; pte_t *ptep; /** * Outputs: * * @pfn: the PFN of the address * @addr_mask: address mask covering pfn * @pgprot: the pgprot_t of the mapping * @writable: whether the mapping is writable * @special: whether the mapping is a special mapping (real PFN maps) */ unsigned long pfn; unsigned long addr_mask; pgprot_t pgprot; bool writable; bool special;};int follow_pfnmap_start(struct follow_pfnmap_args *args);void follow_pfnmap_end(struct follow_pfnmap_args *args);
extern void truncate_pagecache(struct inode *inode, loff_t new);extern void truncate_setsize(struct inode *inode, loff_t newsize);void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);int generic_error_remove_folio(struct address_space *mapping, struct folio *folio);
struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, unsigned long address, struct pt_regs *regs);
#ifdef CONFIG_MMUextern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, unsigned int flags, struct pt_regs *regs);extern int fixup_user_fault(struct mm_struct *mm, unsigned long address, unsigned int fault_flags, bool *unlocked);void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t nr, bool even_cows);void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows);#elsestatic inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, unsigned int flags, struct pt_regs *regs){ /* should never happen if there's no MMU */ BUG(); return VM_FAULT_SIGBUS;}static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address, unsigned int fault_flags, bool *unlocked){ /* should never happen if there's no MMU */ BUG(); return -EFAULT;}static inline void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t nr, bool even_cows) { }static inline void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows) { }#endif
static inline void unmap_shared_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen){ unmap_mapping_range(mapping, holebegin, holelen, 0);}
static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr);
extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags);extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags);
#ifdef CONFIG_BPF_SYSCALLextern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags);#endif
long get_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, int *locked);long pin_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, int *locked);
/* * Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT. */static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, unsigned long addr, int gup_flags, struct vm_area_struct **vmap){ struct page *page; struct vm_area_struct *vma; int got;
if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT))) return ERR_PTR(-EINVAL);
got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL);
if (got < 0) return ERR_PTR(got);
vma = vma_lookup(mm, addr); if (WARN_ON_ONCE(!vma)) { put_page(page); return ERR_PTR(-EINVAL); }
*vmap = vma; return page;}
long get_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages);long pin_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages);long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, struct page **pages, unsigned int gup_flags);long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, struct page **pages, unsigned int gup_flags);long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, struct folio **folios, unsigned int max_folios, pgoff_t *offset);int folio_add_pins(struct folio *folio, unsigned int pins);
int get_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages);int pin_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages);void folio_add_pin(struct folio *folio);
int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim);
struct kvec;struct page *get_dump_page(unsigned long addr, int *locked);
bool folio_mark_dirty(struct folio *folio);bool folio_mark_dirty_lock(struct folio *folio);bool set_page_dirty(struct page *page);int set_page_dirty_lock(struct page *page);
int get_cmdline(struct task_struct *task, char *buffer, int buflen);
/* * Flags used by change_protection(). For now we make it a bitmap so * that we can pass in multiple flags just like parameters. However * for now all the callers are only use one of the flags at the same * time. *//* * Whether we should manually check if we can map individual PTEs writable, * because something (e.g., COW, uffd-wp) blocks that from happening for all * PTEs automatically in a writable mapping. */#define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0)/* Whether this protection change is for NUMA hints */#define MM_CP_PROT_NUMA (1UL << 1)/* Whether this change is for write protecting */#define MM_CP_UFFD_WP (1UL << 2) /* do wp */#define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */#define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ MM_CP_UFFD_WP_RESOLVE)/* Whether this change is for uffd RWP */#define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */#define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */#define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \ MM_CP_UFFD_RWP_RESOLVE)
bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte);extern long change_protection(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long start, unsigned long end, unsigned long cp_flags);extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, struct vm_area_struct *vma, struct vm_area_struct **pprev, unsigned long start, unsigned long end, vm_flags_t newflags);
/* * doesn't attempt to fault and will return short. */int get_user_pages_fast_only(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages);
static inline bool get_user_page_fast_only(unsigned long addr, unsigned int gup_flags, struct page **pagep){ return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1;}/* * per-process(per-mm_struct) statistics. */static inline unsigned long get_mm_counter(struct mm_struct *mm, int member){ return percpu_counter_read_positive(&mm->rss_stat[member]);}
static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member){ return percpu_counter_sum_positive(&mm->rss_stat[member]);}
void mm_trace_rss_stat(struct mm_struct *mm, int member);
static inline void add_mm_counter(struct mm_struct *mm, int member, long value){ percpu_counter_add(&mm->rss_stat[member], value);
mm_trace_rss_stat(mm, member);}
static inline void inc_mm_counter(struct mm_struct *mm, int member){ percpu_counter_inc(&mm->rss_stat[member]);
mm_trace_rss_stat(mm, member);}
static inline void dec_mm_counter(struct mm_struct *mm, int member){ percpu_counter_dec(&mm->rss_stat[member]);
mm_trace_rss_stat(mm, member);}
/* Optimized variant when folio is already known not to be anon */static inline int mm_counter_file(struct folio *folio){ if (folio_test_swapbacked(folio)) return MM_SHMEMPAGES; return MM_FILEPAGES;}
static inline int mm_counter(struct folio *folio){ if (folio_test_anon(folio)) return MM_ANONPAGES; return mm_counter_file(folio);}
static inline unsigned long get_mm_rss(struct mm_struct *mm){ return get_mm_counter(mm, MM_FILEPAGES) + get_mm_counter(mm, MM_ANONPAGES) + get_mm_counter(mm, MM_SHMEMPAGES);}
static inline unsigned long get_mm_rss_sum(struct mm_struct *mm){ return get_mm_counter_sum(mm, MM_FILEPAGES) + get_mm_counter_sum(mm, MM_ANONPAGES) + get_mm_counter_sum(mm, MM_SHMEMPAGES);}
static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm){ return max(mm->hiwater_rss, get_mm_rss(mm));}
static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm){ return max(mm->hiwater_vm, mm->total_vm);}
static inline void update_hiwater_rss(struct mm_struct *mm){ unsigned long _rss = get_mm_rss(mm);
if (data_race(mm->hiwater_rss) < _rss) data_race(mm->hiwater_rss = _rss);}
static inline void update_hiwater_vm(struct mm_struct *mm){ if (mm->hiwater_vm < mm->total_vm) mm->hiwater_vm = mm->total_vm;}
static inline void reset_mm_hiwater_rss(struct mm_struct *mm){ mm->hiwater_rss = get_mm_rss(mm);}
static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm){ unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
if (*maxrss < hiwater_rss) *maxrss = hiwater_rss;}
#ifndef CONFIG_ARCH_HAS_PTE_SPECIALstatic inline int pte_special(pte_t pte){ return 0;}
static inline pte_t pte_mkspecial(pte_t pte){ return pte;}#endif
#ifndef CONFIG_ARCH_SUPPORTS_PMD_PFNMAPstatic inline bool pmd_special(pmd_t pmd){ return false;}
static inline pmd_t pmd_mkspecial(pmd_t pmd){ return pmd;}#endif /* CONFIG_ARCH_SUPPORTS_PMD_PFNMAP */
#ifndef CONFIG_ARCH_SUPPORTS_PUD_PFNMAPstatic inline bool pud_special(pud_t pud){ return false;}
static inline pud_t pud_mkspecial(pud_t pud){ return pud;}#endif /* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */
extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
#ifdef __PAGETABLE_P4D_FOLDEDstatic inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address){ return 0;}#elseint __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);#endif
#if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU)static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address){ return 0;}static inline void mm_inc_nr_puds(struct mm_struct *mm) {}static inline void mm_dec_nr_puds(struct mm_struct *mm) {}
#elseint __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
static inline void mm_inc_nr_puds(struct mm_struct *mm){ if (mm_pud_folded(mm)) return; atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);}
static inline void mm_dec_nr_puds(struct mm_struct *mm){ if (mm_pud_folded(mm)) return; atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);}#endif
#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address){ return 0;}
static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
#elseint __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
static inline void mm_inc_nr_pmds(struct mm_struct *mm){ if (mm_pmd_folded(mm)) return; atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);}
static inline void mm_dec_nr_pmds(struct mm_struct *mm){ if (mm_pmd_folded(mm)) return; atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);}#endif
#ifdef CONFIG_MMUstatic inline void mm_pgtables_bytes_init(struct mm_struct *mm){ atomic_long_set(&mm->pgtables_bytes, 0);}
static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm){ return atomic_long_read(&mm->pgtables_bytes);}
static inline void mm_inc_nr_ptes(struct mm_struct *mm){ atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);}
static inline void mm_dec_nr_ptes(struct mm_struct *mm){ atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);}#else
static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {}static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm){ return 0;}
static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}#endif
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd);int __pte_alloc_kernel(pmd_t *pmd);
#if defined(CONFIG_MMU)
static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address){ return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? NULL : p4d_offset(pgd, address);}
static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address){ return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? NULL : pud_offset(p4d, address);}
static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address){ return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? NULL: pmd_offset(pud, address);}#endif /* CONFIG_MMU */
enum pt_flags { PT_kernel = PG_referenced, PT_reserved = PG_reserved, /* High bits are used for zone/node/section */};
static inline struct ptdesc *virt_to_ptdesc(const void *x){ return page_ptdesc(virt_to_page(x));}
/** * ptdesc_address - Virtual address of page table. * @pt: Page table descriptor. * * Return: The first byte of the page table described by @pt. */static inline void *ptdesc_address(const struct ptdesc *pt){ return folio_address(ptdesc_folio(pt));}
static inline bool pagetable_is_reserved(struct ptdesc *pt){ return test_bit(PT_reserved, &pt->pt_flags.f);}
/** * ptdesc_set_kernel - Mark a ptdesc used to map the kernel * @ptdesc: The ptdesc to be marked * * Kernel page tables often need special handling. Set a flag so that * the handling code knows this ptdesc will not be used for userspace. */static inline void ptdesc_set_kernel(struct ptdesc *ptdesc){ set_bit(PT_kernel, &ptdesc->pt_flags.f);}
/** * ptdesc_clear_kernel - Mark a ptdesc as no longer used to map the kernel * @ptdesc: The ptdesc to be unmarked * * Use when the ptdesc is no longer used to map the kernel and no longer * needs special handling. */static inline void ptdesc_clear_kernel(struct ptdesc *ptdesc){ /* * Note: the 'PG_referenced' bit does not strictly need to be * cleared before freeing the page. But this is nice for * symmetry. */ clear_bit(PT_kernel, &ptdesc->pt_flags.f);}
/** * ptdesc_test_kernel - Check if a ptdesc is used to map the kernel * @ptdesc: The ptdesc being tested * * Call to tell if the ptdesc used to map the kernel. */static inline bool ptdesc_test_kernel(const struct ptdesc *ptdesc){ return test_bit(PT_kernel, &ptdesc->pt_flags.f);}
/** * pagetable_alloc - Allocate pagetables * @gfp: GFP flags * @order: desired pagetable order * * pagetable_alloc allocates memory for page tables as well as a page table * descriptor to describe that memory. * * Return: The ptdesc describing the allocated page tables. */static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order){ struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order);
return page_ptdesc(page);}#define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__))
static inline void __pagetable_free(struct ptdesc *pt){ struct page *page = ptdesc_page(pt);
__free_pages(page, compound_order(page));}
#ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREEvoid pagetable_free_kernel(struct ptdesc *pt);#elsestatic inline void pagetable_free_kernel(struct ptdesc *pt){ __pagetable_free(pt);}#endif/** * pagetable_free - Free pagetables * @pt: The page table descriptor * * pagetable_free frees the memory of all page tables described by a page * table descriptor and the memory for the descriptor itself. */static inline void pagetable_free(struct ptdesc *pt){ if (ptdesc_test_kernel(pt)) { ptdesc_clear_kernel(pt); pagetable_free_kernel(pt); } else { __pagetable_free(pt); }}
#if defined(CONFIG_SPLIT_PTE_PTLOCKS)#if ALLOC_SPLIT_PTLOCKSvoid __init ptlock_cache_init(void);bool ptlock_alloc(struct ptdesc *ptdesc);void ptlock_free(struct ptdesc *ptdesc);
static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc){ return ptdesc->ptl;}#else /* ALLOC_SPLIT_PTLOCKS */static inline void ptlock_cache_init(void){}
static inline bool ptlock_alloc(struct ptdesc *ptdesc){ return true;}
static inline void ptlock_free(struct ptdesc *ptdesc){}
static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc){ return &ptdesc->ptl;}#endif /* ALLOC_SPLIT_PTLOCKS */
static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd){ return ptlock_ptr(page_ptdesc(pmd_page(*pmd)));}
static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte){ BUILD_BUG_ON(IS_ENABLED(CONFIG_HIGHPTE)); BUILD_BUG_ON(MAX_PTRS_PER_PTE * sizeof(pte_t) > PAGE_SIZE); return ptlock_ptr(virt_to_ptdesc(pte));}
static inline bool ptlock_init(struct ptdesc *ptdesc){ /* * prep_new_page() initialize page->private (and therefore page->ptl) * with 0. Make sure nobody took it in use in between. * * It can happen if arch try to use slab for page table allocation: * slab code uses page->slab_cache, which share storage with page->ptl. */ VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc)); if (!ptlock_alloc(ptdesc)) return false; spin_lock_init(ptlock_ptr(ptdesc)); return true;}
#else /* !defined(CONFIG_SPLIT_PTE_PTLOCKS) *//* * We use mm->page_table_lock to guard all pagetable pages of the mm. */static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd){ return &mm->page_table_lock;}static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte){ return &mm->page_table_lock;}static inline void ptlock_cache_init(void) {}static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; }static inline void ptlock_free(struct ptdesc *ptdesc) {}#endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */
static inline void __pagetable_ctor(struct ptdesc *ptdesc){ struct folio *folio = ptdesc_folio(ptdesc);
__folio_set_pgtable(folio); lruvec_stat_add_folio(folio, NR_PAGETABLE);}
static inline void pagetable_dtor(struct ptdesc *ptdesc){ struct folio *folio = ptdesc_folio(ptdesc);
ptlock_free(ptdesc); __folio_clear_pgtable(folio); lruvec_stat_sub_folio(folio, NR_PAGETABLE);}
static inline void pagetable_dtor_free(struct ptdesc *ptdesc){ pagetable_dtor(ptdesc); pagetable_free(ptdesc);}
static inline bool pagetable_pte_ctor(struct mm_struct *mm, struct ptdesc *ptdesc){ if (mm != &init_mm && !ptlock_init(ptdesc)) return false; __pagetable_ctor(ptdesc); return true;}
pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr){ return __pte_offset_map(pmd, addr, NULL);}
pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp);
pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp);pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp, spinlock_t **ptlp);
#define pte_unmap_unlock(pte, ptl) do { \ spin_unlock(ptl); \ pte_unmap(pte); \} while (0)
#define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd))
#define pte_alloc_map(mm, pmd, address) \ (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address))
#define pte_alloc_map_lock(mm, pmd, address, ptlp) \ (pte_alloc(mm, pmd) ? \ NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
#define pte_alloc_kernel(pmd, address) \ ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \ NULL: pte_offset_kernel(pmd, address))
#if defined(CONFIG_SPLIT_PMD_PTLOCKS)
static inline struct page *pmd_pgtable_page(pmd_t *pmd){ unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); return virt_to_page((void *)((unsigned long) pmd & mask));}
static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd){ return page_ptdesc(pmd_pgtable_page(pmd));}
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd){ return ptlock_ptr(pmd_ptdesc(pmd));}
static inline bool pmd_ptlock_init(struct ptdesc *ptdesc){#ifdef CONFIG_TRANSPARENT_HUGEPAGE ptdesc->pmd_huge_pte = NULL;#endif return ptlock_init(ptdesc);}
#define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte)
#else
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd){ return &mm->page_table_lock;}
static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; }
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
#endif
static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd){ spinlock_t *ptl = pmd_lockptr(mm, pmd); spin_lock(ptl); return ptl;}
static inline bool pagetable_pmd_ctor(struct mm_struct *mm, struct ptdesc *ptdesc){ if (mm != &init_mm && !pmd_ptlock_init(ptdesc)) return false; ptdesc_pmd_pts_init(ptdesc); __pagetable_ctor(ptdesc); return true;}
/* * No scalability reason to split PUD locks yet, but follow the same pattern * as the PMD locks to make it easier if we decide to. The VM should not be * considered ready to switch to split PUD locks yet; there may be places * which need to be converted from page_table_lock. */static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud){ return &mm->page_table_lock;}
static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud){ spinlock_t *ptl = pud_lockptr(mm, pud);
spin_lock(ptl); return ptl;}
static inline void pagetable_pud_ctor(struct ptdesc *ptdesc){ __pagetable_ctor(ptdesc);}
static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc){ __pagetable_ctor(ptdesc);}
static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc){ __pagetable_ctor(ptdesc);}
extern void __init pagecache_init(void);extern void free_initmem(void);
/* * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK) * into the buddy system. The freed pages will be poisoned with pattern * "poison" if it's within range [0, UCHAR_MAX]. * Return pages freed into the buddy system. */extern unsigned long free_reserved_area(void *start, void *end, int poison, const char *s);
extern void adjust_managed_page_count(struct page *page, long count);
void free_reserved_pages(struct page *page, unsigned int order);
static inline void free_reserved_page(struct page *page){ free_reserved_pages(page, 0);}
static inline void mark_page_reserved(struct page *page){ SetPageReserved(page); adjust_managed_page_count(page, -1);}
static inline void free_reserved_ptdesc(struct ptdesc *pt){ free_reserved_page(ptdesc_page(pt));}
/* * Default method to free all the __init memory into the buddy system. * The freed pages will be poisoned with pattern "poison" if it's within * range [0, UCHAR_MAX]. * Return pages freed into the buddy system. */static inline unsigned long free_initmem_default(int poison){ extern char __init_begin[], __init_end[];
return free_reserved_area(&__init_begin, &__init_end, poison, "unused kernel image (initmem)");}
static inline unsigned long get_num_physpages(void){ int nid; unsigned long phys_pages = 0;
for_each_online_node(nid) phys_pages += node_present_pages(nid);
return phys_pages;}
/* * FIXME: Using memblock node mappings, an architecture may initialise its * zones, allocate the backing mem_map and account for memory holes in an * architecture independent manner. * * An architecture is expected to register range of page frames backed by * physical memory with memblock_add[_node]() before calling * free_area_init() passing in the PFN each zone ends at. At a basic * usage, an architecture is expected to do something like * * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn, * max_highmem_pfn}; * for_each_valid_physical_page_range() * memblock_add_node(base, size, nid, MEMBLOCK_NONE) * free_area_init(max_zone_pfns); */void arch_zone_limits_init(unsigned long *max_zone_pfn);unsigned long node_map_pfn_alignment(void);extern unsigned long absent_pages_in_range(unsigned long start_pfn, unsigned long end_pfn);extern void get_pfn_range_for_nid(unsigned int nid, unsigned long *start_pfn, unsigned long *end_pfn);
#ifndef CONFIG_NUMAstatic inline int early_pfn_to_nid(unsigned long pfn){ return 0;}#else/* please see mm/page_alloc.c */extern int __meminit early_pfn_to_nid(unsigned long pfn);#endif
extern void mem_init(void);extern void __init mmap_init(void);
extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx);static inline void show_mem(void){ __show_mem(0, NULL, MAX_NR_ZONES - 1);}extern long si_mem_available(void);extern void si_meminfo(struct sysinfo * val);extern void si_meminfo_node(struct sysinfo *val, int nid);
extern __printf(3, 4)void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...);
extern void setup_per_cpu_pageset(void);
/* nommu.c */extern atomic_long_t mmap_pages_allocated;extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
/* interval_tree.c */void mapping_rmap_tree_insert(struct vm_area_struct *vma, struct address_space *mapping);void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, struct address_space *mapping);void mapping_rmap_tree_remove(struct vm_area_struct *vma, struct address_space *mapping);struct vm_area_struct *mapping_rmap_tree_iter_first(struct address_space *mapping, pgoff_t pgoff_start, pgoff_t pgoff_last);struct vm_area_struct *mapping_rmap_tree_iter_next(struct vm_area_struct *vma, pgoff_t pgoff_start, pgoff_t pgoff_last);
#define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start, \ pgoff_last); \ vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ pgoff_last))
void anon_rmap_tree_insert(struct anon_vma_chain *avc, struct anon_vma *anon_vma);void anon_rmap_tree_remove(struct anon_vma_chain *avc, struct anon_vma *anon_vma);struct anon_vma_chain *anon_rmap_tree_iter_first(struct anon_vma *anon_vma, pgoff_t pgoff_start, pgoff_t pgoff_last);struct anon_vma_chain *anon_rmap_tree_iter_next(struct anon_vma_chain *avc, pgoff_t pgoff_start, pgoff_t pgoff_last);#ifdef CONFIG_DEBUG_VM_RBvoid anon_rmap_tree_verify(struct anon_vma_chain *avc);#endif
#define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \ avc; avc = anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last))
/* mmap.c */extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin);extern void exit_mmap(struct mm_struct *);bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, bool write);
static inline int check_data_rlimit(unsigned long rlim, unsigned long new, unsigned long start, unsigned long end_data, unsigned long start_data){ if (rlim < RLIM_INFINITY) { if (((new - start) + (end_data - start_data)) > rlim) return -ENOSPC; }
return 0;}
extern int mm_take_all_locks(struct mm_struct *mm);extern void mm_drop_all_locks(struct mm_struct *mm);
extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);extern struct file *get_mm_exe_file(struct mm_struct *mm);extern struct file *get_task_exe_file(struct task_struct *task);
extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
extern bool vma_is_special_mapping(const struct vm_area_struct *vma, const struct vm_special_mapping *sm);struct vm_area_struct *_install_special_mapping(struct mm_struct *mm, unsigned long addr, unsigned long len, vm_flags_t vm_flags, const struct vm_special_mapping *spec);
unsigned long randomize_stack_top(unsigned long stack_top);unsigned long randomize_page(unsigned long start, unsigned long range);
unsigned long__get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, vma_flags_t vma_flags);
static inline unsigned longget_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags){ return __get_unmapped_area(file, addr, len, pgoff, flags, EMPTY_VMA_FLAGS);}
unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf);extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, bool unlock);int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, struct mm_struct *mm, unsigned long start, unsigned long end, struct list_head *uf, bool unlock);extern int do_munmap(struct mm_struct *, unsigned long, size_t, struct list_head *uf);extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior);
#ifdef CONFIG_MMUextern int __mm_populate(unsigned long addr, unsigned long len, int ignore_errors);static inline void mm_populate(unsigned long addr, unsigned long len){ /* Ignore errors */ (void) __mm_populate(addr, len, 1);}#elsestatic inline void mm_populate(unsigned long addr, unsigned long len) {}#endif
/* This takes the mm semaphore itself */int __must_check vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec);int vm_munmap(unsigned long start, size_t len);unsigned long __must_check vm_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset);unsigned long __must_check vm_mmap_shadow_stack(unsigned long addr, unsigned long len, unsigned long flags);
struct vm_unmapped_area_info {#define VM_UNMAPPED_AREA_TOPDOWN 1 unsigned long flags; unsigned long length; unsigned long low_limit; unsigned long high_limit; unsigned long align_mask; unsigned long align_offset; unsigned long start_gap;};
extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info);
/* truncate.c */void truncate_inode_pages(struct address_space *mapping, loff_t lstart);void truncate_inode_pages_range(struct address_space *mapping, loff_t lstart, uoff_t lend);void truncate_inode_pages_final(struct address_space *mapping);
/* generic vm_area_ops exported for stackable file systems */extern vm_fault_t filemap_fault(struct vm_fault *vmf);extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff);extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);
extern unsigned long stack_guard_gap;/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */int expand_stack_locked(struct vm_area_struct *vma, unsigned long address);struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr);
/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, struct vm_area_struct **pprev);
/* * Look up the first VMA which intersects the interval [start_addr, end_addr) * NULL if none. Assume start_addr < end_addr. */struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, unsigned long start_addr, unsigned long end_addr);
/** * vma_lookup() - Find a VMA at a specific address * @mm: The process address space. * @addr: The user address. * * Return: The vm_area_struct at the given address, %NULL otherwise. */static inlinestruct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr){ return mtree_load(&mm->mm_mt, addr);}
static inline unsigned long stack_guard_start_gap(const struct vm_area_struct *vma){ if (vma->vm_flags & VM_GROWSDOWN) return stack_guard_gap;
/* See reasoning around the VM_SHADOW_STACK definition */ if (vma->vm_flags & VM_SHADOW_STACK) return PAGE_SIZE;
return 0;}
static inline unsigned long vm_start_gap(const struct vm_area_struct *vma){ unsigned long gap = stack_guard_start_gap(vma); unsigned long vm_start = vma->vm_start;
vm_start -= gap; if (vm_start > vma->vm_start) vm_start = 0; return vm_start;}
static inline unsigned long vm_end_gap(const struct vm_area_struct *vma){ unsigned long vm_end = vma->vm_end;
if (vma->vm_flags & VM_GROWSUP) { vm_end += stack_guard_gap; if (vm_end < vma->vm_end) vm_end = -PAGE_SIZE; } return vm_end;}
static inline unsigned long vma_pages(const struct vm_area_struct *vma){ return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;}
/** * vma_start_pgoff() - Get the page offset of the start of @vma * @vma: The VMA whose page offset is required. * * If the VMA is file-backed, this is the page offset into the file. * * If @vma is anonymous, this is the virtual page offset of the start of the * VMA - if unfaulted, then vma->vm_start >> PAGE_SHIFT, if faulted then the * virtual page offset at the time of first fault. * * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the * page offset within the file. * * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap * regions have their page offset set to the first PFN in the range. * * Returns: The page offset of the start of @vma. */static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma){ return vma->vm_pgoff;}
/** * vma_end_pgoff() - Get the page offset of the exclusive end of @vma * @vma: The VMA whose end page offset is required. * * This returns the exclusive end page offset of @vma, which is useful for * expressing page offset ranges. * * See the description of vma_start_pgoff() for a description of VMA page * offsets. * * Returns: The exclusive end page offset of @vma. */static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma){ return vma_start_pgoff(vma) + vma_pages(vma);}
/** * vma_last_pgoff() - Get the page offset of the last page in @vma * @vma: The VMA whose last page offset is required. * * This returns the last page offset contained within @vma. * * See the description of vma_start_pgoff() for a description of VMA page * offsets. * * Returns: The last page offset of @vma. */static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma){ return vma_end_pgoff(vma) - 1;}
/** * vma_start_anon_pgoff() - Get the anonymous page offset of the start of @vma * @vma: The VMA whose anonymous page offset is required. * * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the * anonymous page offset at the time of first fault. * * If the VMA is anonymous, this returns the same value as vma_start_pgoff(). * * This value is used for tracking MAP_PRIVATE file-backed mappings by their * anonymous page offset. * * Returns: The anonymous page offset of the start of @vma. */static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma){ pgoff_t pgoff = 0;
#ifdef CONFIG_64BIT pgoff += vma->__vm_anon_pgoff_hi; pgoff <<= 32;#endif pgoff += vma->__vm_anon_pgoff_lo; return pgoff;}
/** * vma_end_anon_pgoff() - Get the anonymous page offset of the exclusive end of * @vma. * @vma: The VMA whose end anonymous page offset is required. * * This returns the anonymous exclusive end page offset of @vma, which is useful * for expressing page offset ranges. * * See the description of vma_start_anon_pgoff() for a description of VMA * anonymous page offsets. * * Returns: The exclusive end anonymous page offset of @vma. */static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma){ return vma_start_anon_pgoff(vma) + vma_pages(vma);}
/** * vma_last_anon_pgoff() - Get the anonymous page offset of the last page in * @vma. * @vma: The VMA whose last anonymous page offset is required. * * See the description of vma_start_anon_pgoff() for a description of VMA * anonymous page offsets. * * Returns: The last anonymous page offset of @vma. */static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma){ return vma_end_anon_pgoff(vma) - 1;}
static inline unsigned long vma_desc_size(const struct vm_area_desc *desc){ return desc->end - desc->start;}
static inline unsigned long vma_desc_pages(const struct vm_area_desc *desc){ return vma_desc_size(desc) >> PAGE_SHIFT;}
/** * mmap_action_remap - helper for mmap_prepare hook to specify that a pure PFN * remap is required. * @desc: The VMA descriptor for the VMA requiring remap. * @start: The virtual address to start the remap from, must be within the VMA. * @start_pfn: The first PFN in the range to remap. * @size: The size of the range to remap, in bytes, at most spanning to the end * of the VMA. */static inline void mmap_action_remap(struct vm_area_desc *desc, unsigned long start, unsigned long start_pfn, unsigned long size){ struct mmap_action *action = &desc->action;
/* [start, start + size) must be within the VMA. */ WARN_ON_ONCE(start < desc->start || start >= desc->end); WARN_ON_ONCE(start + size > desc->end);
action->type = MMAP_REMAP_PFN; action->remap.start = start; action->remap.start_pfn = start_pfn; action->remap.size = size; action->remap.pgprot = desc->page_prot;}
/** * mmap_action_remap_full - helper for mmap_prepare hook to specify that the * entirety of a VMA should be PFN remapped. * @desc: The VMA descriptor for the VMA requiring remap. * @start_pfn: The first PFN in the range to remap. */static inline void mmap_action_remap_full(struct vm_area_desc *desc, unsigned long start_pfn){ mmap_action_remap(desc, desc->start, start_pfn, vma_desc_size(desc));}
/** * mmap_action_ioremap - helper for mmap_prepare hook to specify that a pure PFN * I/O remap is required. * @desc: The VMA descriptor for the VMA requiring remap. * @start: The virtual address to start the remap from, must be within the VMA. * @start_pfn: The first PFN in the range to remap. * @size: The size of the range to remap, in bytes, at most spanning to the end * of the VMA. */static inline void mmap_action_ioremap(struct vm_area_desc *desc, unsigned long start, unsigned long start_pfn, unsigned long size){ mmap_action_remap(desc, start, start_pfn, size); desc->action.type = MMAP_IO_REMAP_PFN;}
/** * mmap_action_ioremap_full - helper for mmap_prepare hook to specify that the * entirety of a VMA should be PFN I/O remapped. * @desc: The VMA descriptor for the VMA requiring remap. * @start_pfn: The first PFN in the range to remap. */static inline void mmap_action_ioremap_full(struct vm_area_desc *desc, unsigned long start_pfn){ mmap_action_ioremap(desc, desc->start, start_pfn, vma_desc_size(desc));}
/** * mmap_action_simple_ioremap - helper for mmap_prepare hook to specify that the * physical range in [start_phys_addr, start_phys_addr + size) should be I/O * remapped. * @desc: The VMA descriptor for the VMA requiring remap. * @start_phys_addr: Start of the physical memory to be mapped. * @size: Size of the area to map. * * NOTE: Some drivers might want to tweak desc->page_prot for purposes of * write-combine or similar. */static inline void mmap_action_simple_ioremap(struct vm_area_desc *desc, phys_addr_t start_phys_addr, unsigned long size){ struct mmap_action *action = &desc->action;
action->simple_ioremap.start_phys_addr = start_phys_addr; action->simple_ioremap.size = size; action->type = MMAP_SIMPLE_IO_REMAP;}
/** * mmap_action_map_kernel_pages - helper for mmap_prepare hook to specify that * @num kernel pages contained in the @pages array should be mapped to userland * starting at virtual address @start. * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. * @start: The virtual address from which to map them. * @pages: An array of struct page pointers describing the memory to map. * @nr_pages: The number of entries in the @pages aray. */static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc, unsigned long start, struct page **pages, unsigned long nr_pages){ struct mmap_action *action = &desc->action;
action->type = MMAP_MAP_KERNEL_PAGES; action->map_kernel.start = start; action->map_kernel.pages = pages; action->map_kernel.nr_pages = nr_pages; action->map_kernel.pgoff = desc->pgoff;}
/** * mmap_action_map_kernel_pages_full - helper for mmap_prepare hook to specify that * kernel pages contained in the @pages array should be mapped to userland * from @desc->start to @desc->end. * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. * @pages: An array of struct page pointers describing the memory to map. * * The caller must ensure that @pages contains sufficient entries to cover the * entire range described by @desc. */static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc, struct page **pages){ mmap_action_map_kernel_pages(desc, desc->start, pages, vma_desc_pages(desc));}
int mmap_action_prepare(struct vm_area_desc *desc);int mmap_action_complete(struct vm_area_struct *vma, struct mmap_action *action, bool is_compat);
/* Look up the first VMA which exactly match the interval vm_start ... vm_end */static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, unsigned long vm_start, unsigned long vm_end){ struct vm_area_struct *vma = vma_lookup(mm, vm_start);
if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) vma = NULL;
return vma;}
/** * range_is_subset - Is the specified inner range a subset of the outer range? * @outer_start: The start of the outer range. * @outer_end: The exclusive end of the outer range. * @inner_start: The start of the inner range. * @inner_end: The exclusive end of the inner range. * * Returns: %true if [inner_start, inner_end) is a subset of [outer_start, * outer_end), otherwise %false. */static inline bool range_is_subset(unsigned long outer_start, unsigned long outer_end, unsigned long inner_start, unsigned long inner_end){ return outer_start <= inner_start && inner_end <= outer_end;}
/** * range_in_vma - is the specified [@start, @end) range a subset of the VMA? * @vma: The VMA against which we want to check [@start, @end). * @start: The start of the range we wish to check. * @end: The exclusive end of the range we wish to check. * * Returns: %true if [@start, @end) is a subset of [@vma->vm_start, * @vma->vm_end), %false otherwise. */static inline bool range_in_vma(const struct vm_area_struct *vma, unsigned long start, unsigned long end){ if (!vma) return false;
return range_is_subset(vma->vm_start, vma->vm_end, start, end);}
/** * range_in_vma_desc - is the specified [@start, @end) range a subset of the VMA * described by @desc, a VMA descriptor? * @desc: The VMA descriptor against which we want to check [@start, @end). * @start: The start of the range we wish to check. * @end: The exclusive end of the range we wish to check. * * Returns: %true if [@start, @end) is a subset of [@desc->start, @desc->end), * %false otherwise. */static inline bool range_in_vma_desc(const struct vm_area_desc *desc, unsigned long start, unsigned long end){ if (!desc) return false;
return range_is_subset(desc->start, desc->end, start, end);}
#ifdef CONFIG_MMUpgprot_t vm_get_page_prot(vm_flags_t vm_flags);
static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags){ const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags);
return vm_get_page_prot(vm_flags);}
static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma){ return vma_flags_to_page_prot(vma->flags);}
void vma_set_page_prot(struct vm_area_struct *vma);#elsestatic inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags){ return __pgprot(0);}static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags){ return __pgprot(0);}static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma){ return __pgprot(0);}static inline void vma_set_page_prot(struct vm_area_struct *vma){ vma->vm_page_prot = vma_get_page_prot(vma);}#endif
void vma_set_file(struct vm_area_struct *vma, struct file *file);
#ifdef CONFIG_NUMA_BALANCINGunsigned long change_prot_numa(struct vm_area_struct *vma, unsigned long start, unsigned long end);#endif
struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, unsigned long addr);int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot);
int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num);int map_kernel_pages_prepare(struct vm_area_desc *desc);int map_kernel_pages_complete(struct vm_area_struct *vma, struct mmap_action *action);int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num);int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num);vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write);vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn);vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot);vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn);vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn);int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, unsigned long addr, struct page *page){ int err = vm_insert_page(vma, addr, page);
if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) return VM_FAULT_SIGBUS;
return VM_FAULT_NOPAGE;}
#ifndef io_remap_pfn_range_pfnstatic inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, unsigned long size){ return pfn;}#endif
static inline int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long orig_pfn, unsigned long size, pgprot_t orig_prot){ const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); const pgprot_t prot = pgprot_decrypted(orig_prot);
return remap_pfn_range(vma, addr, pfn, size, prot);}
static inline vm_fault_t vmf_error(int err){ if (err == -ENOMEM) return VM_FAULT_OOM; else if (err == -EHWPOISON) return VM_FAULT_HWPOISON; return VM_FAULT_SIGBUS;}
/* * Convert errno to return value for ->page_mkwrite() calls. * * This should eventually be merged with vmf_error() above, but will need a * careful audit of all vmf_error() callers. */static inline vm_fault_t vmf_fs_error(int err){ if (err == 0) return VM_FAULT_LOCKED; if (err == -EFAULT || err == -EAGAIN) return VM_FAULT_NOPAGE; if (err == -ENOMEM) return VM_FAULT_OOM; /* -ENOSPC, -EDQUOT, -EIO ... */ return VM_FAULT_SIGBUS;}
static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags){ if (vm_fault & VM_FAULT_OOM) return -ENOMEM; if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT; if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) return -EFAULT; return 0;}
/* * Indicates whether GUP can follow a PROT_NONE mapped page, or whether * a (NUMA hinting or userfaultfd RWP) fault is required. */static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, unsigned int flags){ /* * VM_UFFD_RWP uses protnone as an access-tracking marker, not for * NUMA hinting. GUP must always take a fault so the access is * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT. * * Only do so while the VMA is accessible. If it has been made * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard * below: forcing a fault there would loop, as handle_mm_fault() makes * no progress on protnone in an inaccessible VMA, and the access is * denied regardless of RWP anyway. */ if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma)) return false;
/* * If callers don't want to honor NUMA hinting faults, no need to * determine if we would actually have to trigger a NUMA hinting fault. */ if (!(flags & FOLL_HONOR_NUMA_FAULT)) return true;
/* * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs. * * Requiring a fault here even for inaccessible VMAs would mean that * FOLL_FORCE cannot make any progress, because handle_mm_fault() * refuses to process NUMA hinting faults in inaccessible VMAs. */ return !vma_is_accessible(vma);}
typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data);extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data);extern int apply_to_existing_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data);
#ifdef CONFIG_PAGE_POISONINGextern void __kernel_poison_pages(struct page *page, int numpages);extern void __kernel_unpoison_pages(struct page *page, int numpages);extern bool _page_poisoning_enabled_early;DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled);static inline bool page_poisoning_enabled(void){ return _page_poisoning_enabled_early;}/* * For use in fast paths after init_mem_debugging() has run, or when a * false negative result is not harmful when called too early. */static inline bool page_poisoning_enabled_static(void){ return static_branch_unlikely(&_page_poisoning_enabled);}static inline void kernel_poison_pages(struct page *page, int numpages){ if (page_poisoning_enabled_static()) __kernel_poison_pages(page, numpages);}static inline void kernel_unpoison_pages(struct page *page, int numpages){ if (page_poisoning_enabled_static()) __kernel_unpoison_pages(page, numpages);}#elsestatic inline bool page_poisoning_enabled(void) { return false; }static inline bool page_poisoning_enabled_static(void) { return false; }static inline void __kernel_poison_pages(struct page *page, int nunmpages) { }static inline void kernel_poison_pages(struct page *page, int numpages) { }static inline void kernel_unpoison_pages(struct page *page, int numpages) { }#endif
DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);static inline bool want_init_on_alloc(gfp_t flags){ if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, &init_on_alloc)) return true; return flags & __GFP_ZERO;}
DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);static inline bool want_init_on_free(void){ return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON, &init_on_free);}
extern bool _debug_pagealloc_enabled_early;DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
static inline bool debug_pagealloc_enabled(void){ return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && _debug_pagealloc_enabled_early;}
/* * For use in fast paths after mem_debugging_and_hardening_init() has run, * or when a false negative result is not harmful when called too early. */static inline bool debug_pagealloc_enabled_static(void){ if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC)) return false;
return static_branch_unlikely(&_debug_pagealloc_enabled);}
/* * To support DEBUG_PAGEALLOC architecture must ensure that * __kernel_map_pages() never fails */extern void __kernel_map_pages(struct page *page, int numpages, int enable);#ifdef CONFIG_DEBUG_PAGEALLOCstatic inline void debug_pagealloc_map_pages(struct page *page, int numpages){ iommu_debug_check_unmapped(page, numpages);
if (debug_pagealloc_enabled_static()) __kernel_map_pages(page, numpages, 1);}
static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages){ iommu_debug_check_unmapped(page, numpages);
if (debug_pagealloc_enabled_static()) __kernel_map_pages(page, numpages, 0);}
extern unsigned int _debug_guardpage_minorder;DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
static inline unsigned int debug_guardpage_minorder(void){ return _debug_guardpage_minorder;}
static inline bool debug_guardpage_enabled(void){ return static_branch_unlikely(&_debug_guardpage_enabled);}
static inline bool page_is_guard(const struct page *page){ if (!debug_guardpage_enabled()) return false;
return PageGuard(page);}
bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order);static inline bool set_page_guard(struct zone *zone, struct page *page, unsigned int order){ if (!debug_guardpage_enabled()) return false; return __set_page_guard(zone, page, order);}
void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order);static inline void clear_page_guard(struct zone *zone, struct page *page, unsigned int order){ if (!debug_guardpage_enabled()) return; __clear_page_guard(zone, page, order);}
#else /* CONFIG_DEBUG_PAGEALLOC */static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {}static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {}static inline unsigned int debug_guardpage_minorder(void) { return 0; }static inline bool debug_guardpage_enabled(void) { return false; }static inline bool page_is_guard(const struct page *page) { return false; }static inline bool set_page_guard(struct zone *zone, struct page *page, unsigned int order) { return false; }static inline void clear_page_guard(struct zone *zone, struct page *page, unsigned int order) {}#endif /* CONFIG_DEBUG_PAGEALLOC */
#ifndef clear_pages/** * clear_pages() - clear a page range for kernel-internal use. * @addr: start address * @npages: number of pages * * Use clear_user_pages() instead when clearing a page range to be * mapped to user space. * * Does absolutely no exception handling. * * Note that even though the clearing operation is preemptible, clear_pages() * does not (and on architectures where it reduces to a few long-running * instructions, might not be able to) call cond_resched() to check if * rescheduling is required. * * When running under preemptible models this is not a problem. Under * cooperatively scheduled models, however, the caller is expected to * limit @npages to no more than PROCESS_PAGES_NON_PREEMPT_BATCH. */static inline void clear_pages(void *addr, unsigned int npages){ do { clear_page(addr); addr += PAGE_SIZE; } while (--npages);}#endif
#ifndef PROCESS_PAGES_NON_PREEMPT_BATCH#ifdef clear_pages/* * The architecture defines clear_pages(), and we assume that it is * generally "fast". So choose a batch size large enough to allow the processor * headroom for optimizing the operation and yet small enough that we see * reasonable preemption latency for when this optimization is not possible * (ex. slow microarchitectures, memory bandwidth saturation.) * * With a value of 32MB and assuming a memory bandwidth of ~10GBps, this should * result in worst case preemption latency of around 3ms when clearing pages. * * (See comment above clear_pages() for why preemption latency is a concern * here.) */#define PROCESS_PAGES_NON_PREEMPT_BATCH (SZ_32M >> PAGE_SHIFT)#else /* !clear_pages *//* * The architecture does not provide a clear_pages() implementation. Assume * that clear_page() -- which clear_pages() will fallback to -- is relatively * slow and choose a small value for PROCESS_PAGES_NON_PREEMPT_BATCH. */#define PROCESS_PAGES_NON_PREEMPT_BATCH 1#endif#endif
#ifdef __HAVE_ARCH_GATE_AREAextern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);extern int in_gate_area_no_mm(unsigned long addr);extern int in_gate_area(struct mm_struct *mm, unsigned long addr);#elsestatic inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm){ return NULL;}static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }static inline int in_gate_area(struct mm_struct *mm, unsigned long addr){ return 0;}#endif /* __HAVE_ARCH_GATE_AREA */
bool process_shares_mm(const struct task_struct *p, const struct mm_struct *mm);
void drop_slab(void);
#ifndef CONFIG_MMU#define randomize_va_space 0#elseextern int randomize_va_space;#endif
const char * arch_vma_name(struct vm_area_struct *vma);#ifdef CONFIG_MMUvoid print_vma_addr(char *prefix, unsigned long rip);#elsestatic inline void print_vma_addr(char *prefix, unsigned long rip){}#endif
unsigned long section_map_size(void);struct page * __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap);void *vmemmap_alloc_block(unsigned long size, int node);struct vmem_altmap;void *vmemmap_alloc_block_buf(unsigned long size, int node, struct vmem_altmap *altmap);void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);void vmemmap_set_pmd(pmd_t *pmd, void *p, int node, unsigned long addr, unsigned long next);int vmemmap_check_pmd(pmd_t *pmd, int node, unsigned long addr, unsigned long next);int vmemmap_populate_basepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap);int vmemmap_populate_hugepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap);int vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap);int vmemmap_populate_hvo(unsigned long start, unsigned long end, unsigned int order, struct zone *zone, unsigned long headsize);void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, unsigned long headsize);void vmemmap_populate_print_last(void);#ifdef CONFIG_MEMORY_HOTPLUGvoid vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap);#endif
#ifdef CONFIG_SPARSEMEM_VMEMMAPstatic inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap){ /* number of pfns from base where pfn_to_page() is valid */ if (altmap) return altmap->reserve + altmap->free; return 0;}
static inline void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns){ altmap->alloc -= nr_pfns;}#elsestatic inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap){ return 0;}
static inline void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns){}#endif
#define VMEMMAP_RESERVE_NR 2#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAPstatic inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap){ unsigned long nr_pages; unsigned long nr_vmemmap_pages;
if (!pgmap || !is_power_of_2(sizeof(struct page))) return false;
nr_pages = pgmap_vmemmap_nr(pgmap); nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT); /* * For vmemmap optimization with DAX we need minimum 2 vmemmap * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst */ return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR);}/* * If we don't have an architecture override, use the generic rule */#ifndef vmemmap_can_optimize#define vmemmap_can_optimize __vmemmap_can_optimize#endif
#elsestatic inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap){ return false;}#endif
enum mf_flags { MF_COUNT_INCREASED = 1 << 0, MF_ACTION_REQUIRED = 1 << 1, MF_MUST_KILL = 1 << 2, MF_SOFT_OFFLINE = 1 << 3, MF_UNPOISON = 1 << 4, MF_SW_SIMULATED = 1 << 5, MF_NO_RETRY = 1 << 6, MF_MEM_PRE_REMOVE = 1 << 7,};int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, unsigned long count, int mf_flags);extern int memory_failure(unsigned long pfn, int flags);extern int unpoison_memory(unsigned long pfn);extern atomic_long_t num_poisoned_pages __read_mostly;extern int soft_offline_page(unsigned long pfn, int flags);#ifdef CONFIG_MEMORY_FAILURE/* * Sysfs entries for memory failure handling statistics. */extern const struct attribute_group memory_failure_attr_group;extern void memory_failure_queue(unsigned long pfn, int flags);void num_poisoned_pages_inc(unsigned long pfn);void num_poisoned_pages_sub(unsigned long pfn, long i);#elsestatic inline void memory_failure_queue(unsigned long pfn, int flags){}
static inline void num_poisoned_pages_inc(unsigned long pfn){}
static inline void num_poisoned_pages_sub(unsigned long pfn, long i){}#endif
#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)extern void memblk_nr_poison_inc(unsigned long pfn);extern void memblk_nr_poison_sub(unsigned long pfn, long i);#elsestatic inline void memblk_nr_poison_inc(unsigned long pfn){}
static inline void memblk_nr_poison_sub(unsigned long pfn, long i){}#endif
#ifndef arch_memory_failurestatic inline int arch_memory_failure(unsigned long pfn, int flags){ return -ENXIO;}#endif
#ifndef arch_is_platform_pagestatic inline bool arch_is_platform_page(u64 paddr){ return false;}#endif
/* * Error handlers for various types of pages. */enum mf_result { MF_IGNORED, /* Error: cannot be handled */ MF_FAILED, /* Error: handling failed */ MF_DELAYED, /* Will be handled later */ MF_RECOVERED, /* Successfully recovered */};
enum mf_action_page_type { MF_MSG_KERNEL, MF_MSG_KERNEL_HIGH_ORDER, MF_MSG_DIFFERENT_COMPOUND, MF_MSG_HUGE, MF_MSG_FREE_HUGE, MF_MSG_GET_HWPOISON, MF_MSG_UNMAP_FAILED, MF_MSG_DIRTY_SWAPCACHE, MF_MSG_CLEAN_SWAPCACHE, MF_MSG_DIRTY_MLOCKED_LRU, MF_MSG_CLEAN_MLOCKED_LRU, MF_MSG_DIRTY_UNEVICTABLE_LRU, MF_MSG_CLEAN_UNEVICTABLE_LRU, MF_MSG_DIRTY_LRU, MF_MSG_CLEAN_LRU, MF_MSG_TRUNCATED_LRU, MF_MSG_BUDDY, MF_MSG_DAX, MF_MSG_UNSPLIT_THP, MF_MSG_ALREADY_POISONED, MF_MSG_PFN_MAP, MF_MSG_UNKNOWN,};
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)void folio_zero_user(struct folio *folio, unsigned long addr_hint);int copy_user_large_folio(struct folio *dst, struct folio *src, unsigned long addr_hint, struct vm_area_struct *vma);long copy_folio_from_user(struct folio *dst_folio, const void __user *usr_src, bool allow_pagefault);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
#if MAX_NUMNODES > 1void __init setup_nr_node_ids(void);#elsestatic inline void setup_nr_node_ids(void) {}#endif
extern int memcmp_pages(struct page *page1, struct page *page2);
static inline int pages_identical(struct page *page1, struct page *page2){ return !memcmp_pages(page1, page2);}
#ifdef CONFIG_MAPPING_DIRTY_HELPERSunsigned long clean_record_shared_mapping_range(struct address_space *mapping, pgoff_t first_index, pgoff_t nr, pgoff_t bitmap_pgoff, unsigned long *bitmap, pgoff_t *start, pgoff_t *end);
unsigned long wp_shared_mapping_range(struct address_space *mapping, pgoff_t first_index, pgoff_t nr);#endif
#ifdef CONFIG_ANON_VMA_NAMEint set_anon_vma_name(unsigned long addr, unsigned long size, const char __user *uname);#elsestatic inlineint set_anon_vma_name(unsigned long addr, unsigned long size, const char __user *uname){ return -EINVAL;}#endif
#ifdef CONFIG_UNACCEPTED_MEMORY
bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size);void accept_memory(phys_addr_t start, unsigned long size);
#else
static inline bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size){ return false;}
static inline void accept_memory(phys_addr_t start, unsigned long size){}
#endif
static inline bool pfn_is_unaccepted_memory(unsigned long pfn){ return range_contains_unaccepted_memory(pfn << PAGE_SHIFT, PAGE_SIZE);}
void vma_pgtable_walk_begin(struct vm_area_struct *vma);void vma_pgtable_walk_end(struct vm_area_struct *vma);
int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size);int reserve_mem_release_by_name(const char *name);
#ifdef CONFIG_64BITvoid mseal_mmap_page_zero(void);#elsestatic inline void mseal_mmap_page_zero(void) {}#endif
/* * user_alloc_needs_zeroing checks if a user folio from page allocator needs to * be zeroed or not. */static inline bool user_alloc_needs_zeroing(void){ /* * for user folios, arch with cache aliasing requires cache flush and * arc changes folio->flags to make icache coherent with dcache, so * always return false to make caller use * clear_user_page()/clear_user_highpage(). */ return cpu_dcache_is_aliasing() || cpu_icache_is_aliasing() || !static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, &init_on_alloc);}
int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status);int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status);int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
/* * DMA mapping IDs for page_pool * * When DMA-mapping a page, page_pool allocates an ID (from an xarray) and * stashes it in the upper bits of page->pp_magic. We always want to be able to * unambiguously identify page pool pages (using page_pool_page_is_pp()). Non-PP * pages can have arbitrary kernel pointers stored in the same field as pp_magic * (since it overlaps with page->lru.next), so we must ensure that we cannot * mistake a valid kernel pointer with any of the values we write into this * field. * * On architectures that set POISON_POINTER_DELTA, this is already ensured, * since this value becomes part of PP_SIGNATURE; meaning we can just use the * space between the PP_SIGNATURE value (without POISON_POINTER_DELTA), and the * lowest bits of POISON_POINTER_DELTA. On arches where POISON_POINTER_DELTA is * 0, we use the lowest bit of PAGE_OFFSET as the boundary if that value is * known at compile-time. * * If the value of PAGE_OFFSET is not known at compile time, or if it is too * small to leave at least 8 bits available above PP_SIGNATURE, we define the * number of bits to be 0, which turns off the DMA index tracking altogether * (see page_pool_register_dma_index()). */#define PP_DMA_INDEX_SHIFT (1 + __fls(PP_SIGNATURE - POISON_POINTER_DELTA))#if POISON_POINTER_DELTA > 0/* PP_SIGNATURE includes POISON_POINTER_DELTA, so limit the size of the DMA * index to not overlap with that if set */#define PP_DMA_INDEX_BITS MIN(32, __ffs(POISON_POINTER_DELTA) - PP_DMA_INDEX_SHIFT)#else/* Use the lowest bit of PAGE_OFFSET if there's at least 8 bits available; see above */#define PP_DMA_INDEX_MIN_OFFSET (1 << (PP_DMA_INDEX_SHIFT + 8))#define PP_DMA_INDEX_BITS ((__builtin_constant_p(PAGE_OFFSET) && \ PAGE_OFFSET >= PP_DMA_INDEX_MIN_OFFSET && \ !(PAGE_OFFSET & (PP_DMA_INDEX_MIN_OFFSET - 1))) ? \ MIN(32, __ffs(PAGE_OFFSET) - PP_DMA_INDEX_SHIFT) : 0)
#endif
#define PP_DMA_INDEX_MASK GENMASK(PP_DMA_INDEX_BITS + PP_DMA_INDEX_SHIFT - 1, \ PP_DMA_INDEX_SHIFT)
/* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for * the head page of compound page and bit 1 for pfmemalloc page, as well as the * bits used for the DMA index. page_is_pfmemalloc() is checked in * __page_pool_put_page() to avoid recycling the pfmemalloc page. */#define PP_MAGIC_MASK ~(PP_DMA_INDEX_MASK | 0x3UL)
#ifdef CONFIG_PAGE_POOLstatic inline bool page_pool_page_is_pp(const struct page *page){ return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE;}#elsestatic inline bool page_pool_page_is_pp(const struct page *page){ return false;}#endif
#define PAGE_SNAPSHOT_FAITHFUL (1 << 0)#define PAGE_SNAPSHOT_PG_BUDDY (1 << 1)#define PAGE_SNAPSHOT_PG_IDLE (1 << 2)
struct page_snapshot { struct folio folio_snapshot; struct page page_snapshot; unsigned long pfn; unsigned long idx; unsigned long flags;};
static inline bool snapshot_page_is_faithful(const struct page_snapshot *ps){ return ps->flags & PAGE_SNAPSHOT_FAITHFUL;}
void snapshot_page(struct page_snapshot *ps, const struct page *page);
void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, struct vm_area_struct *vma, unsigned long addr, bool uffd_wp);
#endif /* _LINUX_MM_H */