Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_SWAP_H#define _LINUX_SWAP_H
#include <linux/spinlock.h>#include <linux/linkage.h>#include <linux/mmzone.h>#include <linux/list.h>#include <linux/memcontrol.h>#include <linux/sched.h>#include <linux/node.h>#include <linux/fs.h>#include <linux/pagemap.h>#include <linux/atomic.h>#include <linux/page-flags.h>#include <uapi/linux/mempolicy.h>#include <asm/page.h>
struct notifier_block;
struct bio;
#define SWAP_FLAG_PREFER 0x8000 /* set if swap priority specified */#define SWAP_FLAG_PRIO_MASK 0x7fff#define SWAP_FLAG_DISCARD 0x10000 /* enable discard for swap */#define SWAP_FLAG_DISCARD_ONCE 0x20000 /* discard swap area at swapon-time */#define SWAP_FLAG_DISCARD_PAGES 0x40000 /* discard page-clusters after use */
#define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES)#define SWAP_BATCH 64
static inline int current_is_kswapd(void){ return current->flags & PF_KSWAPD;}
/* * MAX_SWAPFILES defines the maximum number of swaptypes: things which can * be swapped to. The swap type and the offset into that swap type are * encoded into pte's and into pgoff_t's in the swapcache. Using five bits * for the type means that the maximum number of swapcache pages is 27 bits * on 32-bit-pgoff_t architectures. And that assumes that the architecture packs * the type/offset into the pte as 5/27 as well. */#define MAX_SWAPFILES_SHIFT 5
/* * Use some of the swap files numbers for other purposes. This * is a convenient way to hook into the VM to trigger special * actions on faults. */
/* * PTE markers are used to persist information onto PTEs that otherwise * should be a none pte. As its name "PTE" hints, it should only be * applied to the leaves of pgtables. */#define SWP_PTE_MARKER_NUM 1#define SWP_PTE_MARKER (MAX_SWAPFILES + SWP_HWPOISON_NUM + \ SWP_MIGRATION_NUM + SWP_DEVICE_NUM)
/* * Unaddressable device memory support. See include/linux/hmm.h and * Documentation/mm/hmm.rst. Short description is we need struct pages for * device memory that is unaddressable (inaccessible) by CPU, so that we can * migrate part of a process memory to device memory. * * When a page is migrated from CPU to device, we set the CPU page table entry * to a special SWP_DEVICE_{READ|WRITE} entry. * * When a page is mapped by the device for exclusive access we set the CPU page * table entries to a special SWP_DEVICE_EXCLUSIVE entry. */#ifdef CONFIG_DEVICE_PRIVATE#define SWP_DEVICE_NUM 3#define SWP_DEVICE_WRITE (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM)#define SWP_DEVICE_READ (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM+1)#define SWP_DEVICE_EXCLUSIVE (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM+2)#else#define SWP_DEVICE_NUM 0#endif
/* * Page migration support. * * SWP_MIGRATION_READ_EXCLUSIVE is only applicable to anonymous pages and * indicates that the referenced (part of) an anonymous page is exclusive to * a single process. For SWP_MIGRATION_WRITE, that information is implicit: * (part of) an anonymous page that are mapped writable are exclusive to a * single process. */#ifdef CONFIG_MIGRATION#define SWP_MIGRATION_NUM 3#define SWP_MIGRATION_READ (MAX_SWAPFILES + SWP_HWPOISON_NUM)#define SWP_MIGRATION_READ_EXCLUSIVE (MAX_SWAPFILES + SWP_HWPOISON_NUM + 1)#define SWP_MIGRATION_WRITE (MAX_SWAPFILES + SWP_HWPOISON_NUM + 2)#else#define SWP_MIGRATION_NUM 0#endif
/* * Handling of hardware poisoned pages with memory corruption. */#ifdef CONFIG_MEMORY_FAILURE#define SWP_HWPOISON_NUM 1#define SWP_HWPOISON MAX_SWAPFILES#else#define SWP_HWPOISON_NUM 0#endif
#define MAX_SWAPFILES \ ((1 << MAX_SWAPFILES_SHIFT) - SWP_DEVICE_NUM - \ SWP_MIGRATION_NUM - SWP_HWPOISON_NUM - \ SWP_PTE_MARKER_NUM)
/* * Magic header for a swap area. The first part of the union is * what the swap magic looks like for the old (limited to 128MB) * swap area format, the second part of the union adds - in the * old reserved area - some extra information. Note that the first * kilobyte is reserved for boot loader or disk label stuff... * * Having the magic at the end of the PAGE_SIZE makes detecting swap * areas somewhat tricky on machines that support multiple page sizes. * For 2.5 we'll probably want to move the magic to just beyond the * bootbits... */union swap_header { struct { char reserved[PAGE_SIZE - 10]; char magic[10]; /* SWAP-SPACE or SWAPSPACE2 */ } magic; struct { char bootbits[1024]; /* Space for disklabel etc. */ __u32 version; __u32 last_page; __u32 nr_badpages; unsigned char sws_uuid[16]; unsigned char sws_volume[16]; __u32 padding[117]; __u32 badpages[1]; } info;};
/* * current->reclaim_state points to one of these when a task is running * memory reclaim */struct reclaim_state { /* pages reclaimed outside of LRU-based reclaim */ unsigned long reclaimed;#ifdef CONFIG_LRU_GEN /* per-thread mm walk data */ struct lru_gen_mm_walk *mm_walk;#endif};
/* * mm_account_reclaimed_pages(): account reclaimed pages outside of LRU-based * reclaim * @pages: number of pages reclaimed * * If the current process is undergoing a reclaim operation, increment the * number of reclaimed pages by @pages. */static inline void mm_account_reclaimed_pages(unsigned long pages){ if (current->reclaim_state) current->reclaim_state->reclaimed += pages;}
#ifdef __KERNEL__
struct address_space;struct sysinfo;struct writeback_control;struct zone;
/* * A swap extent maps a range of a swapfile's PAGE_SIZE pages onto a range of * disk blocks. A rbtree of swap extents maps the entire swapfile (Where the * term `swapfile' refers to either a blockdevice or an IS_REG file). Apart * from setup, they're handled identically. * * We always assume that blocks are of size PAGE_SIZE. */struct swap_extent { struct rb_node rb_node; pgoff_t start_page; pgoff_t nr_pages; sector_t start_block;};
/* * Max bad pages in the new format.. */#define MAX_SWAP_BADPAGES \ ((offsetof(union swap_header, magic.magic) - \ offsetof(union swap_header, info.badpages)) / sizeof(int))
enum { SWP_USED = (1 << 0), /* is slot in swap_info[] used? */ SWP_WRITEOK = (1 << 1), /* ok to write to this swap? */ SWP_DISCARDABLE = (1 << 2), /* blkdev support discard */ SWP_DISCARDING = (1 << 3), /* now discarding a free cluster */ SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */ SWP_BLKDEV = (1 << 6), /* its a block device */ SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */ SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */ SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */ SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */ SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */ SWP_SYNCHRONOUS_IO = (1 << 12), /* synchronous IO is efficient */ SWP_HIBERNATION = (1 << 13), /* pinned for hibernation */ /* add others here before... */};
#define SWAP_CLUSTER_MAX 32UL#define SWAP_CLUSTER_MAX_SKIPPED (SWAP_CLUSTER_MAX << 10)#define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX
/* * The first page in the swap file is the swap header, which is always marked * bad to prevent it from being allocated as an entry. This also prevents the * cluster to which it belongs being marked free. Therefore 0 is safe to use as * a sentinel to indicate an entry is not valid. */#define SWAP_ENTRY_INVALID 0
#ifdef CONFIG_THP_SWAP#define SWAP_NR_ORDERS (PMD_ORDER + 1)#else#define SWAP_NR_ORDERS 1#endif
/* * We keep using same cluster for rotational device so IO will be sequential. * The purpose is to optimize SWAP throughput on these device. */struct swap_sequential_cluster { unsigned int next[SWAP_NR_ORDERS]; /* Likely next allocation offset */};
/* * The in-memory structure used to track swap areas. */struct swap_info_struct { struct percpu_ref users; /* indicate and keep swap device valid. */ unsigned long flags; /* SWP_USED etc: see above */ signed short prio; /* swap priority of this type */ struct plist_node list; /* entry in swap_active_head */ signed char type; /* strange name for an index */ unsigned int max; /* size of this swap device */ struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */ struct list_head free_clusters; /* free clusters list */ struct list_head full_clusters; /* full clusters list */ struct list_head nonfull_clusters[SWAP_NR_ORDERS]; /* list of cluster that contains at least one free slot */ struct list_head frag_clusters[SWAP_NR_ORDERS]; /* list of cluster that are fragmented or contented */ unsigned int pages; /* total of usable pages of swap */ atomic_long_t inuse_pages; /* number of those currently in use */ struct swap_sequential_cluster *global_cluster; /* Use one global cluster for rotating device */ spinlock_t global_cluster_lock; /* Serialize usage of global cluster */ struct rb_root swap_extent_root;/* root of the swap extent rbtree */ struct block_device *bdev; /* swap device or bdev of swap file */ struct file *swap_file; /* seldom referenced */ struct completion comp; /* seldom referenced */ spinlock_t lock; /* * protect map scan related fields like * inuse_pages and all cluster lists. * Other fields are only changed * at swapon/swapoff, so are protected * by swap_lock. changing flags need * hold this lock and swap_lock. If * both locks need hold, hold swap_lock * first. */ struct work_struct discard_work; /* discard worker */ struct work_struct reclaim_work; /* reclaim worker */ struct list_head discard_clusters; /* discard clusters list */ struct plist_node avail_list; /* entry in swap_avail_head */};
static inline swp_entry_t page_swap_entry(struct page *page){ struct folio *folio = page_folio(page); swp_entry_t entry = folio->swap;
entry.val += folio_page_idx(folio, page); return entry;}
/* linux/mm/workingset.c */bool workingset_test_recent(void *shadow, bool file, bool *workingset, bool flush);void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg);void workingset_refault(struct folio *folio, void *shadow);void workingset_activation(struct folio *folio);
/* linux/mm/page_alloc.c */extern unsigned long totalreserve_pages;
/* Definition of global_zone_page_state not available yet */#define nr_free_pages() global_zone_page_state(NR_FREE_PAGES)
/* linux/mm/swap.c */void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, unsigned int nr_io, unsigned int nr_rotated);void lru_note_cost_refault(struct folio *);void folio_add_lru(struct folio *);void folio_add_lru_vma(struct folio *, struct vm_area_struct *);void mark_page_accessed(struct page *);void folio_mark_accessed(struct folio *);
static inline bool folio_may_be_lru_cached(struct folio *folio){ /* * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. * Holding small numbers of low-order mTHP folios in per-CPU LRU cache * will be sensible, but nobody has implemented and tested that yet. */ return !folio_test_large(folio);}
extern atomic_t lru_disable_count;
static inline bool lru_cache_disabled(void){ return atomic_read(&lru_disable_count);}
static inline void lru_cache_enable(void){ atomic_dec(&lru_disable_count);}
extern void lru_cache_disable(void);extern void lru_add_drain(void);extern void lru_add_drain_cpu(int cpu);extern void lru_add_drain_cpu_zone(struct zone *zone);extern void lru_add_drain_all(void);void folio_deactivate(struct folio *folio);void folio_mark_lazyfree(struct folio *folio);extern void swap_setup(void);
/* linux/mm/vmscan.c */extern unsigned long zone_reclaimable_pages(struct zone *zone);extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, gfp_t gfp_mask, nodemask_t *mask);unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx);
#define MEMCG_RECLAIM_MAY_SWAP (1 << 1)#define MEMCG_RECLAIM_PROACTIVE (1 << 2)#define MIN_SWAPPINESS 0#define MAX_SWAPPINESS 200
/* Just reclaim from anon folios in proactive memory reclaim */#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1)
extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages, gfp_t gfp_mask, unsigned int reclaim_options, int *swappiness);extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, gfp_t gfp_mask, bool noswap, pg_data_t *pgdat, unsigned long *nr_scanned);extern unsigned long shrink_all_memory(unsigned long nr_pages);extern int vm_swappiness;long remove_mapping(struct address_space *mapping, struct folio *folio);
#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)extern int reclaim_register_node(struct node *node);extern void reclaim_unregister_node(struct node *node);
#else
static inline int reclaim_register_node(struct node *node){ return 0;}
static inline void reclaim_unregister_node(struct node *node){}#endif /* CONFIG_SYSFS && CONFIG_NUMA */
#ifdef CONFIG_NUMAextern int sysctl_min_unmapped_ratio;extern int sysctl_min_slab_ratio;#endif
void check_move_unevictable_folios(struct folio_batch *fbatch);
extern void __meminit kswapd_run(int nid);extern void __meminit kswapd_stop(int nid);
#ifdef CONFIG_SWAP
int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block);int generic_swapfile_activate(struct swap_info_struct *, struct file *, sector_t *);
static inline unsigned long total_swapcache_pages(void){ return global_node_page_state(NR_SWAPCACHE);}
void free_swap_cache(struct folio *folio);void free_folio_and_swap_cache(struct folio *folio);void free_pages_and_swap_cache(struct encoded_page **, int);/* linux/mm/swapfile.c */extern atomic_long_t nr_swap_pages;extern long total_swap_pages;extern atomic_t nr_rotate_swap;
/* Swap 50% full? Release swapcache more aggressively.. */static inline bool vm_swap_full(void){ return atomic_long_read(&nr_swap_pages) * 2 < total_swap_pages;}
static inline long get_nr_swap_pages(void){ return atomic_long_read(&nr_swap_pages);}
extern void si_swapinfo(struct sysinfo *);extern int pin_hibernation_swap_type(dev_t device, sector_t offset);extern void unpin_hibernation_swap_type(int type);extern int find_hibernation_swap_type(dev_t device, sector_t offset);int find_first_swap(dev_t *device);extern unsigned int count_swap_pages(int, int);extern sector_t swapdev_block(int, pgoff_t);extern int __swap_count(swp_entry_t entry);extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry);extern int swp_swapcount(swp_entry_t entry);struct backing_dev_info;extern struct swap_info_struct *get_swap_device(swp_entry_t entry);sector_t swap_folio_sector(struct folio *folio);
/* * If there is an existing swap slot reference (swap entry) and the caller * guarantees that there is no race modification of it (e.g., PTL * protecting the swap entry in page table; shmem's cmpxchg protects t * he swap entry in shmem mapping), these two helpers below can be used * to put/dup the entries directly. * * All entries must be allocated by folio_alloc_swap(). And they must have * a swap count > 1. See comments of folio_*_swap helpers for more info. */int swap_dup_entry_direct(swp_entry_t entry);void swap_put_entries_direct(swp_entry_t entry, int nr);
/* * folio_free_swap tries to free the swap entries pinned by a swap cache * folio, it has to be here to be called by other components. */bool folio_free_swap(struct folio *folio);
/* Allocate / free (hibernation) exclusive entries */swp_entry_t swap_alloc_hibernation_slot(int type);void swap_free_hibernation_slot(swp_entry_t entry);
static inline void put_swap_device(struct swap_info_struct *si){ percpu_ref_put(&si->users);}
#else /* CONFIG_SWAP */static inline struct swap_info_struct *get_swap_device(swp_entry_t entry){ return NULL;}
static inline void put_swap_device(struct swap_info_struct *si){}
#define get_nr_swap_pages() 0L#define total_swap_pages 0L#define total_swapcache_pages() 0UL#define vm_swap_full() 0
#define si_swapinfo(val) \ do { (val)->freeswap = (val)->totalswap = 0; } while (0)#define free_folio_and_swap_cache(folio) \ folio_put(folio)#define free_pages_and_swap_cache(pages, nr) \ release_pages((pages), (nr));
static inline void free_swap_cache(struct folio *folio){}
static inline int swap_dup_entry_direct(swp_entry_t ent){ return 0;}
static inline void swap_put_entries_direct(swp_entry_t ent, int nr){}
static inline int __swap_count(swp_entry_t entry){ return 0;}
static inline bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry){ return false;}
static inline int swp_swapcount(swp_entry_t entry){ return 0;}
static inline bool folio_free_swap(struct folio *folio){ return false;}
static inline int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block){ return -EINVAL;}#endif /* CONFIG_SWAP */#ifdef CONFIG_MEMCGstatic inline int mem_cgroup_swappiness(struct mem_cgroup *memcg){ /* Cgroup2 doesn't have per-cgroup swappiness */ if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) return READ_ONCE(vm_swappiness);
/* root ? */ if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg)) return READ_ONCE(vm_swappiness);
return READ_ONCE(memcg->swappiness);}
void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);#elsestatic inline int mem_cgroup_swappiness(struct mem_cgroup *mem){ return READ_ONCE(vm_swappiness);}#endif
#if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp);static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp){ if (mem_cgroup_disabled()) return; __folio_throttle_swaprate(folio, gfp);}#elsestatic inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp){}#endif
#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)int __mem_cgroup_try_charge_swap(struct folio *folio);static inline int mem_cgroup_try_charge_swap(struct folio *folio){ if (mem_cgroup_disabled()) return 0; return __mem_cgroup_try_charge_swap(folio);}
extern void __mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages);static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages){ if (mem_cgroup_disabled()) return; __mem_cgroup_uncharge_swap(id, nr_pages);}
extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);extern bool mem_cgroup_swap_full(struct folio *folio);#elsestatic inline int mem_cgroup_try_charge_swap(struct folio *folio){ return 0;}
static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages){}
static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg){ return get_nr_swap_pages();}
static inline bool mem_cgroup_swap_full(struct folio *folio){ return vm_swap_full();}#endif
/* for_each_managed_zone_pgdat - helper macro to iterate over all managed zones in a pgdat up to * and including the specified highidx * @zone: The current zone in the iterator * @pgdat: The pgdat which node_zones are being iterated * @idx: The index variable * @highidx: The index of the highest zone to return * * This macro iterates through all managed zones up to and including the specified highidx. * The zone iterator enters an invalid state after macro call and must be reinitialized * before it can be used again. */#define for_each_managed_zone_pgdat(zone, pgdat, idx, highidx) \ for ((idx) = 0, (zone) = (pgdat)->node_zones; \ (idx) <= (highidx); \ (idx)++, (zone)++) \ if (!managed_zone(zone)) \ continue; \ else
#endif /* __KERNEL__*/#endif /* _LINUX_SWAP_H */