Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
C
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_PAGE_REF_H#define _LINUX_PAGE_REF_H
#include <linux/atomic.h>#include <linux/mm_types.h>#include <linux/page-flags.h>#include <linux/tracepoint-defs.h>
DECLARE_TRACEPOINT(page_ref_set);DECLARE_TRACEPOINT(page_ref_mod);DECLARE_TRACEPOINT(page_ref_mod_and_test);DECLARE_TRACEPOINT(page_ref_mod_and_return);DECLARE_TRACEPOINT(page_ref_mod_unless);DECLARE_TRACEPOINT(page_ref_freeze);DECLARE_TRACEPOINT(page_ref_unfreeze);
#ifdef CONFIG_DEBUG_PAGE_REF
/* * Ideally we would want to use the trace_<tracepoint>_enabled() helper * functions. But due to include header file issues, that is not * feasible. Instead we have to open code the static key functions. * * See trace_##name##_enabled(void) in include/linux/tracepoint.h */#define page_ref_tracepoint_active(t) tracepoint_enabled(t)
extern void __page_ref_set(struct page *page, int v);extern void __page_ref_mod(struct page *page, int v);extern void __page_ref_mod_and_test(struct page *page, int v, int ret);extern void __page_ref_mod_and_return(struct page *page, int v, int ret);extern void __page_ref_mod_unless(struct page *page, int v, int u);extern void __page_ref_freeze(struct page *page, int v, int ret);extern void __page_ref_unfreeze(struct page *page, int v);
#else
#define page_ref_tracepoint_active(t) false
static inline void __page_ref_set(struct page *page, int v){}static inline void __page_ref_mod(struct page *page, int v){}static inline void __page_ref_mod_and_test(struct page *page, int v, int ret){}static inline void __page_ref_mod_and_return(struct page *page, int v, int ret){}static inline void __page_ref_mod_unless(struct page *page, int v, int u){}static inline void __page_ref_freeze(struct page *page, int v, int ret){}static inline void __page_ref_unfreeze(struct page *page, int v){}
#endif
static inline int page_ref_count(const struct page *page){ return atomic_read(&page->_refcount);}
/** * folio_ref_count - The reference count on this folio. * @folio: The folio. * * Folios contain a reference count. When that reference count reaches * zero, the folio is referred to as frozen. At this point, it will * usually be returned to the memory allocator, but some parts of the * kernel freeze folios in order to perform unusual operations on them * such as splitting or migration. * * The refcount is usually incremented by calls to folio_get() and * decremented by calls to folio_put(). Some typical users of the * folio refcount: * * - Each reference from a page table * - The page cache * - Filesystem private data * - The LRU list * - Pipes * - Direct IO which references this page in the process address space * * The reference count has three components: expected, temporary and * spurious. The expected reference count of a folio is that which * we would logically expect it to be from just reading the code. * Temporary refcounts are gained by threads which need a temporary * reference to make sure the folio isn't reallocated while they use it. * Spurious refcounts are gained by threads which, thanks to RCU walks * of the page tables or file cache, find a stale pointer to a folio. * These threads will drop the refcount after discoveering the pointer * is stale, but it can surprise other users to see the spurious refcount * on a freshly allocated folio (eg they may see a refcount of 2 instead * of 1). * * Return: The number of references to this folio. */static inline int folio_ref_count(const struct folio *folio){ return page_ref_count(&folio->page);}
static inline int page_count(const struct page *page){ return folio_ref_count(page_folio(page));}
static inline void set_page_count(struct page *page, int v){ atomic_set(&page->_refcount, v); if (page_ref_tracepoint_active(page_ref_set)) __page_ref_set(page, v);}
static inline void folio_set_count(struct folio *folio, int v){ set_page_count(&folio->page, v);}
/* * Setup the page count before being freed into the page allocator for * the first time (boot or memory hotplug) */static inline void init_page_count(struct page *page){ set_page_count(page, 1);}
static inline void page_ref_add(struct page *page, int nr){ atomic_add(nr, &page->_refcount); if (page_ref_tracepoint_active(page_ref_mod)) __page_ref_mod(page, nr);}
static inline void folio_ref_add(struct folio *folio, int nr){ page_ref_add(&folio->page, nr);}
static inline void page_ref_sub(struct page *page, int nr){ atomic_sub(nr, &page->_refcount); if (page_ref_tracepoint_active(page_ref_mod)) __page_ref_mod(page, -nr);}
static inline void folio_ref_sub(struct folio *folio, int nr){ page_ref_sub(&folio->page, nr);}
static inline int folio_ref_sub_return(struct folio *folio, int nr){ int ret = atomic_sub_return(nr, &folio->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_return)) __page_ref_mod_and_return(&folio->page, -nr, ret); return ret;}
static inline void page_ref_inc(struct page *page){ atomic_inc(&page->_refcount); if (page_ref_tracepoint_active(page_ref_mod)) __page_ref_mod(page, 1);}
static inline void folio_ref_inc(struct folio *folio){ page_ref_inc(&folio->page);}
static inline void page_ref_dec(struct page *page){ atomic_dec(&page->_refcount); if (page_ref_tracepoint_active(page_ref_mod)) __page_ref_mod(page, -1);}
static inline void folio_ref_dec(struct folio *folio){ page_ref_dec(&folio->page);}
static inline int page_ref_sub_and_test(struct page *page, int nr){ int ret = atomic_sub_and_test(nr, &page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_test)) __page_ref_mod_and_test(page, -nr, ret); return ret;}
static inline int folio_ref_sub_and_test(struct folio *folio, int nr){ return page_ref_sub_and_test(&folio->page, nr);}
static inline int page_ref_inc_return(struct page *page){ int ret = atomic_inc_return(&page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_return)) __page_ref_mod_and_return(page, 1, ret); return ret;}
static inline int folio_ref_inc_return(struct folio *folio){ return page_ref_inc_return(&folio->page);}
static inline int page_ref_dec_and_test(struct page *page){ int ret = atomic_dec_and_test(&page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_test)) __page_ref_mod_and_test(page, -1, ret); return ret;}
static inline int folio_ref_dec_and_test(struct folio *folio){ return page_ref_dec_and_test(&folio->page);}
static inline int page_ref_dec_return(struct page *page){ int ret = atomic_dec_return(&page->_refcount);
if (page_ref_tracepoint_active(page_ref_mod_and_return)) __page_ref_mod_and_return(page, -1, ret); return ret;}
static inline int folio_ref_dec_return(struct folio *folio){ return page_ref_dec_return(&folio->page);}
static inline bool page_ref_add_unless_zero(struct page *page, int nr){ bool ret = atomic_add_unless(&page->_refcount, nr, 0);
if (page_ref_tracepoint_active(page_ref_mod_unless)) __page_ref_mod_unless(page, nr, ret); return ret;}
static inline bool folio_ref_add_unless_zero(struct folio *folio, int nr){ return page_ref_add_unless_zero(&folio->page, nr);}
/** * folio_try_get - Attempt to increase the refcount on a folio. * @folio: The folio. * * If you do not already have a reference to a folio, you can attempt to * get one using this function. It may fail if, for example, the folio * has been freed since you found a pointer to it, or it is frozen for * the purposes of splitting or migration. * * Return: True if the reference count was successfully incremented. */static inline bool folio_try_get(struct folio *folio){ return folio_ref_add_unless_zero(folio, 1);}
static inline bool folio_ref_try_add(struct folio *folio, int count){ return folio_ref_add_unless_zero(folio, count);}
static inline int page_ref_freeze(struct page *page, int count){ int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count);
if (page_ref_tracepoint_active(page_ref_freeze)) __page_ref_freeze(page, count, ret); return ret;}
static inline int folio_ref_freeze(struct folio *folio, int count){ return page_ref_freeze(&folio->page, count);}
static inline void page_ref_unfreeze(struct page *page, int count){ VM_BUG_ON_PAGE(page_count(page) != 0, page); VM_BUG_ON(count == 0);
atomic_set_release(&page->_refcount, count); if (page_ref_tracepoint_active(page_ref_unfreeze)) __page_ref_unfreeze(page, count);}
static inline void folio_ref_unfreeze(struct folio *folio, int count){ page_ref_unfreeze(&folio->page, count);}#endif