Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_RCULIST_NULLS_H#define _LINUX_RCULIST_NULLS_H
#ifdef __KERNEL__
/* * RCU-protected list version */#include <linux/list_nulls.h>#include <linux/rcupdate.h>
/** * hlist_nulls_del_init_rcu - deletes entry from hash list with re-initialization * @n: the element to delete from the hash list. * * Note: hlist_nulls_unhashed() on the node return true after this. It is * useful for RCU based read lockfree traversal if the writer side * must know if the list entry is still hashed or already unhashed. * * In particular, it means that we can not poison the forward pointers * that may still be used for walking the hash list and we can only * zero the pprev pointer so list_unhashed() will return true after * this. * * The caller must take whatever precautions are necessary (such as * holding appropriate locks) to avoid racing with another * list-mutation primitive, such as hlist_nulls_add_head_rcu() or * hlist_nulls_del_rcu(), running on this same list. However, it is * perfectly legal to run concurrently with the _rcu list-traversal * primitives, such as hlist_nulls_for_each_entry_rcu(). */static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n){ if (!hlist_nulls_unhashed(n)) { __hlist_nulls_del(n); WRITE_ONCE(n->pprev, NULL); }}
/** * hlist_nulls_first_rcu - returns the first element of the hash list. * @head: the head of the list. */#define hlist_nulls_first_rcu(head) \ (*((struct hlist_nulls_node __rcu __force **)&(head)->first))
/** * hlist_nulls_next_rcu - returns the element of the list after @node. * @node: element of the list. */#define hlist_nulls_next_rcu(node) \ (*((struct hlist_nulls_node __rcu __force **)&(node)->next))
/** * hlist_nulls_pprev_rcu - returns the dereferenced pprev of @node. * @node: element of the list. */#define hlist_nulls_pprev_rcu(node) \ (*((struct hlist_nulls_node __rcu __force **)(node)->pprev))
/** * hlist_nulls_del_rcu - deletes entry from hash list without re-initialization * @n: the element to delete from the hash list. * * Note: hlist_nulls_unhashed() on entry does not return true after this, * the entry is in an undefined state. It is useful for RCU based * lockfree traversal. * * In particular, it means that we can not poison the forward * pointers that may still be used for walking the hash list. * * The caller must take whatever precautions are necessary * (such as holding appropriate locks) to avoid racing * with another list-mutation primitive, such as hlist_nulls_add_head_rcu() * or hlist_nulls_del_rcu(), running on this same list. * However, it is perfectly legal to run concurrently with * the _rcu list-traversal primitives, such as * hlist_nulls_for_each_entry(). */static inline void hlist_nulls_del_rcu(struct hlist_nulls_node *n){ __hlist_nulls_del(n); WRITE_ONCE(n->pprev, LIST_POISON2);}
/** * hlist_nulls_add_head_rcu * @n: the element to add to the hash list. * @h: the list to add to. * * Description: * Adds the specified element to the specified hlist_nulls, * while permitting racing traversals. * * The caller must take whatever precautions are necessary * (such as holding appropriate locks) to avoid racing * with another list-mutation primitive, such as hlist_nulls_add_head_rcu() * or hlist_nulls_del_rcu(), running on this same list. * However, it is perfectly legal to run concurrently with * the _rcu list-traversal primitives, such as * hlist_nulls_for_each_entry_rcu(), used to prevent memory-consistency * problems on Alpha CPUs. Regardless of the type of CPU, the * list-traversal primitive must be guarded by rcu_read_lock(). */static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, struct hlist_nulls_head *h){ struct hlist_nulls_node *first = h->first;
WRITE_ONCE(n->next, first); WRITE_ONCE(n->pprev, &h->first); rcu_assign_pointer(hlist_nulls_first_rcu(h), n); if (!is_a_nulls(first)) WRITE_ONCE(first->pprev, &n->next);}
/** * hlist_nulls_add_tail_rcu * @n: the element to add to the hash list. * @h: the list to add to. * * Description: * Adds the specified element to the specified hlist_nulls, * while permitting racing traversals. * * The caller must take whatever precautions are necessary * (such as holding appropriate locks) to avoid racing * with another list-mutation primitive, such as hlist_nulls_add_head_rcu() * or hlist_nulls_del_rcu(), running on this same list. * However, it is perfectly legal to run concurrently with * the _rcu list-traversal primitives, such as * hlist_nulls_for_each_entry_rcu(), used to prevent memory-consistency * problems on Alpha CPUs. Regardless of the type of CPU, the * list-traversal primitive must be guarded by rcu_read_lock(). */static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n, struct hlist_nulls_head *h){ struct hlist_nulls_node *i, *last = NULL;
/* Note: write side code, so rcu accessors are not needed. */ for (i = h->first; !is_a_nulls(i); i = i->next) last = i;
if (last) { WRITE_ONCE(n->next, last->next); WRITE_ONCE(n->pprev, &last->next); rcu_assign_pointer(hlist_nulls_next_rcu(last), n); } else { hlist_nulls_add_head_rcu(n, h); }}
/* after that hlist_nulls_del will work */static inline void hlist_nulls_add_fake(struct hlist_nulls_node *n){ WRITE_ONCE(n->pprev, &n->next); WRITE_ONCE(n->next, (struct hlist_nulls_node *)NULLS_MARKER(NULL));}
/** * hlist_nulls_replace_rcu - replace an old entry by a new one * @old: the element to be replaced * @new: the new element to insert * * Description: * Replace the old entry with the new one in a RCU-protected hlist_nulls, while * permitting racing traversals. * * The caller must take whatever precautions are necessary (such as holding * appropriate locks) to avoid racing with another list-mutation primitive, such * as hlist_nulls_add_head_rcu() or hlist_nulls_del_rcu(), running on this same * list. However, it is perfectly legal to run concurrently with the _rcu * list-traversal primitives, such as hlist_nulls_for_each_entry_rcu(). */static inline void hlist_nulls_replace_rcu(struct hlist_nulls_node *old, struct hlist_nulls_node *new){ struct hlist_nulls_node *next = old->next;
WRITE_ONCE(new->next, next); WRITE_ONCE(new->pprev, old->pprev); rcu_assign_pointer(hlist_nulls_pprev_rcu(new), new); if (!is_a_nulls(next)) WRITE_ONCE(next->pprev, &new->next);}
/** * hlist_nulls_replace_init_rcu - replace an old entry by a new one and * initialize the old * @old: the element to be replaced * @new: the new element to insert * * Description: * Replace the old entry with the new one in a RCU-protected hlist_nulls, while * permitting racing traversals, and reinitialize the old entry. * * Note: @old must be hashed. * * The caller must take whatever precautions are necessary (such as holding * appropriate locks) to avoid racing with another list-mutation primitive, such * as hlist_nulls_add_head_rcu() or hlist_nulls_del_rcu(), running on this same * list. However, it is perfectly legal to run concurrently with the _rcu * list-traversal primitives, such as hlist_nulls_for_each_entry_rcu(). */static inline void hlist_nulls_replace_init_rcu(struct hlist_nulls_node *old, struct hlist_nulls_node *new){ hlist_nulls_replace_rcu(old, new); WRITE_ONCE(old->pprev, NULL);}
/** * hlist_nulls_for_each_entry_rcu - iterate over rcu list of given type * @tpos: the type * to use as a loop cursor. * @pos: the &struct hlist_nulls_node to use as a loop cursor. * @head: the head of the list. * @member: the name of the hlist_nulls_node within the struct. * * The barrier() is needed to make sure compiler doesn't cache first element [1], * as this loop can be restarted [2] * [1] Documentation/memory-barriers.txt around line 1533 * [2] Documentation/RCU/rculist_nulls.rst around line 146 */#define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ for (({barrier();}), \ pos = rcu_dereference_raw(hlist_nulls_first_rcu(head)); \ (!is_a_nulls(pos)) && \ ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos)))
/** * hlist_nulls_for_each_entry_safe - * iterate over list of given type safe against removal of list entry * @tpos: the type * to use as a loop cursor. * @pos: the &struct hlist_nulls_node to use as a loop cursor. * @head: the head of the list. * @member: the name of the hlist_nulls_node within the struct. */#define hlist_nulls_for_each_entry_safe(tpos, pos, head, member) \ for (({barrier();}), \ pos = rcu_dereference_raw(hlist_nulls_first_rcu(head)); \ (!is_a_nulls(pos)) && \ ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); \ pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos)); 1; });)#endif#endif