tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib, Add lock-less NULL terminated single list
Cmpxchg is used to implement adding new entry to the list, deleting
all entries from the list, deleting first entry of the list and some
other operations.
Because this is a single list, so the tail can not be accessed in O(1).
If there are multiple producers and multiple consumers, llist_add can
be used in producers and llist_del_all can be used in consumers. They
can work simultaneously without lock. But llist_del_first can not be
used here. Because llist_del_first depends on list->first->next does
not changed if list->first is not changed during its operation, but
llist_del_first, llist_add, llist_add (or llist_del_all, llist_add,
llist_add) sequence in another consumer may violate that.
If there are multiple producers and one consumer, llist_add can be
used in producers and llist_del_all or llist_del_first can be used in
the consumer.
This can be summarized as follow:
| add | del_first | del_all
add | - | - | -
del_first | | L | L
del_all | | | -
Where "-" stands for no lock is needed, while "L" stands for lock is
needed.
The list entries deleted via llist_del_all can be traversed with
traversing function such as llist_for_each etc. But the list entries
can not be traversed safely before deleted from the list. The order
of deleted entries is from the newest to the oldest added one. If you
want to traverse from the oldest to the newest, you must reverse the
order by yourself before traversing.
The basic atomic operation of this list is cmpxchg on long. On
architectures that don't have NMI-safe cmpxchg implementation, the
list can NOT be used in NMI handler. So code uses the list in NMI
handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
Signed-off-by: Huang Ying <ying.huang@intel.com>
Reviewed-by: Andi Kleen <ak@linux.intel.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Len Brown <len.brown@intel.com>
authored by
Huang Ying
and committed by
Len Brown
f49f23ab
df013ffb
+260
+126
include/linux/llist.h
+126
include/linux/llist.h
···
1
+
#ifndef LLIST_H
2
+
#define LLIST_H
3
+
/*
4
+
* Lock-less NULL terminated single linked list
5
+
*
6
+
* If there are multiple producers and multiple consumers, llist_add
7
+
* can be used in producers and llist_del_all can be used in
8
+
* consumers. They can work simultaneously without lock. But
9
+
* llist_del_first can not be used here. Because llist_del_first
10
+
* depends on list->first->next does not changed if list->first is not
11
+
* changed during its operation, but llist_del_first, llist_add,
12
+
* llist_add (or llist_del_all, llist_add, llist_add) sequence in
13
+
* another consumer may violate that.
14
+
*
15
+
* If there are multiple producers and one consumer, llist_add can be
16
+
* used in producers and llist_del_all or llist_del_first can be used
17
+
* in the consumer.
18
+
*
19
+
* This can be summarized as follow:
20
+
*
21
+
* | add | del_first | del_all
22
+
* add | - | - | -
23
+
* del_first | | L | L
24
+
* del_all | | | -
25
+
*
26
+
* Where "-" stands for no lock is needed, while "L" stands for lock
27
+
* is needed.
28
+
*
29
+
* The list entries deleted via llist_del_all can be traversed with
30
+
* traversing function such as llist_for_each etc. But the list
31
+
* entries can not be traversed safely before deleted from the list.
32
+
* The order of deleted entries is from the newest to the oldest added
33
+
* one. If you want to traverse from the oldest to the newest, you
34
+
* must reverse the order by yourself before traversing.
35
+
*
36
+
* The basic atomic operation of this list is cmpxchg on long. On
37
+
* architectures that don't have NMI-safe cmpxchg implementation, the
38
+
* list can NOT be used in NMI handler. So code uses the list in NMI
39
+
* handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
40
+
*/
41
+
42
+
struct llist_head {
43
+
struct llist_node *first;
44
+
};
45
+
46
+
struct llist_node {
47
+
struct llist_node *next;
48
+
};
49
+
50
+
#define LLIST_HEAD_INIT(name) { NULL }
51
+
#define LLIST_HEAD(name) struct llist_head name = LLIST_HEAD_INIT(name)
52
+
53
+
/**
54
+
* init_llist_head - initialize lock-less list head
55
+
* @head: the head for your lock-less list
56
+
*/
57
+
static inline void init_llist_head(struct llist_head *list)
58
+
{
59
+
list->first = NULL;
60
+
}
61
+
62
+
/**
63
+
* llist_entry - get the struct of this entry
64
+
* @ptr: the &struct llist_node pointer.
65
+
* @type: the type of the struct this is embedded in.
66
+
* @member: the name of the llist_node within the struct.
67
+
*/
68
+
#define llist_entry(ptr, type, member) \
69
+
container_of(ptr, type, member)
70
+
71
+
/**
72
+
* llist_for_each - iterate over some deleted entries of a lock-less list
73
+
* @pos: the &struct llist_node to use as a loop cursor
74
+
* @node: the first entry of deleted list entries
75
+
*
76
+
* In general, some entries of the lock-less list can be traversed
77
+
* safely only after being deleted from list, so start with an entry
78
+
* instead of list head.
79
+
*
80
+
* If being used on entries deleted from lock-less list directly, the
81
+
* traverse order is from the newest to the oldest added entry. If
82
+
* you want to traverse from the oldest to the newest, you must
83
+
* reverse the order by yourself before traversing.
84
+
*/
85
+
#define llist_for_each(pos, node) \
86
+
for ((pos) = (node); pos; (pos) = (pos)->next)
87
+
88
+
/**
89
+
* llist_for_each_entry - iterate over some deleted entries of lock-less list of given type
90
+
* @pos: the type * to use as a loop cursor.
91
+
* @node: the fist entry of deleted list entries.
92
+
* @member: the name of the llist_node with the struct.
93
+
*
94
+
* In general, some entries of the lock-less list can be traversed
95
+
* safely only after being removed from list, so start with an entry
96
+
* instead of list head.
97
+
*
98
+
* If being used on entries deleted from lock-less list directly, the
99
+
* traverse order is from the newest to the oldest added entry. If
100
+
* you want to traverse from the oldest to the newest, you must
101
+
* reverse the order by yourself before traversing.
102
+
*/
103
+
#define llist_for_each_entry(pos, node, member) \
104
+
for ((pos) = llist_entry((node), typeof(*(pos)), member); \
105
+
&(pos)->member != NULL; \
106
+
(pos) = llist_entry((pos)->member.next, typeof(*(pos)), member))
107
+
108
+
/**
109
+
* llist_empty - tests whether a lock-less list is empty
110
+
* @head: the list to test
111
+
*
112
+
* Not guaranteed to be accurate or up to date. Just a quick way to
113
+
* test whether the list is empty without deleting something from the
114
+
* list.
115
+
*/
116
+
static inline int llist_empty(const struct llist_head *head)
117
+
{
118
+
return ACCESS_ONCE(head->first) == NULL;
119
+
}
120
+
121
+
void llist_add(struct llist_node *new, struct llist_head *head);
122
+
void llist_add_batch(struct llist_node *new_first, struct llist_node *new_last,
123
+
struct llist_head *head);
124
+
struct llist_node *llist_del_first(struct llist_head *head);
125
+
struct llist_node *llist_del_all(struct llist_head *head);
126
+
#endif /* LLIST_H */
+3
lib/Kconfig
+3
lib/Kconfig
+2
lib/Makefile
+2
lib/Makefile
+129
lib/llist.c
+129
lib/llist.c
···
1
+
/*
2
+
* Lock-less NULL terminated single linked list
3
+
*
4
+
* The basic atomic operation of this list is cmpxchg on long. On
5
+
* architectures that don't have NMI-safe cmpxchg implementation, the
6
+
* list can NOT be used in NMI handler. So code uses the list in NMI
7
+
* handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
8
+
*
9
+
* Copyright 2010,2011 Intel Corp.
10
+
* Author: Huang Ying <ying.huang@intel.com>
11
+
*
12
+
* This program is free software; you can redistribute it and/or
13
+
* modify it under the terms of the GNU General Public License version
14
+
* 2 as published by the Free Software Foundation;
15
+
*
16
+
* This program is distributed in the hope that it will be useful,
17
+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
18
+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19
+
* GNU General Public License for more details.
20
+
*
21
+
* You should have received a copy of the GNU General Public License
22
+
* along with this program; if not, write to the Free Software
23
+
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24
+
*/
25
+
#include <linux/kernel.h>
26
+
#include <linux/module.h>
27
+
#include <linux/interrupt.h>
28
+
#include <linux/llist.h>
29
+
30
+
#include <asm/system.h>
31
+
32
+
/**
33
+
* llist_add - add a new entry
34
+
* @new: new entry to be added
35
+
* @head: the head for your lock-less list
36
+
*/
37
+
void llist_add(struct llist_node *new, struct llist_head *head)
38
+
{
39
+
struct llist_node *entry, *old_entry;
40
+
41
+
#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
42
+
BUG_ON(in_nmi());
43
+
#endif
44
+
45
+
entry = head->first;
46
+
do {
47
+
old_entry = entry;
48
+
new->next = entry;
49
+
cpu_relax();
50
+
} while ((entry = cmpxchg(&head->first, old_entry, new)) != old_entry);
51
+
}
52
+
EXPORT_SYMBOL_GPL(llist_add);
53
+
54
+
/**
55
+
* llist_add_batch - add several linked entries in batch
56
+
* @new_first: first entry in batch to be added
57
+
* @new_last: last entry in batch to be added
58
+
* @head: the head for your lock-less list
59
+
*/
60
+
void llist_add_batch(struct llist_node *new_first, struct llist_node *new_last,
61
+
struct llist_head *head)
62
+
{
63
+
struct llist_node *entry, *old_entry;
64
+
65
+
#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
66
+
BUG_ON(in_nmi());
67
+
#endif
68
+
69
+
entry = head->first;
70
+
do {
71
+
old_entry = entry;
72
+
new_last->next = entry;
73
+
cpu_relax();
74
+
} while ((entry = cmpxchg(&head->first, old_entry, new_first)) != old_entry);
75
+
}
76
+
EXPORT_SYMBOL_GPL(llist_add_batch);
77
+
78
+
/**
79
+
* llist_del_first - delete the first entry of lock-less list
80
+
* @head: the head for your lock-less list
81
+
*
82
+
* If list is empty, return NULL, otherwise, return the first entry
83
+
* deleted, this is the newest added one.
84
+
*
85
+
* Only one llist_del_first user can be used simultaneously with
86
+
* multiple llist_add users without lock. Because otherwise
87
+
* llist_del_first, llist_add, llist_add (or llist_del_all, llist_add,
88
+
* llist_add) sequence in another user may change @head->first->next,
89
+
* but keep @head->first. If multiple consumers are needed, please
90
+
* use llist_del_all or use lock between consumers.
91
+
*/
92
+
struct llist_node *llist_del_first(struct llist_head *head)
93
+
{
94
+
struct llist_node *entry, *old_entry, *next;
95
+
96
+
#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
97
+
BUG_ON(in_nmi());
98
+
#endif
99
+
100
+
entry = head->first;
101
+
do {
102
+
if (entry == NULL)
103
+
return NULL;
104
+
old_entry = entry;
105
+
next = entry->next;
106
+
cpu_relax();
107
+
} while ((entry = cmpxchg(&head->first, old_entry, next)) != old_entry);
108
+
109
+
return entry;
110
+
}
111
+
EXPORT_SYMBOL_GPL(llist_del_first);
112
+
113
+
/**
114
+
* llist_del_all - delete all entries from lock-less list
115
+
* @head: the head of lock-less list to delete all entries
116
+
*
117
+
* If list is empty, return NULL, otherwise, delete all entries and
118
+
* return the pointer to the first entry. The order of entries
119
+
* deleted is from the newest to the oldest added one.
120
+
*/
121
+
struct llist_node *llist_del_all(struct llist_head *head)
122
+
{
123
+
#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
124
+
BUG_ON(in_nmi());
125
+
#endif
126
+
127
+
return xchg(&head->first, NULL);
128
+
}
129
+
EXPORT_SYMBOL_GPL(llist_del_all);