Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1#ifndef _LINUX_PERCPU_RWSEM_H
2#define _LINUX_PERCPU_RWSEM_H
3
4#include <linux/mutex.h>
5#include <linux/percpu.h>
6#include <linux/rcupdate.h>
7#include <linux/delay.h>
8
9struct percpu_rw_semaphore {
10 unsigned __percpu *counters;
11 bool locked;
12 struct mutex mtx;
13};
14
15#define light_mb() barrier()
16#define heavy_mb() synchronize_sched_expedited()
17
18static inline void percpu_down_read(struct percpu_rw_semaphore *p)
19{
20 rcu_read_lock_sched();
21 if (unlikely(p->locked)) {
22 rcu_read_unlock_sched();
23 mutex_lock(&p->mtx);
24 this_cpu_inc(*p->counters);
25 mutex_unlock(&p->mtx);
26 return;
27 }
28 this_cpu_inc(*p->counters);
29 rcu_read_unlock_sched();
30 light_mb(); /* A, between read of p->locked and read of data, paired with D */
31}
32
33static inline void percpu_up_read(struct percpu_rw_semaphore *p)
34{
35 light_mb(); /* B, between read of the data and write to p->counter, paired with C */
36 this_cpu_dec(*p->counters);
37}
38
39static inline unsigned __percpu_count(unsigned __percpu *counters)
40{
41 unsigned total = 0;
42 int cpu;
43
44 for_each_possible_cpu(cpu)
45 total += ACCESS_ONCE(*per_cpu_ptr(counters, cpu));
46
47 return total;
48}
49
50static inline void percpu_down_write(struct percpu_rw_semaphore *p)
51{
52 mutex_lock(&p->mtx);
53 p->locked = true;
54 synchronize_sched_expedited(); /* make sure that all readers exit the rcu_read_lock_sched region */
55 while (__percpu_count(p->counters))
56 msleep(1);
57 heavy_mb(); /* C, between read of p->counter and write to data, paired with B */
58}
59
60static inline void percpu_up_write(struct percpu_rw_semaphore *p)
61{
62 heavy_mb(); /* D, between write to data and write to p->locked, paired with A */
63 p->locked = false;
64 mutex_unlock(&p->mtx);
65}
66
67static inline int percpu_init_rwsem(struct percpu_rw_semaphore *p)
68{
69 p->counters = alloc_percpu(unsigned);
70 if (unlikely(!p->counters))
71 return -ENOMEM;
72 p->locked = false;
73 mutex_init(&p->mtx);
74 return 0;
75}
76
77static inline void percpu_free_rwsem(struct percpu_rw_semaphore *p)
78{
79 free_percpu(p->counters);
80 p->counters = NULL; /* catch use after free bugs */
81}
82
83#endif