Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320// SPDX-License-Identifier: GPL-2.0-only#include <linux/atomic.h>#include <linux/percpu.h>#include <linux/wait.h>#include <linux/lockdep.h>#include <linux/percpu-rwsem.h>#include <linux/rcupdate.h>#include <linux/sched.h>#include <linux/sched/task.h>#include <linux/sched/debug.h>#include <linux/errno.h>#include <trace/events/lock.h>
int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, const char *name, struct lock_class_key *key){ sem->read_count = alloc_percpu(int); if (unlikely(!sem->read_count)) return -ENOMEM;
rcu_sync_init(&sem->rss); rcuwait_init(&sem->writer); init_waitqueue_head(&sem->waiters); atomic_set(&sem->block, 0);#ifdef CONFIG_DEBUG_LOCK_ALLOC debug_check_no_locks_freed((void *)sem, sizeof(*sem)); lockdep_init_map(&sem->dep_map, name, key, 0);#endif return 0;}EXPORT_SYMBOL_GPL(__percpu_init_rwsem);
void percpu_free_rwsem(struct percpu_rw_semaphore *sem){ /* * XXX: temporary kludge. The error path in alloc_super() * assumes that percpu_free_rwsem() is safe after kzalloc(). */ if (!sem->read_count) return;
rcu_sync_dtor(&sem->rss); free_percpu(sem->read_count); sem->read_count = NULL; /* catch use after free bugs */}EXPORT_SYMBOL_GPL(percpu_free_rwsem);
static bool __percpu_down_read_trylock(struct percpu_rw_semaphore *sem){ this_cpu_inc(*sem->read_count);
/* * Due to having preemption disabled the decrement happens on * the same CPU as the increment, avoiding the * increment-on-one-CPU-and-decrement-on-another problem. * * If the reader misses the writer's assignment of sem->block, then the * writer is guaranteed to see the reader's increment. * * Conversely, any readers that increment their sem->read_count after * the writer looks are guaranteed to see the sem->block value, which * in turn means that they are guaranteed to immediately decrement * their sem->read_count, so that it doesn't matter that the writer * missed them. */
smp_mb(); /* A matches D */
/* * If !sem->block the critical section starts here, matched by the * release in percpu_up_write(). */ if (likely(!atomic_read_acquire(&sem->block))) return true;
this_cpu_dec(*sem->read_count);
/* Prod writer to re-evaluate readers_active_check() */ rcuwait_wake_up(&sem->writer);
return false;}
static inline bool __percpu_down_write_trylock(struct percpu_rw_semaphore *sem){ if (atomic_read(&sem->block)) return false;
return atomic_xchg(&sem->block, 1) == 0;}
static bool __percpu_rwsem_trylock(struct percpu_rw_semaphore *sem, bool reader){ if (reader) { bool ret;
preempt_disable(); ret = __percpu_down_read_trylock(sem); preempt_enable();
return ret; } return __percpu_down_write_trylock(sem);}
/* * The return value of wait_queue_entry::func means: * * <0 - error, wakeup is terminated and the error is returned * 0 - no wakeup, a next waiter is tried * >0 - woken, if EXCLUSIVE, counted towards @nr_exclusive. * * We use EXCLUSIVE for both readers and writers to preserve FIFO order, * and play games with the return value to allow waking multiple readers. * * Specifically, we wake readers until we've woken a single writer, or until a * trylock fails. */static int percpu_rwsem_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, int wake_flags, void *key){ bool reader = wq_entry->flags & WQ_FLAG_CUSTOM; struct percpu_rw_semaphore *sem = key; struct task_struct *p;
/* concurrent against percpu_down_write(), can get stolen */ if (!__percpu_rwsem_trylock(sem, reader)) return 1;
p = get_task_struct(wq_entry->private); list_del_init(&wq_entry->entry); smp_store_release(&wq_entry->private, NULL);
wake_up_process(p); put_task_struct(p);
return !reader; /* wake (readers until) 1 writer */}
static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader, bool freeze){ DEFINE_WAIT_FUNC(wq_entry, percpu_rwsem_wake_function); bool wait;
spin_lock_irq(&sem->waiters.lock); /* * Serialize against the wakeup in percpu_up_write(), if we fail * the trylock, the wakeup must see us on the list. */ wait = !__percpu_rwsem_trylock(sem, reader); if (wait) { wq_entry.flags |= WQ_FLAG_EXCLUSIVE | reader * WQ_FLAG_CUSTOM; __add_wait_queue_entry_tail(&sem->waiters, &wq_entry); } spin_unlock_irq(&sem->waiters.lock);
while (wait) { set_current_state(TASK_UNINTERRUPTIBLE | (freeze ? TASK_FREEZABLE : 0)); if (!smp_load_acquire(&wq_entry.private)) break; schedule(); } __set_current_state(TASK_RUNNING);}
bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try, bool freeze){ if (__percpu_down_read_trylock(sem)) return true;
if (try) return false;
trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_READ); preempt_enable(); percpu_rwsem_wait(sem, /* .reader = */ true, freeze); preempt_disable(); trace_contention_end(sem, 0);
return true;}EXPORT_SYMBOL_GPL(__percpu_down_read);
#define per_cpu_sum(var) \({ \ TYPEOF_UNQUAL(var) __sum = 0; \ int cpu; \ compiletime_assert_atomic_type(__sum); \ for_each_possible_cpu(cpu) \ __sum += per_cpu(var, cpu); \ __sum; \})
bool percpu_is_read_locked(struct percpu_rw_semaphore *sem){ return per_cpu_sum(*sem->read_count) != 0 && !atomic_read(&sem->block);}EXPORT_SYMBOL_GPL(percpu_is_read_locked);
/* * Return true if the modular sum of the sem->read_count per-CPU variable is * zero. If this sum is zero, then it is stable due to the fact that if any * newly arriving readers increment a given counter, they will immediately * decrement that same counter. * * Assumes sem->block is set. */static bool readers_active_check(struct percpu_rw_semaphore *sem){ if (per_cpu_sum(*sem->read_count) != 0) return false;
/* * If we observed the decrement; ensure we see the entire critical * section. */
smp_mb(); /* C matches B */
return true;}
void __sched percpu_down_write(struct percpu_rw_semaphore *sem){ bool contended = false;
might_sleep(); rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
/* Notify readers to take the slow path. */ rcu_sync_enter(&sem->rss);
/* * Try set sem->block; this provides writer-writer exclusion. * Having sem->block set makes new readers block. */ if (!__percpu_down_write_trylock(sem)) { trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_WRITE); percpu_rwsem_wait(sem, /* .reader = */ false, false); contended = true; }
/* smp_mb() implied by __percpu_down_write_trylock() on success -- D matches A */
/* * If they don't see our store of sem->block, then we are guaranteed to * see their sem->read_count increment, and therefore will wait for * them. */
/* Wait for all active readers to complete. */ rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); if (contended) trace_contention_end(sem, 0);}EXPORT_SYMBOL_GPL(percpu_down_write);
void percpu_up_write(struct percpu_rw_semaphore *sem){ rwsem_release(&sem->dep_map, _RET_IP_);
if (trace_contended_release_enabled() && wq_has_sleeper(&sem->waiters)) trace_call__contended_release(sem);
/* * Signal the writer is done, no fast path yet. * * One reason that we cannot just immediately flip to readers_fast is * that new readers might fail to see the results of this writer's * critical section. * * Therefore we force it through the slow path which guarantees an * acquire and thereby guarantees the critical section's consistency. */ atomic_set_release(&sem->block, 0);
/* * Prod any pending reader/writer to make progress. */ __wake_up(&sem->waiters, TASK_NORMAL, 1, sem);
/* * Once this completes (at least one RCU-sched grace period hence) the * reader fast path will be available again. Safe to use outside the * exclusive write lock because its counting. */ rcu_sync_exit(&sem->rss);}EXPORT_SYMBOL_GPL(percpu_up_write);
void __percpu_up_read(struct percpu_rw_semaphore *sem){ lockdep_assert_preemption_disabled(); /* * After percpu_up_write() completes, rcu_sync_is_idle() can still * return false during the grace period, forcing readers into this * slowpath. Only trace when a writer is actually waiting for * readers to drain. */ if (trace_contended_release_enabled() && rcuwait_active(&sem->writer)) trace_call__contended_release(sem); /* * slowpath; reader will only ever wake a single blocked * writer. */ smp_mb(); /* B matches C */ /* * In other words, if they see our decrement (presumably to * aggregate zero, as that is the only time it matters) they * will also see our critical section. */ this_cpu_dec(*sem->read_count); rcuwait_wake_up(&sem->writer);}EXPORT_SYMBOL_GPL(__percpu_up_read);