Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (c) 2008 Intel Corporation * Author: Matthew Wilcox <willy@linux.intel.com> * * This file implements counting semaphores. * A counting semaphore may be acquired 'n' times before sleeping. * See mutex.c for single-acquisition sleeping locks which enforce * rules which allow code to be debugged more easily. */
/* * Some notes on the implementation: * * The spinlock controls access to the other members of the semaphore. * down_trylock() and up() can be called from interrupt context, so we * have to disable interrupts when taking the lock. It turns out various * parts of the kernel expect to be able to use down() on a semaphore in * interrupt context when they know it will succeed, so we have to use * irqsave variants for down(), down_interruptible() and down_killable() * too. * * The ->count variable represents how many more tasks can acquire this * semaphore. If it's zero, there may be waiters. */
#include <linux/compiler.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/sched.h>#include <linux/sched/debug.h>#include <linux/sched/wake_q.h>#include <linux/semaphore.h>#include <linux/spinlock.h>#include <linux/ftrace.h>#include <trace/events/lock.h>#include <linux/hung_task.h>
static noinline void __down(struct semaphore *sem);static noinline int __down_interruptible(struct semaphore *sem);static noinline int __down_killable(struct semaphore *sem);static noinline int __down_timeout(struct semaphore *sem, long timeout);static noinline void __up(struct semaphore *sem, struct wake_q_head *wake_q);
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKERstatic inline void hung_task_sem_set_holder(struct semaphore *sem){ WRITE_ONCE((sem)->last_holder, (unsigned long)current);}
static inline void hung_task_sem_clear_if_holder(struct semaphore *sem){ if (READ_ONCE((sem)->last_holder) == (unsigned long)current) WRITE_ONCE((sem)->last_holder, 0UL);}
unsigned long sem_last_holder(struct semaphore *sem){ return READ_ONCE(sem->last_holder);}#elsestatic inline void hung_task_sem_set_holder(struct semaphore *sem){}static inline void hung_task_sem_clear_if_holder(struct semaphore *sem){}unsigned long sem_last_holder(struct semaphore *sem){ return 0UL;}#endif
static inline void __sem_acquire(struct semaphore *sem){ sem->count--; hung_task_sem_set_holder(sem);}
/** * down - acquire the semaphore * @sem: the semaphore to be acquired * * Acquires the semaphore. If no more tasks are allowed to acquire the * semaphore, calling this function will put the task to sleep until the * semaphore is released. * * Use of this function is deprecated, please use down_interruptible() or * down_killable() instead. */void __sched down(struct semaphore *sem){ unsigned long flags;
might_sleep(); raw_spin_lock_irqsave(&sem->lock, flags); if (likely(sem->count > 0)) __sem_acquire(sem); else __down(sem); raw_spin_unlock_irqrestore(&sem->lock, flags);}EXPORT_SYMBOL(down);
/** * down_interruptible - acquire the semaphore unless interrupted * @sem: the semaphore to be acquired * * Attempts to acquire the semaphore. If no more tasks are allowed to * acquire the semaphore, calling this function will put the task to sleep. * If the sleep is interrupted by a signal, this function will return -EINTR. * If the semaphore is successfully acquired, this function returns 0. */int __sched down_interruptible(struct semaphore *sem){ unsigned long flags; int result = 0;
might_sleep(); raw_spin_lock_irqsave(&sem->lock, flags); if (likely(sem->count > 0)) __sem_acquire(sem); else result = __down_interruptible(sem); raw_spin_unlock_irqrestore(&sem->lock, flags);
return result;}EXPORT_SYMBOL(down_interruptible);
/** * down_killable - acquire the semaphore unless killed * @sem: the semaphore to be acquired * * Attempts to acquire the semaphore. If no more tasks are allowed to * acquire the semaphore, calling this function will put the task to sleep. * If the sleep is interrupted by a fatal signal, this function will return * -EINTR. If the semaphore is successfully acquired, this function returns * 0. */int __sched down_killable(struct semaphore *sem){ unsigned long flags; int result = 0;
might_sleep(); raw_spin_lock_irqsave(&sem->lock, flags); if (likely(sem->count > 0)) __sem_acquire(sem); else result = __down_killable(sem); raw_spin_unlock_irqrestore(&sem->lock, flags);
return result;}EXPORT_SYMBOL(down_killable);
/** * down_trylock - try to acquire the semaphore, without waiting * @sem: the semaphore to be acquired * * Try to acquire the semaphore atomically. Returns 0 if the semaphore has * been acquired successfully or 1 if it cannot be acquired. * * NOTE: This return value is inverted from both spin_trylock and * mutex_trylock! Be careful about this when converting code. * * Unlike mutex_trylock, this function can be used from interrupt context, * and the semaphore can be released by any task or interrupt. */int __sched down_trylock(struct semaphore *sem){ unsigned long flags; int count;
raw_spin_lock_irqsave(&sem->lock, flags); count = sem->count - 1; if (likely(count >= 0)) __sem_acquire(sem); raw_spin_unlock_irqrestore(&sem->lock, flags);
return (count < 0);}EXPORT_SYMBOL(down_trylock);
/** * down_timeout - acquire the semaphore within a specified time * @sem: the semaphore to be acquired * @timeout: how long to wait before failing * * Attempts to acquire the semaphore. If no more tasks are allowed to * acquire the semaphore, calling this function will put the task to sleep. * If the semaphore is not released within the specified number of jiffies, * this function returns -ETIME. It returns 0 if the semaphore was acquired. */int __sched down_timeout(struct semaphore *sem, long timeout){ unsigned long flags; int result = 0;
might_sleep(); raw_spin_lock_irqsave(&sem->lock, flags); if (likely(sem->count > 0)) __sem_acquire(sem); else result = __down_timeout(sem, timeout); raw_spin_unlock_irqrestore(&sem->lock, flags);
return result;}EXPORT_SYMBOL(down_timeout);
/** * up - release the semaphore * @sem: the semaphore to release * * Release the semaphore. Unlike mutexes, up() may be called from any * context and even by tasks which have never called down(). */void __sched up(struct semaphore *sem){ unsigned long flags; DEFINE_WAKE_Q(wake_q);
raw_spin_lock_irqsave(&sem->lock, flags);
hung_task_sem_clear_if_holder(sem);
if (likely(!sem->first_waiter)) sem->count++; else __up(sem, &wake_q);
if (trace_contended_release_enabled() && !wake_q_empty(&wake_q)) trace_call__contended_release(sem);
raw_spin_unlock_irqrestore(&sem->lock, flags); if (!wake_q_empty(&wake_q)) wake_up_q(&wake_q);}EXPORT_SYMBOL(up);
/* Functions for the contended case */
struct semaphore_waiter { struct list_head list; struct task_struct *task; bool up;};
static inlinevoid sem_del_waiter(struct semaphore *sem, struct semaphore_waiter *waiter){ if (list_empty(&waiter->list)) { sem->first_waiter = NULL; return; }
if (sem->first_waiter == waiter) { sem->first_waiter = list_first_entry(&waiter->list, struct semaphore_waiter, list); } list_del(&waiter->list);}
/* * Because this function is inlined, the 'state' parameter will be * constant, and thus optimised away by the compiler. Likewise the * 'timeout' parameter for the cases without timeouts. */static inline int __sched ___down_common(struct semaphore *sem, long state, long timeout){ struct semaphore_waiter waiter, *first;
first = sem->first_waiter; if (first) { list_add_tail(&waiter.list, &first->list); } else { INIT_LIST_HEAD(&waiter.list); sem->first_waiter = &waiter; } waiter.task = current; waiter.up = false;
for (;;) { if (signal_pending_state(state, current)) goto interrupted; if (unlikely(timeout <= 0)) goto timed_out; __set_current_state(state); raw_spin_unlock_irq(&sem->lock); timeout = schedule_timeout(timeout); raw_spin_lock_irq(&sem->lock); if (waiter.up) { hung_task_sem_set_holder(sem); return 0; } }
timed_out: sem_del_waiter(sem, &waiter); return -ETIME;
interrupted: sem_del_waiter(sem, &waiter); return -EINTR;}
static inline int __sched __down_common(struct semaphore *sem, long state, long timeout){ int ret;
hung_task_set_blocker(sem, BLOCKER_TYPE_SEM);
trace_contention_begin(sem, 0); ret = ___down_common(sem, state, timeout); trace_contention_end(sem, ret);
hung_task_clear_blocker();
return ret;}
static noinline void __sched __down(struct semaphore *sem){ __down_common(sem, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);}
static noinline int __sched __down_interruptible(struct semaphore *sem){ return __down_common(sem, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);}
static noinline int __sched __down_killable(struct semaphore *sem){ return __down_common(sem, TASK_KILLABLE, MAX_SCHEDULE_TIMEOUT);}
static noinline int __sched __down_timeout(struct semaphore *sem, long timeout){ return __down_common(sem, TASK_UNINTERRUPTIBLE, timeout);}
static noinline void __sched __up(struct semaphore *sem, struct wake_q_head *wake_q){ struct semaphore_waiter *waiter = sem->first_waiter;
sem_del_waiter(sem, waiter); waiter->up = true; wake_q_add(wake_q, waiter->task);}