Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
C
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */#ifndef _LINUX_RSEQ_H#define _LINUX_RSEQ_H
#ifdef CONFIG_RSEQ#include <linux/sched.h>
#include <uapi/linux/rseq.h>
void __rseq_handle_slowpath(struct pt_regs *regs);
static __always_inline bool rseq_v2(struct task_struct *t){ return IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY) && likely(t->rseq.event.has_rseq > 1);}
/* Invoked from resume_user_mode_work() */static inline void rseq_handle_slowpath(struct pt_regs *regs){ if (IS_ENABLED(CONFIG_GENERIC_ENTRY)) { if (current->rseq.event.slowpath) __rseq_handle_slowpath(regs); } else { if (current->rseq.event.sched_switch && current->rseq.event.has_rseq) __rseq_handle_slowpath(regs); }}
void __rseq_signal_deliver(int sig, struct pt_regs *regs);
/* * Invoked from signal delivery to fixup based on the register context before * switching to the signal delivery context. */static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs){ if (rseq_v2(current)) { /* has_rseq is implied in rseq_v2() */ if (current->rseq.event.user_irq) __rseq_signal_deliver(ksig->sig, regs); } else { if (current->rseq.event.has_rseq) __rseq_signal_deliver(ksig->sig, regs); }}
static inline void rseq_raise_notify_resume(struct task_struct *t){ set_tsk_thread_flag(t, TIF_RSEQ);}
/* Invoked from context switch to force evaluation on exit to user */static __always_inline void rseq_sched_switch_event(struct task_struct *t){ struct rseq_event *ev = &t->rseq.event;
/* * Only apply the user_irq optimization for RSEQ ABI V2 registrations. * Legacy users like TCMalloc rely on the original ABI V1 behaviour * which updates IDs on every context swtich. */ if (rseq_v2(t)) { /* * Avoid a boat load of conditionals by using simple logic to * determine whether TIF_NOTIFY_RESUME or TIF_RSEQ needs to be * raised. * * It's required when the CPU or MM CID has changed or the entry * was via interrupt from user space. ev->has_rseq does not have * to be evaluated here because rseq_v2() implies has_rseq. */ bool raise = ev->user_irq | ev->ids_changed;
if (raise) { ev->sched_switch = true; rseq_raise_notify_resume(t); } } else { if (ev->has_rseq) { t->rseq.event.ids_changed = true; t->rseq.event.sched_switch = true; rseq_raise_notify_resume(t); } }}
/* * Invoked from __set_task_cpu() when a task migrates or from * mm_cid_schedin() when the CID changes to enforce an IDs update. * * This does not raise TIF_NOTIFY_RESUME as that happens in * rseq_sched_switch_event(). */static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t){ t->rseq.event.ids_changed = true;}
/* Enforce a full update after RSEQ registration and when execve() failed */static inline void rseq_force_update(void){ if (current->rseq.event.has_rseq) { current->rseq.event.ids_changed = true; current->rseq.event.sched_switch = true; rseq_raise_notify_resume(current); }}
/* * KVM/HYPERV invoke resume_user_mode_work() before entering guest mode, * which clears TIF_NOTIFY_RESUME on architectures that don't use the * generic TIF bits and therefore can't provide a separate TIF_RSEQ flag. * * To avoid updating user space RSEQ in that case just to do it eventually * again before returning to user space, because __rseq_handle_slowpath() * does nothing when invoked with NULL register state. * * After returning from guest mode, before exiting to userspace, hypervisors * must invoke this function to re-raise TIF_NOTIFY_RESUME if necessary. */static inline void rseq_virt_userspace_exit(void){ /* * The generic optimization for deferring RSEQ updates until the next * exit relies on having a dedicated TIF_RSEQ. */ if (!IS_ENABLED(CONFIG_HAVE_GENERIC_TIF_BITS) && current->rseq.event.sched_switch) rseq_raise_notify_resume(current);}
static inline void rseq_reset(struct task_struct *t){ /* Protect against preemption and membarrier IPI */ guard(irqsave)(); memset(&t->rseq, 0, sizeof(t->rseq)); t->rseq.ids.cpu_id = RSEQ_CPU_ID_UNINITIALIZED;}
static inline void rseq_execve(struct task_struct *t){ rseq_reset(t);}
/* * If parent process has a registered restartable sequences area, the * child inherits. Unregister rseq for a clone with CLONE_VM set. * * On fork, keep the IDs (CPU, MMCID) of the parent, which avoids a fault * on the COW page on exit to user space, when the child stays on the same * CPU as the parent. That's obviously not guaranteed, but in overcommit * scenarios it is more likely and optimizes for the fork/exec case without * taking the fault. */static inline void rseq_fork(struct task_struct *t, u64 clone_flags){ if (clone_flags & CLONE_VM) rseq_reset(t); else t->rseq = current->rseq;}
/* * Value returned by getauxval(AT_RSEQ_ALIGN) and expected by rseq * registration. This is the active rseq area size rounded up to next * power of 2, which guarantees that the rseq structure will always be * aligned on the nearest power of two large enough to contain it, even * as it grows. */static inline unsigned int rseq_alloc_align(void){ return 1U << get_count_order(offsetof(struct rseq, end));}
#else /* CONFIG_RSEQ */static inline bool rseq_v2(struct task_struct *t) { return false; }static inline void rseq_handle_slowpath(struct pt_regs *regs) { }static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }static inline void rseq_sched_switch_event(struct task_struct *t) { }static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }static inline void rseq_force_update(void) { }static inline void rseq_virt_userspace_exit(void) { }static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }static inline void rseq_execve(struct task_struct *t) { }#endif /* !CONFIG_RSEQ */
#ifdef CONFIG_DEBUG_RSEQvoid rseq_syscall(struct pt_regs *regs);#else /* CONFIG_DEBUG_RSEQ */static inline void rseq_syscall(struct pt_regs *regs) { }#endif /* !CONFIG_DEBUG_RSEQ */
#ifdef CONFIG_RSEQ_SLICE_EXTENSIONvoid rseq_syscall_enter_work(long syscall);int rseq_slice_extension_prctl(unsigned long arg2, unsigned long arg3);#else /* CONFIG_RSEQ_SLICE_EXTENSION */static inline void rseq_syscall_enter_work(long syscall) { }static inline int rseq_slice_extension_prctl(unsigned long arg2, unsigned long arg3){ return -ENOTSUPP;}#endif /* !CONFIG_RSEQ_SLICE_EXTENSION */
#endif /* _LINUX_RSEQ_H */