Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333// SPDX-License-Identifier: GPL-2.0-or-later/* * Copyright (C) 2023 SiFive * Author: Andy Chiu <andy.chiu@sifive.com> */#include <linux/export.h>#include <linux/sched/signal.h>#include <linux/types.h>#include <linux/slab.h>#include <linux/sched.h>#include <linux/uaccess.h>#include <linux/prctl.h>
#include <asm/thread_info.h>#include <asm/processor.h>#include <asm/insn.h>#include <asm/vector.h>#include <asm/csr.h>#include <asm/elf.h>#include <asm/ptrace.h>#include <asm/bug.h>
static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);static struct kmem_cache *riscv_v_user_cachep;#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVEstatic struct kmem_cache *riscv_v_kernel_cachep;#endif
unsigned long riscv_v_vsize __read_mostly;EXPORT_SYMBOL_GPL(riscv_v_vsize);
int riscv_v_setup_vsize(void){ unsigned long this_vsize;
/* * There are 32 vector registers with vlenb length. * * If the thead,vlenb property was provided by the firmware, use that * instead of probing the CSRs. */ if (thead_vlenb_of) { riscv_v_vsize = thead_vlenb_of * 32; return 0; }
riscv_v_enable(); this_vsize = csr_read(CSR_VLENB) * 32; riscv_v_disable();
if (!riscv_v_vsize) { riscv_v_vsize = this_vsize; return 0; }
if (riscv_v_vsize != this_vsize) { WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems"); return -EOPNOTSUPP; }
return 0;}
void __init riscv_v_setup_ctx_cache(void){ if (!(has_vector() || has_xtheadvector())) return;
update_regset_vector_info(riscv_v_vsize);
riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx", riscv_v_vsize, 16, SLAB_PANIC, 0, riscv_v_vsize, NULL);#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx", riscv_v_vsize, 16, SLAB_PANIC, NULL);#endif}
bool insn_is_vector(u32 insn_buf){ u32 opcode = insn_buf & __INSN_OPCODE_MASK; u32 width, csr;
/* * All V-related instructions, including CSR operations are 4-Byte. So, * do not handle if the instruction length is not 4-Byte. */ if (unlikely(GET_INSN_LENGTH(insn_buf) != 4)) return false;
switch (opcode) { case RVV_OPCODE_VECTOR: return true; case RVV_OPCODE_VL: case RVV_OPCODE_VS: width = RVV_EXTRACT_VL_VS_WIDTH(insn_buf); if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64) return true;
break; case RVG_OPCODE_SYSTEM: csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf); if ((csr >= CSR_VSTART && csr <= CSR_VCSR) || (csr >= CSR_VL && csr <= CSR_VLENB)) return true; }
return false;}
static int riscv_v_thread_ctx_alloc(struct kmem_cache *cache, struct __riscv_v_ext_state *ctx){ void *datap;
datap = kmem_cache_zalloc(cache, GFP_KERNEL); if (!datap) return -ENOMEM;
ctx->datap = datap; memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); ctx->vlenb = riscv_v_vsize / 32;
return 0;}
void riscv_v_thread_alloc(struct task_struct *tsk){#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE riscv_v_thread_ctx_alloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate);#endif}
void riscv_v_thread_free(struct task_struct *tsk){ if (tsk->thread.vstate.datap) kmem_cache_free(riscv_v_user_cachep, tsk->thread.vstate.datap);#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE if (tsk->thread.kernel_vstate.datap) kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap);#endif}
#define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)#define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)#define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)#define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk){ return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);}
static inline int riscv_v_ctrl_get_next(struct task_struct *tsk){ return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);}
static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk){ return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);}
static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt, bool inherit){ unsigned long ctrl;
ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK; ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt); if (inherit) ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT; tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK; tsk->thread.vstate_ctrl |= ctrl;}
bool riscv_v_vstate_ctrl_user_allowed(void){ return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;}EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
bool riscv_v_first_use_handler(struct pt_regs *regs){ u32 __user *epc = (u32 __user *)regs->epc; u32 insn = (u32)regs->badaddr;
if (!(has_vector() || has_xtheadvector())) return false;
/* Do not handle if V is not supported, or disabled */ if (!riscv_v_vstate_ctrl_user_allowed()) return false;
/* If V has been enabled then it is not the first-use trap */ if (riscv_v_vstate_query(regs)) return false;
/* Get the instruction */ if (!insn) { if (__get_user(insn, epc)) return false; }
/* Filter out non-V instructions */ if (!insn_is_vector(insn)) return false;
/* Sanity check. datap should be null by the time of the first-use trap */ WARN_ON(current->thread.vstate.datap);
/* * Now we sure that this is a V instruction. And it executes in the * context where VS has been off. So, try to allocate the user's V * context and resume execution. */ if (riscv_v_thread_ctx_alloc(riscv_v_user_cachep, ¤t->thread.vstate)) { force_sig(SIGBUS); return true; }
riscv_v_vstate_on(regs); riscv_v_vstate_set_restore(current, regs);
return true;}
void riscv_v_vstate_ctrl_init(struct task_struct *tsk){ bool inherit; int cur, next;
if (!(has_vector() || has_xtheadvector())) return;
next = riscv_v_ctrl_get_next(tsk); if (!next) { if (READ_ONCE(riscv_v_implicit_uacc)) cur = PR_RISCV_V_VSTATE_CTRL_ON; else cur = PR_RISCV_V_VSTATE_CTRL_OFF; } else { cur = next; } /* Clear next mask if inherit-bit is not set */ inherit = riscv_v_ctrl_test_inherit(tsk); if (!inherit) next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
riscv_v_ctrl_set(tsk, cur, next, inherit);}
long riscv_v_vstate_ctrl_get_current(void){ if (!(has_vector() || has_xtheadvector())) return -EINVAL;
return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;}
long riscv_v_vstate_ctrl_set_current(unsigned long arg){ bool inherit; int cur, next;
if (!(has_vector() || has_xtheadvector())) return -EINVAL;
if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK) return -EINVAL;
cur = VSTATE_CTRL_GET_CUR(arg); switch (cur) { case PR_RISCV_V_VSTATE_CTRL_OFF: /* Do not allow user to turn off V if current is not off */ if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF) return -EPERM;
break; case PR_RISCV_V_VSTATE_CTRL_ON: break; case PR_RISCV_V_VSTATE_CTRL_DEFAULT: cur = riscv_v_ctrl_get_cur(current); break; default: return -EINVAL; }
next = VSTATE_CTRL_GET_NEXT(arg); inherit = VSTATE_CTRL_GET_INHERIT(arg); switch (next) { case PR_RISCV_V_VSTATE_CTRL_DEFAULT: case PR_RISCV_V_VSTATE_CTRL_OFF: case PR_RISCV_V_VSTATE_CTRL_ON: riscv_v_ctrl_set(current, cur, next, inherit); return 0; }
return -EINVAL;}
#ifdef CONFIG_SYSCTL
static const struct ctl_table riscv_v_default_vstate_table[] = { { .procname = "riscv_v_default_allow", .data = &riscv_v_implicit_uacc, .maxlen = sizeof(riscv_v_implicit_uacc), .mode = 0644, .proc_handler = proc_dobool, },};
static int __init riscv_v_sysctl_init(void){ if (has_vector() || has_xtheadvector()) if (!register_sysctl("abi", riscv_v_default_vstate_table)) return -EINVAL; return 0;}
#else /* ! CONFIG_SYSCTL */static int __init riscv_v_sysctl_init(void) { return 0; }#endif /* ! CONFIG_SYSCTL */
static int __init riscv_v_init(void){ return riscv_v_sysctl_init();}core_initcall(riscv_v_init);