arch/riscv/kernel/vector.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2023 SiFive
   4  * Author: Andy Chiu <andy.chiu@sifive.com>
   5  */
   6 #include <linux/export.h>
   7 #include <linux/sched/signal.h>
   8 #include <linux/types.h>
   9 #include <linux/slab.h>
  10 #include <linux/sched.h>
  11 #include <linux/uaccess.h>
  12 #include <linux/prctl.h>
  13
  14 #include <asm/thread_info.h>
  15 #include <asm/processor.h>
  16 #include <asm/insn.h>
  17 #include <asm/vector.h>
  18 #include <asm/csr.h>
  19 #include <asm/elf.h>
  20 #include <asm/ptrace.h>
  21 #include <asm/bug.h>
  22
  23 static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
  24
  25 unsigned long riscv_v_vsize __read_mostly;
  26 EXPORT_SYMBOL_GPL(riscv_v_vsize);
  27
  28 int riscv_v_setup_vsize(void)
  29 {
  30         unsigned long this_vsize;
  31
  32         /* There are 32 vector registers with vlenb length. */
  33         riscv_v_enable();
  34         this_vsize = csr_read(CSR_VLENB) * 32;
  35         riscv_v_disable();
  36
  37         if (!riscv_v_vsize) {
  38                 riscv_v_vsize = this_vsize;
  39                 return 0;
  40         }
  41
  42         if (riscv_v_vsize != this_vsize) {
  43                 WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
  44                 return -EOPNOTSUPP;
  45         }
  46
  47         return 0;
  48 }
  49
  50 static bool insn_is_vector(u32 insn_buf)
  51 {
  52         u32 opcode = insn_buf & __INSN_OPCODE_MASK;
  53         u32 width, csr;
  54
  55         /*
  56          * All V-related instructions, including CSR operations are 4-Byte. So,
  57          * do not handle if the instruction length is not 4-Byte.
  58          */
  59         if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
  60                 return false;
  61
  62         switch (opcode) {
  63         case RVV_OPCODE_VECTOR:
  64                 return true;
  65         case RVV_OPCODE_VL:
  66         case RVV_OPCODE_VS:
  67                 width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
  68                 if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
  69                     width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
  70                         return true;
  71
  72                 break;
  73         case RVG_OPCODE_SYSTEM:
  74                 csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
  75                 if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
  76                     (csr >= CSR_VL && csr <= CSR_VLENB))
  77                         return true;
  78         }
  79
  80         return false;
  81 }
  82
  83 static int riscv_v_thread_zalloc(void)
  84 {
  85         void *datap;
  86
  87         datap = kzalloc(riscv_v_vsize, GFP_KERNEL);
  88         if (!datap)
  89                 return -ENOMEM;
  90
  91         current->thread.vstate.datap = datap;
  92         memset(&current->thread.vstate, 0, offsetof(struct __riscv_v_ext_state,
  93                                                     datap));
  94         return 0;
  95 }
  96
  97 #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
  98 #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
  99 #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
 100 #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
 101 static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
 102 {
 103         return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
 104 }
 105
 106 static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
 107 {
 108         return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
 109 }
 110
 111 static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
 112 {
 113         return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
 114 }
 115
 116 static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
 117                                     bool inherit)
 118 {
 119         unsigned long ctrl;
 120
 121         ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
 122         ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
 123         if (inherit)
 124                 ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
 125         tsk->thread.vstate_ctrl = ctrl;
 126 }
 127
 128 bool riscv_v_vstate_ctrl_user_allowed(void)
 129 {
 130         return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
 131 }
 132 EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
 133
 134 bool riscv_v_first_use_handler(struct pt_regs *regs)
 135 {
 136         u32 __user *epc = (u32 __user *)regs->epc;
 137         u32 insn = (u32)regs->badaddr;
 138
 139         /* Do not handle if V is not supported, or disabled */
 140         if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V))
 141                 return false;
 142
 143         /* If V has been enabled then it is not the first-use trap */
 144         if (riscv_v_vstate_query(regs))
 145                 return false;
 146
 147         /* Get the instruction */
 148         if (!insn) {
 149                 if (__get_user(insn, epc))
 150                         return false;
 151         }
 152
 153         /* Filter out non-V instructions */
 154         if (!insn_is_vector(insn))
 155                 return false;
 156
 157         /* Sanity check. datap should be null by the time of the first-use trap */
 158         WARN_ON(current->thread.vstate.datap);
 159
 160         /*
 161          * Now we sure that this is a V instruction. And it executes in the
 162          * context where VS has been off. So, try to allocate the user's V
 163          * context and resume execution.
 164          */
 165         if (riscv_v_thread_zalloc()) {
 166                 force_sig(SIGBUS);
 167                 return true;
 168         }
 169         riscv_v_vstate_on(regs);
 170         riscv_v_vstate_restore(current, regs);
 171         return true;
 172 }
 173
 174 void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
 175 {
 176         bool inherit;
 177         int cur, next;
 178
 179         if (!has_vector())
 180                 return;
 181
 182         next = riscv_v_ctrl_get_next(tsk);
 183         if (!next) {
 184                 if (READ_ONCE(riscv_v_implicit_uacc))
 185                         cur = PR_RISCV_V_VSTATE_CTRL_ON;
 186                 else
 187                         cur = PR_RISCV_V_VSTATE_CTRL_OFF;
 188         } else {
 189                 cur = next;
 190         }
 191         /* Clear next mask if inherit-bit is not set */
 192         inherit = riscv_v_ctrl_test_inherit(tsk);
 193         if (!inherit)
 194                 next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
 195
 196         riscv_v_ctrl_set(tsk, cur, next, inherit);
 197 }
 198
 199 long riscv_v_vstate_ctrl_get_current(void)
 200 {
 201         if (!has_vector())
 202                 return -EINVAL;
 203
 204         return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
 205 }
 206
 207 long riscv_v_vstate_ctrl_set_current(unsigned long arg)
 208 {
 209         bool inherit;
 210         int cur, next;
 211
 212         if (!has_vector())
 213                 return -EINVAL;
 214
 215         if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
 216                 return -EINVAL;
 217
 218         cur = VSTATE_CTRL_GET_CUR(arg);
 219         switch (cur) {
 220         case PR_RISCV_V_VSTATE_CTRL_OFF:
 221                 /* Do not allow user to turn off V if current is not off */
 222                 if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
 223                         return -EPERM;
 224
 225                 break;
 226         case PR_RISCV_V_VSTATE_CTRL_ON:
 227                 break;
 228         case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
 229                 cur = riscv_v_ctrl_get_cur(current);
 230                 break;
 231         default:
 232                 return -EINVAL;
 233         }
 234
 235         next = VSTATE_CTRL_GET_NEXT(arg);
 236         inherit = VSTATE_CTRL_GET_INHERIT(arg);
 237         switch (next) {
 238         case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
 239         case PR_RISCV_V_VSTATE_CTRL_OFF:
 240         case PR_RISCV_V_VSTATE_CTRL_ON:
 241                 riscv_v_ctrl_set(current, cur, next, inherit);
 242                 return 0;
 243         }
 244
 245         return -EINVAL;
 246 }
 247
 248 #ifdef CONFIG_SYSCTL
 249
 250 static struct ctl_table riscv_v_default_vstate_table[] = {
 251         {
 252                 .procname       = "riscv_v_default_allow",
 253                 .data           = &riscv_v_implicit_uacc,
 254                 .maxlen         = sizeof(riscv_v_implicit_uacc),
 255                 .mode           = 0644,
 256                 .proc_handler   = proc_dobool,
 257         },
 258 };
 259
 260 static int __init riscv_v_sysctl_init(void)
 261 {
 262         if (has_vector())
 263                 if (!register_sysctl("abi", riscv_v_default_vstate_table))
 264                         return -EINVAL;
 265         return 0;
 266 }
 267
 268 #else /* ! CONFIG_SYSCTL */
 269 static int __init riscv_v_sysctl_init(void) { return 0; }
 270 #endif /* ! CONFIG_SYSCTL */
 271
 272 static int riscv_v_init(void)
 273 {
 274         return riscv_v_sysctl_init();
 275 }
 276 core_initcall(riscv_v_init);