arch/loongarch/kernel/unaligned.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Handle unaligned accesses by emulation.
   4  *
   5  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
   6  *
   7  * Derived from MIPS:
   8  * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
   9  * Copyright (C) 1999 Silicon Graphics, Inc.
  10  * Copyright (C) 2014 Imagination Technologies Ltd.
  11  */
  12 #include <linux/mm.h>
  13 #include <linux/sched.h>
  14 #include <linux/signal.h>
  15 #include <linux/debugfs.h>
  16 #include <linux/perf_event.h>
  17
  18 #include <asm/asm.h>
  19 #include <asm/branch.h>
  20 #include <asm/fpu.h>
  21 #include <asm/inst.h>
  22
  23 #include "access-helper.h"
  24
  25 #ifdef CONFIG_DEBUG_FS
  26 static u32 unaligned_instructions_user;
  27 static u32 unaligned_instructions_kernel;
  28 #endif
  29
  30 static inline unsigned long read_fpr(unsigned int idx)
  31 {
  32 #define READ_FPR(idx, __value)          \
  33         __asm__ __volatile__("movfr2gr.d %0, $f"#idx"\n\t" : "=r"(__value));
  34
  35         unsigned long __value;
  36
  37         switch (idx) {
  38         case 0:
  39                 READ_FPR(0, __value);
  40                 break;
  41         case 1:
  42                 READ_FPR(1, __value);
  43                 break;
  44         case 2:
  45                 READ_FPR(2, __value);
  46                 break;
  47         case 3:
  48                 READ_FPR(3, __value);
  49                 break;
  50         case 4:
  51                 READ_FPR(4, __value);
  52                 break;
  53         case 5:
  54                 READ_FPR(5, __value);
  55                 break;
  56         case 6:
  57                 READ_FPR(6, __value);
  58                 break;
  59         case 7:
  60                 READ_FPR(7, __value);
  61                 break;
  62         case 8:
  63                 READ_FPR(8, __value);
  64                 break;
  65         case 9:
  66                 READ_FPR(9, __value);
  67                 break;
  68         case 10:
  69                 READ_FPR(10, __value);
  70                 break;
  71         case 11:
  72                 READ_FPR(11, __value);
  73                 break;
  74         case 12:
  75                 READ_FPR(12, __value);
  76                 break;
  77         case 13:
  78                 READ_FPR(13, __value);
  79                 break;
  80         case 14:
  81                 READ_FPR(14, __value);
  82                 break;
  83         case 15:
  84                 READ_FPR(15, __value);
  85                 break;
  86         case 16:
  87                 READ_FPR(16, __value);
  88                 break;
  89         case 17:
  90                 READ_FPR(17, __value);
  91                 break;
  92         case 18:
  93                 READ_FPR(18, __value);
  94                 break;
  95         case 19:
  96                 READ_FPR(19, __value);
  97                 break;
  98         case 20:
  99                 READ_FPR(20, __value);
 100                 break;
 101         case 21:
 102                 READ_FPR(21, __value);
 103                 break;
 104         case 22:
 105                 READ_FPR(22, __value);
 106                 break;
 107         case 23:
 108                 READ_FPR(23, __value);
 109                 break;
 110         case 24:
 111                 READ_FPR(24, __value);
 112                 break;
 113         case 25:
 114                 READ_FPR(25, __value);
 115                 break;
 116         case 26:
 117                 READ_FPR(26, __value);
 118                 break;
 119         case 27:
 120                 READ_FPR(27, __value);
 121                 break;
 122         case 28:
 123                 READ_FPR(28, __value);
 124                 break;
 125         case 29:
 126                 READ_FPR(29, __value);
 127                 break;
 128         case 30:
 129                 READ_FPR(30, __value);
 130                 break;
 131         case 31:
 132                 READ_FPR(31, __value);
 133                 break;
 134         default:
 135                 panic("unexpected idx '%d'", idx);
 136         }
 137 #undef READ_FPR
 138         return __value;
 139 }
 140
 141 static inline void write_fpr(unsigned int idx, unsigned long value)
 142 {
 143 #define WRITE_FPR(idx, value)           \
 144         __asm__ __volatile__("movgr2fr.d $f"#idx", %0\n\t" :: "r"(value));
 145
 146         switch (idx) {
 147         case 0:
 148                 WRITE_FPR(0, value);
 149                 break;
 150         case 1:
 151                 WRITE_FPR(1, value);
 152                 break;
 153         case 2:
 154                 WRITE_FPR(2, value);
 155                 break;
 156         case 3:
 157                 WRITE_FPR(3, value);
 158                 break;
 159         case 4:
 160                 WRITE_FPR(4, value);
 161                 break;
 162         case 5:
 163                 WRITE_FPR(5, value);
 164                 break;
 165         case 6:
 166                 WRITE_FPR(6, value);
 167                 break;
 168         case 7:
 169                 WRITE_FPR(7, value);
 170                 break;
 171         case 8:
 172                 WRITE_FPR(8, value);
 173                 break;
 174         case 9:
 175                 WRITE_FPR(9, value);
 176                 break;
 177         case 10:
 178                 WRITE_FPR(10, value);
 179                 break;
 180         case 11:
 181                 WRITE_FPR(11, value);
 182                 break;
 183         case 12:
 184                 WRITE_FPR(12, value);
 185                 break;
 186         case 13:
 187                 WRITE_FPR(13, value);
 188                 break;
 189         case 14:
 190                 WRITE_FPR(14, value);
 191                 break;
 192         case 15:
 193                 WRITE_FPR(15, value);
 194                 break;
 195         case 16:
 196                 WRITE_FPR(16, value);
 197                 break;
 198         case 17:
 199                 WRITE_FPR(17, value);
 200                 break;
 201         case 18:
 202                 WRITE_FPR(18, value);
 203                 break;
 204         case 19:
 205                 WRITE_FPR(19, value);
 206                 break;
 207         case 20:
 208                 WRITE_FPR(20, value);
 209                 break;
 210         case 21:
 211                 WRITE_FPR(21, value);
 212                 break;
 213         case 22:
 214                 WRITE_FPR(22, value);
 215                 break;
 216         case 23:
 217                 WRITE_FPR(23, value);
 218                 break;
 219         case 24:
 220                 WRITE_FPR(24, value);
 221                 break;
 222         case 25:
 223                 WRITE_FPR(25, value);
 224                 break;
 225         case 26:
 226                 WRITE_FPR(26, value);
 227                 break;
 228         case 27:
 229                 WRITE_FPR(27, value);
 230                 break;
 231         case 28:
 232                 WRITE_FPR(28, value);
 233                 break;
 234         case 29:
 235                 WRITE_FPR(29, value);
 236                 break;
 237         case 30:
 238                 WRITE_FPR(30, value);
 239                 break;
 240         case 31:
 241                 WRITE_FPR(31, value);
 242                 break;
 243         default:
 244                 panic("unexpected idx '%d'", idx);
 245         }
 246 #undef WRITE_FPR
 247 }
 248
 249 void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int *pc)
 250 {
 251         bool fp = false;
 252         bool sign, write;
 253         bool user = user_mode(regs);
 254         unsigned int res, size = 0;
 255         unsigned long value = 0;
 256         union loongarch_instruction insn;
 257
 258         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
 259
 260         __get_inst(&insn.word, pc, user);
 261
 262         switch (insn.reg2i12_format.opcode) {
 263         case ldh_op:
 264                 size = 2;
 265                 sign = true;
 266                 write = false;
 267                 break;
 268         case ldhu_op:
 269                 size = 2;
 270                 sign = false;
 271                 write = false;
 272                 break;
 273         case sth_op:
 274                 size = 2;
 275                 sign = true;
 276                 write = true;
 277                 break;
 278         case ldw_op:
 279                 size = 4;
 280                 sign = true;
 281                 write = false;
 282                 break;
 283         case ldwu_op:
 284                 size = 4;
 285                 sign = false;
 286                 write = false;
 287                 break;
 288         case stw_op:
 289                 size = 4;
 290                 sign = true;
 291                 write = true;
 292                 break;
 293         case ldd_op:
 294                 size = 8;
 295                 sign = true;
 296                 write = false;
 297                 break;
 298         case std_op:
 299                 size = 8;
 300                 sign = true;
 301                 write = true;
 302                 break;
 303         case flds_op:
 304                 size = 4;
 305                 fp = true;
 306                 sign = true;
 307                 write = false;
 308                 break;
 309         case fsts_op:
 310                 size = 4;
 311                 fp = true;
 312                 sign = true;
 313                 write = true;
 314                 break;
 315         case fldd_op:
 316                 size = 8;
 317                 fp = true;
 318                 sign = true;
 319                 write = false;
 320                 break;
 321         case fstd_op:
 322                 size = 8;
 323                 fp = true;
 324                 sign = true;
 325                 write = true;
 326                 break;
 327         }
 328
 329         switch (insn.reg2i14_format.opcode) {
 330         case ldptrw_op:
 331                 size = 4;
 332                 sign = true;
 333                 write = false;
 334                 break;
 335         case stptrw_op:
 336                 size = 4;
 337                 sign = true;
 338                 write = true;
 339                 break;
 340         case ldptrd_op:
 341                 size = 8;
 342                 sign = true;
 343                 write = false;
 344                 break;
 345         case stptrd_op:
 346                 size = 8;
 347                 sign = true;
 348                 write = true;
 349                 break;
 350         }
 351
 352         switch (insn.reg3_format.opcode) {
 353         case ldxh_op:
 354                 size = 2;
 355                 sign = true;
 356                 write = false;
 357                 break;
 358         case ldxhu_op:
 359                 size = 2;
 360                 sign = false;
 361                 write = false;
 362                 break;
 363         case stxh_op:
 364                 size = 2;
 365                 sign = true;
 366                 write = true;
 367                 break;
 368         case ldxw_op:
 369                 size = 4;
 370                 sign = true;
 371                 write = false;
 372                 break;
 373         case ldxwu_op:
 374                 size = 4;
 375                 sign = false;
 376                 write = false;
 377                 break;
 378         case stxw_op:
 379                 size = 4;
 380                 sign = true;
 381                 write = true;
 382                 break;
 383         case ldxd_op:
 384                 size = 8;
 385                 sign = true;
 386                 write = false;
 387                 break;
 388         case stxd_op:
 389                 size = 8;
 390                 sign = true;
 391                 write = true;
 392                 break;
 393         case fldxs_op:
 394                 size = 4;
 395                 fp = true;
 396                 sign = true;
 397                 write = false;
 398                 break;
 399         case fstxs_op:
 400                 size = 4;
 401                 fp = true;
 402                 sign = true;
 403                 write = true;
 404                 break;
 405         case fldxd_op:
 406                 size = 8;
 407                 fp = true;
 408                 sign = true;
 409                 write = false;
 410                 break;
 411         case fstxd_op:
 412                 size = 8;
 413                 fp = true;
 414                 sign = true;
 415                 write = true;
 416                 break;
 417         }
 418
 419         if (!size)
 420                 goto sigbus;
 421         if (user && !access_ok(addr, size))
 422                 goto sigbus;
 423
 424         if (!write) {
 425                 res = unaligned_read(addr, &value, size, sign);
 426                 if (res)
 427                         goto fault;
 428
 429                 /* Rd is the same field in any formats */
 430                 if (!fp)
 431                         regs->regs[insn.reg3_format.rd] = value;
 432                 else {
 433                         if (is_fpu_owner())
 434                                 write_fpr(insn.reg3_format.rd, value);
 435                         else
 436                                 set_fpr64(&current->thread.fpu.fpr[insn.reg3_format.rd], 0, value);
 437                 }
 438         } else {
 439                 /* Rd is the same field in any formats */
 440                 if (!fp)
 441                         value = regs->regs[insn.reg3_format.rd];
 442                 else {
 443                         if (is_fpu_owner())
 444                                 value = read_fpr(insn.reg3_format.rd);
 445                         else
 446                                 value = get_fpr64(&current->thread.fpu.fpr[insn.reg3_format.rd], 0);
 447                 }
 448
 449                 res = unaligned_write(addr, value, size);
 450                 if (res)
 451                         goto fault;
 452         }
 453
 454 #ifdef CONFIG_DEBUG_FS
 455         if (user)
 456                 unaligned_instructions_user++;
 457         else
 458                 unaligned_instructions_kernel++;
 459 #endif
 460
 461         compute_return_era(regs);
 462
 463         return;
 464
 465 fault:
 466         /* Did we have an exception handler installed? */
 467         if (fixup_exception(regs))
 468                 return;
 469
 470         die_if_kernel("Unhandled kernel unaligned access", regs);
 471         force_sig(SIGSEGV);
 472
 473         return;
 474
 475 sigbus:
 476         die_if_kernel("Unhandled kernel unaligned access", regs);
 477         force_sig(SIGBUS);
 478
 479         return;
 480 }
 481
 482 #ifdef CONFIG_DEBUG_FS
 483 static int __init debugfs_unaligned(void)
 484 {
 485         struct dentry *d;
 486
 487         d = debugfs_create_dir("loongarch", NULL);
 488
 489         debugfs_create_u32("unaligned_instructions_user",
 490                                 S_IRUGO, d, &unaligned_instructions_user);
 491         debugfs_create_u32("unaligned_instructions_kernel",
 492                                 S_IRUGO, d, &unaligned_instructions_kernel);
 493
 494         return 0;
 495 }
 496 arch_initcall(debugfs_unaligned);
 497 #endif