arch/x86/kernel/sev-es-shared.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * AMD Encrypted Register State Support
   4  *
   5  * Author: Joerg Roedel <jroedel@suse.de>
   6  *
   7  * This file is not compiled stand-alone. It contains code shared
   8  * between the pre-decompression boot code and the running Linux kernel
   9  * and is included directly into both code-bases.
  10  */
  11
  12 #ifndef __BOOT_COMPRESSED
  13 #define error(v)        pr_err(v)
  14 #define has_cpuflag(f)  boot_cpu_has(f)
  15 #endif
  16
  17 static bool __init sev_es_check_cpu_features(void)
  18 {
  19         if (!has_cpuflag(X86_FEATURE_RDRAND)) {
  20                 error("RDRAND instruction not supported - no trusted source of randomness available\n");
  21                 return false;
  22         }
  23
  24         return true;
  25 }
  26
  27 static void sev_es_terminate(unsigned int reason)
  28 {
  29         u64 val = GHCB_SEV_TERMINATE;
  30
  31         /*
  32          * Tell the hypervisor what went wrong - only reason-set 0 is
  33          * currently supported.
  34          */
  35         val |= GHCB_SEV_TERMINATE_REASON(0, reason);
  36
  37         /* Request Guest Termination from Hypvervisor */
  38         sev_es_wr_ghcb_msr(val);
  39         VMGEXIT();
  40
  41         while (true)
  42                 asm volatile("hlt\n" : : : "memory");
  43 }
  44
  45 static bool sev_es_negotiate_protocol(void)
  46 {
  47         u64 val;
  48
  49         /* Do the GHCB protocol version negotiation */
  50         sev_es_wr_ghcb_msr(GHCB_SEV_INFO_REQ);
  51         VMGEXIT();
  52         val = sev_es_rd_ghcb_msr();
  53
  54         if (GHCB_INFO(val) != GHCB_SEV_INFO)
  55                 return false;
  56
  57         if (GHCB_PROTO_MAX(val) < GHCB_PROTO_OUR ||
  58             GHCB_PROTO_MIN(val) > GHCB_PROTO_OUR)
  59                 return false;
  60
  61         return true;
  62 }
  63
  64 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
  65 {
  66         ghcb->save.sw_exit_code = 0;
  67         memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
  68 }
  69
  70 static bool vc_decoding_needed(unsigned long exit_code)
  71 {
  72         /* Exceptions don't require to decode the instruction */
  73         return !(exit_code >= SVM_EXIT_EXCP_BASE &&
  74                  exit_code <= SVM_EXIT_LAST_EXCP);
  75 }
  76
  77 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
  78                                       struct pt_regs *regs,
  79                                       unsigned long exit_code)
  80 {
  81         enum es_result ret = ES_OK;
  82
  83         memset(ctxt, 0, sizeof(*ctxt));
  84         ctxt->regs = regs;
  85
  86         if (vc_decoding_needed(exit_code))
  87                 ret = vc_decode_insn(ctxt);
  88
  89         return ret;
  90 }
  91
  92 static void vc_finish_insn(struct es_em_ctxt *ctxt)
  93 {
  94         ctxt->regs->ip += ctxt->insn.length;
  95 }
  96
  97 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
  98                                           struct es_em_ctxt *ctxt,
  99                                           u64 exit_code, u64 exit_info_1,
 100                                           u64 exit_info_2)
 101 {
 102         enum es_result ret;
 103
 104         /* Fill in protocol and format specifiers */
 105         ghcb->protocol_version = GHCB_PROTOCOL_MAX;
 106         ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
 107
 108         ghcb_set_sw_exit_code(ghcb, exit_code);
 109         ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
 110         ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
 111
 112         sev_es_wr_ghcb_msr(__pa(ghcb));
 113         VMGEXIT();
 114
 115         if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
 116                 u64 info = ghcb->save.sw_exit_info_2;
 117                 unsigned long v;
 118
 119                 info = ghcb->save.sw_exit_info_2;
 120                 v = info & SVM_EVTINJ_VEC_MASK;
 121
 122                 /* Check if exception information from hypervisor is sane. */
 123                 if ((info & SVM_EVTINJ_VALID) &&
 124                     ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
 125                     ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
 126                         ctxt->fi.vector = v;
 127                         if (info & SVM_EVTINJ_VALID_ERR)
 128                                 ctxt->fi.error_code = info >> 32;
 129                         ret = ES_EXCEPTION;
 130                 } else {
 131                         ret = ES_VMM_ERROR;
 132                 }
 133         } else if (ghcb->save.sw_exit_info_1 & 0xffffffff) {
 134                 ret = ES_VMM_ERROR;
 135         } else {
 136                 ret = ES_OK;
 137         }
 138
 139         return ret;
 140 }
 141
 142 /*
 143  * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
 144  * page yet, so it only supports the MSR based communication with the
 145  * hypervisor and only the CPUID exit-code.
 146  */
 147 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
 148 {
 149         unsigned int fn = lower_bits(regs->ax, 32);
 150         unsigned long val;
 151
 152         /* Only CPUID is supported via MSR protocol */
 153         if (exit_code != SVM_EXIT_CPUID)
 154                 goto fail;
 155
 156         sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
 157         VMGEXIT();
 158         val = sev_es_rd_ghcb_msr();
 159         if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
 160                 goto fail;
 161         regs->ax = val >> 32;
 162
 163         sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
 164         VMGEXIT();
 165         val = sev_es_rd_ghcb_msr();
 166         if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
 167                 goto fail;
 168         regs->bx = val >> 32;
 169
 170         sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
 171         VMGEXIT();
 172         val = sev_es_rd_ghcb_msr();
 173         if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
 174                 goto fail;
 175         regs->cx = val >> 32;
 176
 177         sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
 178         VMGEXIT();
 179         val = sev_es_rd_ghcb_msr();
 180         if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
 181                 goto fail;
 182         regs->dx = val >> 32;
 183
 184         /*
 185          * This is a VC handler and the #VC is only raised when SEV-ES is
 186          * active, which means SEV must be active too. Do sanity checks on the
 187          * CPUID results to make sure the hypervisor does not trick the kernel
 188          * into the no-sev path. This could map sensitive data unencrypted and
 189          * make it accessible to the hypervisor.
 190          *
 191          * In particular, check for:
 192          *      - Availability of CPUID leaf 0x8000001f
 193          *      - SEV CPUID bit.
 194          *
 195          * The hypervisor might still report the wrong C-bit position, but this
 196          * can't be checked here.
 197          */
 198
 199         if (fn == 0x80000000 && (regs->ax < 0x8000001f))
 200                 /* SEV leaf check */
 201                 goto fail;
 202         else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
 203                 /* SEV bit */
 204                 goto fail;
 205
 206         /* Skip over the CPUID two-byte opcode */
 207         regs->ip += 2;
 208
 209         return;
 210
 211 fail:
 212         sev_es_wr_ghcb_msr(GHCB_SEV_TERMINATE);
 213         VMGEXIT();
 214
 215         /* Shouldn't get here - if we do halt the machine */
 216         while (true)
 217                 asm volatile("hlt\n");
 218 }
 219
 220 static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
 221                                            unsigned long address,
 222                                            bool write)
 223 {
 224         if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
 225                 ctxt->fi.vector     = X86_TRAP_PF;
 226                 ctxt->fi.error_code = X86_PF_USER;
 227                 ctxt->fi.cr2        = address;
 228                 if (write)
 229                         ctxt->fi.error_code |= X86_PF_WRITE;
 230
 231                 return ES_EXCEPTION;
 232         }
 233
 234         return ES_OK;
 235 }
 236
 237 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
 238                                           void *src, char *buf,
 239                                           unsigned int data_size,
 240                                           unsigned int count,
 241                                           bool backwards)
 242 {
 243         int i, b = backwards ? -1 : 1;
 244         unsigned long address = (unsigned long)src;
 245         enum es_result ret;
 246
 247         ret = vc_insn_string_check(ctxt, address, false);
 248         if (ret != ES_OK)
 249                 return ret;
 250
 251         for (i = 0; i < count; i++) {
 252                 void *s = src + (i * data_size * b);
 253                 char *d = buf + (i * data_size);
 254
 255                 ret = vc_read_mem(ctxt, s, d, data_size);
 256                 if (ret != ES_OK)
 257                         break;
 258         }
 259
 260         return ret;
 261 }
 262
 263 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
 264                                            void *dst, char *buf,
 265                                            unsigned int data_size,
 266                                            unsigned int count,
 267                                            bool backwards)
 268 {
 269         int i, s = backwards ? -1 : 1;
 270         unsigned long address = (unsigned long)dst;
 271         enum es_result ret;
 272
 273         ret = vc_insn_string_check(ctxt, address, true);
 274         if (ret != ES_OK)
 275                 return ret;
 276
 277         for (i = 0; i < count; i++) {
 278                 void *d = dst + (i * data_size * s);
 279                 char *b = buf + (i * data_size);
 280
 281                 ret = vc_write_mem(ctxt, d, b, data_size);
 282                 if (ret != ES_OK)
 283                         break;
 284         }
 285
 286         return ret;
 287 }
 288
 289 #define IOIO_TYPE_STR  BIT(2)
 290 #define IOIO_TYPE_IN   1
 291 #define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
 292 #define IOIO_TYPE_OUT  0
 293 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
 294
 295 #define IOIO_REP       BIT(3)
 296
 297 #define IOIO_ADDR_64   BIT(9)
 298 #define IOIO_ADDR_32   BIT(8)
 299 #define IOIO_ADDR_16   BIT(7)
 300
 301 #define IOIO_DATA_32   BIT(6)
 302 #define IOIO_DATA_16   BIT(5)
 303 #define IOIO_DATA_8    BIT(4)
 304
 305 #define IOIO_SEG_ES    (0 << 10)
 306 #define IOIO_SEG_DS    (3 << 10)
 307
 308 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
 309 {
 310         struct insn *insn = &ctxt->insn;
 311         size_t size;
 312         u64 port;
 313
 314         *exitinfo = 0;
 315
 316         switch (insn->opcode.bytes[0]) {
 317         /* INS opcodes */
 318         case 0x6c:
 319         case 0x6d:
 320                 *exitinfo |= IOIO_TYPE_INS;
 321                 *exitinfo |= IOIO_SEG_ES;
 322                 port       = ctxt->regs->dx & 0xffff;
 323                 break;
 324
 325         /* OUTS opcodes */
 326         case 0x6e:
 327         case 0x6f:
 328                 *exitinfo |= IOIO_TYPE_OUTS;
 329                 *exitinfo |= IOIO_SEG_DS;
 330                 port       = ctxt->regs->dx & 0xffff;
 331                 break;
 332
 333         /* IN immediate opcodes */
 334         case 0xe4:
 335         case 0xe5:
 336                 *exitinfo |= IOIO_TYPE_IN;
 337                 port       = (u8)insn->immediate.value & 0xffff;
 338                 break;
 339
 340         /* OUT immediate opcodes */
 341         case 0xe6:
 342         case 0xe7:
 343                 *exitinfo |= IOIO_TYPE_OUT;
 344                 port       = (u8)insn->immediate.value & 0xffff;
 345                 break;
 346
 347         /* IN register opcodes */
 348         case 0xec:
 349         case 0xed:
 350                 *exitinfo |= IOIO_TYPE_IN;
 351                 port       = ctxt->regs->dx & 0xffff;
 352                 break;
 353
 354         /* OUT register opcodes */
 355         case 0xee:
 356         case 0xef:
 357                 *exitinfo |= IOIO_TYPE_OUT;
 358                 port       = ctxt->regs->dx & 0xffff;
 359                 break;
 360
 361         default:
 362                 return ES_DECODE_FAILED;
 363         }
 364
 365         *exitinfo |= port << 16;
 366
 367         switch (insn->opcode.bytes[0]) {
 368         case 0x6c:
 369         case 0x6e:
 370         case 0xe4:
 371         case 0xe6:
 372         case 0xec:
 373         case 0xee:
 374                 /* Single byte opcodes */
 375                 *exitinfo |= IOIO_DATA_8;
 376                 size       = 1;
 377                 break;
 378         default:
 379                 /* Length determined by instruction parsing */
 380                 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
 381                                                      : IOIO_DATA_32;
 382                 size       = (insn->opnd_bytes == 2) ? 2 : 4;
 383         }
 384
 385         switch (insn->addr_bytes) {
 386         case 2:
 387                 *exitinfo |= IOIO_ADDR_16;
 388                 break;
 389         case 4:
 390                 *exitinfo |= IOIO_ADDR_32;
 391                 break;
 392         case 8:
 393                 *exitinfo |= IOIO_ADDR_64;
 394                 break;
 395         }
 396
 397         if (insn_has_rep_prefix(insn))
 398                 *exitinfo |= IOIO_REP;
 399
 400         return vc_ioio_check(ctxt, (u16)port, size);
 401 }
 402
 403 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 404 {
 405         struct pt_regs *regs = ctxt->regs;
 406         u64 exit_info_1, exit_info_2;
 407         enum es_result ret;
 408
 409         ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
 410         if (ret != ES_OK)
 411                 return ret;
 412
 413         if (exit_info_1 & IOIO_TYPE_STR) {
 414
 415                 /* (REP) INS/OUTS */
 416
 417                 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
 418                 unsigned int io_bytes, exit_bytes;
 419                 unsigned int ghcb_count, op_count;
 420                 unsigned long es_base;
 421                 u64 sw_scratch;
 422
 423                 /*
 424                  * For the string variants with rep prefix the amount of in/out
 425                  * operations per #VC exception is limited so that the kernel
 426                  * has a chance to take interrupts and re-schedule while the
 427                  * instruction is emulated.
 428                  */
 429                 io_bytes   = (exit_info_1 >> 4) & 0x7;
 430                 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
 431
 432                 op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
 433                 exit_info_2 = min(op_count, ghcb_count);
 434                 exit_bytes  = exit_info_2 * io_bytes;
 435
 436                 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
 437
 438                 /* Read bytes of OUTS into the shared buffer */
 439                 if (!(exit_info_1 & IOIO_TYPE_IN)) {
 440                         ret = vc_insn_string_read(ctxt,
 441                                                (void *)(es_base + regs->si),
 442                                                ghcb->shared_buffer, io_bytes,
 443                                                exit_info_2, df);
 444                         if (ret)
 445                                 return ret;
 446                 }
 447
 448                 /*
 449                  * Issue an VMGEXIT to the HV to consume the bytes from the
 450                  * shared buffer or to have it write them into the shared buffer
 451                  * depending on the instruction: OUTS or INS.
 452                  */
 453                 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
 454                 ghcb_set_sw_scratch(ghcb, sw_scratch);
 455                 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
 456                                           exit_info_1, exit_info_2);
 457                 if (ret != ES_OK)
 458                         return ret;
 459
 460                 /* Read bytes from shared buffer into the guest's destination. */
 461                 if (exit_info_1 & IOIO_TYPE_IN) {
 462                         ret = vc_insn_string_write(ctxt,
 463                                                    (void *)(es_base + regs->di),
 464                                                    ghcb->shared_buffer, io_bytes,
 465                                                    exit_info_2, df);
 466                         if (ret)
 467                                 return ret;
 468
 469                         if (df)
 470                                 regs->di -= exit_bytes;
 471                         else
 472                                 regs->di += exit_bytes;
 473                 } else {
 474                         if (df)
 475                                 regs->si -= exit_bytes;
 476                         else
 477                                 regs->si += exit_bytes;
 478                 }
 479
 480                 if (exit_info_1 & IOIO_REP)
 481                         regs->cx -= exit_info_2;
 482
 483                 ret = regs->cx ? ES_RETRY : ES_OK;
 484
 485         } else {
 486
 487                 /* IN/OUT into/from rAX */
 488
 489                 int bits = (exit_info_1 & 0x70) >> 1;
 490                 u64 rax = 0;
 491
 492                 if (!(exit_info_1 & IOIO_TYPE_IN))
 493                         rax = lower_bits(regs->ax, bits);
 494
 495                 ghcb_set_rax(ghcb, rax);
 496
 497                 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
 498                 if (ret != ES_OK)
 499                         return ret;
 500
 501                 if (exit_info_1 & IOIO_TYPE_IN) {
 502                         if (!ghcb_rax_is_valid(ghcb))
 503                                 return ES_VMM_ERROR;
 504                         regs->ax = lower_bits(ghcb->save.rax, bits);
 505                 }
 506         }
 507
 508         return ret;
 509 }
 510
 511 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
 512                                       struct es_em_ctxt *ctxt)
 513 {
 514         struct pt_regs *regs = ctxt->regs;
 515         u32 cr4 = native_read_cr4();
 516         enum es_result ret;
 517
 518         ghcb_set_rax(ghcb, regs->ax);
 519         ghcb_set_rcx(ghcb, regs->cx);
 520
 521         if (cr4 & X86_CR4_OSXSAVE)
 522                 /* Safe to read xcr0 */
 523                 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
 524         else
 525                 /* xgetbv will cause #GP - use reset value for xcr0 */
 526                 ghcb_set_xcr0(ghcb, 1);
 527
 528         ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
 529         if (ret != ES_OK)
 530                 return ret;
 531
 532         if (!(ghcb_rax_is_valid(ghcb) &&
 533               ghcb_rbx_is_valid(ghcb) &&
 534               ghcb_rcx_is_valid(ghcb) &&
 535               ghcb_rdx_is_valid(ghcb)))
 536                 return ES_VMM_ERROR;
 537
 538         regs->ax = ghcb->save.rax;
 539         regs->bx = ghcb->save.rbx;
 540         regs->cx = ghcb->save.rcx;
 541         regs->dx = ghcb->save.rdx;
 542
 543         return ES_OK;
 544 }
 545
 546 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
 547                                       struct es_em_ctxt *ctxt,
 548                                       unsigned long exit_code)
 549 {
 550         bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
 551         enum es_result ret;
 552
 553         ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
 554         if (ret != ES_OK)
 555                 return ret;
 556
 557         if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
 558              (!rdtscp || ghcb_rcx_is_valid(ghcb))))
 559                 return ES_VMM_ERROR;
 560
 561         ctxt->regs->ax = ghcb->save.rax;
 562         ctxt->regs->dx = ghcb->save.rdx;
 563         if (rdtscp)
 564                 ctxt->regs->cx = ghcb->save.rcx;
 565
 566         return ES_OK;
 567 }