ivm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Hyper-V Isolation VM interface with paravisor and hypervisor
   4  *
   5  * Author:
   6  *  Tianyu Lan <Tianyu.Lan@microsoft.com>
   7  */
   8
   9 #include <linux/bitfield.h>
  10 #include <linux/hyperv.h>
  11 #include <linux/types.h>
  12 #include <linux/slab.h>
  13 #include <asm/svm.h>
  14 #include <asm/sev.h>
  15 #include <asm/io.h>
  16 #include <asm/mshyperv.h>
  17 #include <asm/hypervisor.h>
  18
  19 #ifdef CONFIG_AMD_MEM_ENCRYPT
  20
  21 #define GHCB_USAGE_HYPERV_CALL  1
  22
  23 union hv_ghcb {
  24         struct ghcb ghcb;
  25         struct {
  26                 u64 hypercalldata[509];
  27                 u64 outputgpa;
  28                 union {
  29                         union {
  30                                 struct {
  31                                         u32 callcode        : 16;
  32                                         u32 isfast          : 1;
  33                                         u32 reserved1       : 14;
  34                                         u32 isnested        : 1;
  35                                         u32 countofelements : 12;
  36                                         u32 reserved2       : 4;
  37                                         u32 repstartindex   : 12;
  38                                         u32 reserved3       : 4;
  39                                 };
  40                                 u64 asuint64;
  41                         } hypercallinput;
  42                         union {
  43                                 struct {
  44                                         u16 callstatus;
  45                                         u16 reserved1;
  46                                         u32 elementsprocessed : 12;
  47                                         u32 reserved2         : 20;
  48                                 };
  49                                 u64 asunit64;
  50                         } hypercalloutput;
  51                 };
  52                 u64 reserved2;
  53         } hypercall;
  54 } __packed __aligned(HV_HYP_PAGE_SIZE);
  55
  56 static u16 hv_ghcb_version __ro_after_init;
  57
  58 u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size)
  59 {
  60         union hv_ghcb *hv_ghcb;
  61         void **ghcb_base;
  62         unsigned long flags;
  63         u64 status;
  64
  65         if (!hv_ghcb_pg)
  66                 return -EFAULT;
  67
  68         WARN_ON(in_nmi());
  69
  70         local_irq_save(flags);
  71         ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg);
  72         hv_ghcb = (union hv_ghcb *)*ghcb_base;
  73         if (!hv_ghcb) {
  74                 local_irq_restore(flags);
  75                 return -EFAULT;
  76         }
  77
  78         hv_ghcb->ghcb.protocol_version = GHCB_PROTOCOL_MAX;
  79         hv_ghcb->ghcb.ghcb_usage = GHCB_USAGE_HYPERV_CALL;
  80
  81         hv_ghcb->hypercall.outputgpa = (u64)output;
  82         hv_ghcb->hypercall.hypercallinput.asuint64 = 0;
  83         hv_ghcb->hypercall.hypercallinput.callcode = control;
  84
  85         if (input_size)
  86                 memcpy(hv_ghcb->hypercall.hypercalldata, input, input_size);
  87
  88         VMGEXIT();
  89
  90         hv_ghcb->ghcb.ghcb_usage = 0xffffffff;
  91         memset(hv_ghcb->ghcb.save.valid_bitmap, 0,
  92                sizeof(hv_ghcb->ghcb.save.valid_bitmap));
  93
  94         status = hv_ghcb->hypercall.hypercalloutput.callstatus;
  95
  96         local_irq_restore(flags);
  97
  98         return status;
  99 }
 100
 101 static inline u64 rd_ghcb_msr(void)
 102 {
 103         return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
 104 }
 105
 106 static inline void wr_ghcb_msr(u64 val)
 107 {
 108         native_wrmsrl(MSR_AMD64_SEV_ES_GHCB, val);
 109 }
 110
 111 static enum es_result hv_ghcb_hv_call(struct ghcb *ghcb, u64 exit_code,
 112                                    u64 exit_info_1, u64 exit_info_2)
 113 {
 114         /* Fill in protocol and format specifiers */
 115         ghcb->protocol_version = hv_ghcb_version;
 116         ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
 117
 118         ghcb_set_sw_exit_code(ghcb, exit_code);
 119         ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
 120         ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
 121
 122         VMGEXIT();
 123
 124         if (ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0))
 125                 return ES_VMM_ERROR;
 126         else
 127                 return ES_OK;
 128 }
 129
 130 void hv_ghcb_terminate(unsigned int set, unsigned int reason)
 131 {
 132         u64 val = GHCB_MSR_TERM_REQ;
 133
 134         /* Tell the hypervisor what went wrong. */
 135         val |= GHCB_SEV_TERM_REASON(set, reason);
 136
 137         /* Request Guest Termination from Hypvervisor */
 138         wr_ghcb_msr(val);
 139         VMGEXIT();
 140
 141         while (true)
 142                 asm volatile("hlt\n" : : : "memory");
 143 }
 144
 145 bool hv_ghcb_negotiate_protocol(void)
 146 {
 147         u64 ghcb_gpa;
 148         u64 val;
 149
 150         /* Save ghcb page gpa. */
 151         ghcb_gpa = rd_ghcb_msr();
 152
 153         /* Do the GHCB protocol version negotiation */
 154         wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
 155         VMGEXIT();
 156         val = rd_ghcb_msr();
 157
 158         if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
 159                 return false;
 160
 161         if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
 162             GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
 163                 return false;
 164
 165         hv_ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val),
 166                              GHCB_PROTOCOL_MAX);
 167
 168         /* Write ghcb page back after negotiating protocol. */
 169         wr_ghcb_msr(ghcb_gpa);
 170         VMGEXIT();
 171
 172         return true;
 173 }
 174
 175 void hv_ghcb_msr_write(u64 msr, u64 value)
 176 {
 177         union hv_ghcb *hv_ghcb;
 178         void **ghcb_base;
 179         unsigned long flags;
 180
 181         if (!hv_ghcb_pg)
 182                 return;
 183
 184         WARN_ON(in_nmi());
 185
 186         local_irq_save(flags);
 187         ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg);
 188         hv_ghcb = (union hv_ghcb *)*ghcb_base;
 189         if (!hv_ghcb) {
 190                 local_irq_restore(flags);
 191                 return;
 192         }
 193
 194         ghcb_set_rcx(&hv_ghcb->ghcb, msr);
 195         ghcb_set_rax(&hv_ghcb->ghcb, lower_32_bits(value));
 196         ghcb_set_rdx(&hv_ghcb->ghcb, upper_32_bits(value));
 197
 198         if (hv_ghcb_hv_call(&hv_ghcb->ghcb, SVM_EXIT_MSR, 1, 0))
 199                 pr_warn("Fail to write msr via ghcb %llx.\n", msr);
 200
 201         local_irq_restore(flags);
 202 }
 203 EXPORT_SYMBOL_GPL(hv_ghcb_msr_write);
 204
 205 void hv_ghcb_msr_read(u64 msr, u64 *value)
 206 {
 207         union hv_ghcb *hv_ghcb;
 208         void **ghcb_base;
 209         unsigned long flags;
 210
 211         /* Check size of union hv_ghcb here. */
 212         BUILD_BUG_ON(sizeof(union hv_ghcb) != HV_HYP_PAGE_SIZE);
 213
 214         if (!hv_ghcb_pg)
 215                 return;
 216
 217         WARN_ON(in_nmi());
 218
 219         local_irq_save(flags);
 220         ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg);
 221         hv_ghcb = (union hv_ghcb *)*ghcb_base;
 222         if (!hv_ghcb) {
 223                 local_irq_restore(flags);
 224                 return;
 225         }
 226
 227         ghcb_set_rcx(&hv_ghcb->ghcb, msr);
 228         if (hv_ghcb_hv_call(&hv_ghcb->ghcb, SVM_EXIT_MSR, 0, 0))
 229                 pr_warn("Fail to read msr via ghcb %llx.\n", msr);
 230         else
 231                 *value = (u64)lower_32_bits(hv_ghcb->ghcb.save.rax)
 232                         | ((u64)lower_32_bits(hv_ghcb->ghcb.save.rdx) << 32);
 233         local_irq_restore(flags);
 234 }
 235 EXPORT_SYMBOL_GPL(hv_ghcb_msr_read);
 236 #endif
 237
 238 enum hv_isolation_type hv_get_isolation_type(void)
 239 {
 240         if (!(ms_hyperv.priv_high & HV_ISOLATION))
 241                 return HV_ISOLATION_TYPE_NONE;
 242         return FIELD_GET(HV_ISOLATION_TYPE, ms_hyperv.isolation_config_b);
 243 }
 244 EXPORT_SYMBOL_GPL(hv_get_isolation_type);
 245
 246 /*
 247  * hv_is_isolation_supported - Check system runs in the Hyper-V
 248  * isolation VM.
 249  */
 250 bool hv_is_isolation_supported(void)
 251 {
 252         if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
 253                 return false;
 254
 255         if (!hypervisor_is_type(X86_HYPER_MS_HYPERV))
 256                 return false;
 257
 258         return hv_get_isolation_type() != HV_ISOLATION_TYPE_NONE;
 259 }
 260
 261 DEFINE_STATIC_KEY_FALSE(isolation_type_snp);
 262
 263 /*
 264  * hv_isolation_type_snp - Check system runs in the AMD SEV-SNP based
 265  * isolation VM.
 266  */
 267 bool hv_isolation_type_snp(void)
 268 {
 269         return static_branch_unlikely(&isolation_type_snp);
 270 }
 271
 272 /*
 273  * hv_mark_gpa_visibility - Set pages visible to host via hvcall.
 274  *
 275  * In Isolation VM, all guest memory is encrypted from host and guest
 276  * needs to set memory visible to host via hvcall before sharing memory
 277  * with host.
 278  */
 279 static int hv_mark_gpa_visibility(u16 count, const u64 pfn[],
 280                            enum hv_mem_host_visibility visibility)
 281 {
 282         struct hv_gpa_range_for_visibility **input_pcpu, *input;
 283         u16 pages_processed;
 284         u64 hv_status;
 285         unsigned long flags;
 286
 287         /* no-op if partition isolation is not enabled */
 288         if (!hv_is_isolation_supported())
 289                 return 0;
 290
 291         if (count > HV_MAX_MODIFY_GPA_REP_COUNT) {
 292                 pr_err("Hyper-V: GPA count:%d exceeds supported:%lu\n", count,
 293                         HV_MAX_MODIFY_GPA_REP_COUNT);
 294                 return -EINVAL;
 295         }
 296
 297         local_irq_save(flags);
 298         input_pcpu = (struct hv_gpa_range_for_visibility **)
 299                         this_cpu_ptr(hyperv_pcpu_input_arg);
 300         input = *input_pcpu;
 301         if (unlikely(!input)) {
 302                 local_irq_restore(flags);
 303                 return -EINVAL;
 304         }
 305
 306         input->partition_id = HV_PARTITION_ID_SELF;
 307         input->host_visibility = visibility;
 308         input->reserved0 = 0;
 309         input->reserved1 = 0;
 310         memcpy((void *)input->gpa_page_list, pfn, count * sizeof(*pfn));
 311         hv_status = hv_do_rep_hypercall(
 312                         HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY, count,
 313                         0, input, &pages_processed);
 314         local_irq_restore(flags);
 315
 316         if (hv_result_success(hv_status))
 317                 return 0;
 318         else
 319                 return -EFAULT;
 320 }
 321
 322 /*
 323  * hv_set_mem_host_visibility - Set specified memory visible to host.
 324  *
 325  * In Isolation VM, all guest memory is encrypted from host and guest
 326  * needs to set memory visible to host via hvcall before sharing memory
 327  * with host. This function works as wrap of hv_mark_gpa_visibility()
 328  * with memory base and size.
 329  */
 330 int hv_set_mem_host_visibility(unsigned long kbuffer, int pagecount, bool visible)
 331 {
 332         enum hv_mem_host_visibility visibility = visible ?
 333                         VMBUS_PAGE_VISIBLE_READ_WRITE : VMBUS_PAGE_NOT_VISIBLE;
 334         u64 *pfn_array;
 335         int ret = 0;
 336         int i, pfn;
 337
 338         if (!hv_is_isolation_supported() || !hv_hypercall_pg)
 339                 return 0;
 340
 341         pfn_array = kmalloc(HV_HYP_PAGE_SIZE, GFP_KERNEL);
 342         if (!pfn_array)
 343                 return -ENOMEM;
 344
 345         for (i = 0, pfn = 0; i < pagecount; i++) {
 346                 pfn_array[pfn] = virt_to_hvpfn((void *)kbuffer + i * HV_HYP_PAGE_SIZE);
 347                 pfn++;
 348
 349                 if (pfn == HV_MAX_MODIFY_GPA_REP_COUNT || i == pagecount - 1) {
 350                         ret = hv_mark_gpa_visibility(pfn, pfn_array,
 351                                                      visibility);
 352                         if (ret)
 353                                 goto err_free_pfn_array;
 354                         pfn = 0;
 355                 }
 356         }
 357
 358  err_free_pfn_array:
 359         kfree(pfn_array);
 360         return ret;
 361 }
 362
 363 /*
 364  * hv_map_memory - map memory to extra space in the AMD SEV-SNP Isolation VM.
 365  */
 366 void *hv_map_memory(void *addr, unsigned long size)
 367 {
 368         unsigned long *pfns = kcalloc(size / PAGE_SIZE,
 369                                       sizeof(unsigned long), GFP_KERNEL);
 370         void *vaddr;
 371         int i;
 372
 373         if (!pfns)
 374                 return NULL;
 375
 376         for (i = 0; i < size / PAGE_SIZE; i++)
 377                 pfns[i] = vmalloc_to_pfn(addr + i * PAGE_SIZE) +
 378                         (ms_hyperv.shared_gpa_boundary >> PAGE_SHIFT);
 379
 380         vaddr = vmap_pfn(pfns, size / PAGE_SIZE, PAGE_KERNEL_IO);
 381         kfree(pfns);
 382
 383         return vaddr;
 384 }
 385
 386 void hv_unmap_memory(void *addr)
 387 {
 388         vunmap(addr);
 389 }