drivers/vfio/pci/pds/cmds.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */
   3
   4 #include <linux/io.h>
   5 #include <linux/types.h>
   6 #include <linux/delay.h>
   7
   8 #include <linux/pds/pds_common.h>
   9 #include <linux/pds/pds_core_if.h>
  10 #include <linux/pds/pds_adminq.h>
  11
  12 #include "vfio_dev.h"
  13 #include "cmds.h"
  14
  15 #define SUSPEND_TIMEOUT_S               5
  16 #define SUSPEND_CHECK_INTERVAL_MS       1
  17
  18 static int pds_vfio_client_adminq_cmd(struct pds_vfio_pci_device *pds_vfio,
  19                                       union pds_core_adminq_cmd *req,
  20                                       union pds_core_adminq_comp *resp,
  21                                       bool fast_poll)
  22 {
  23         struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
  24         union pds_core_adminq_cmd cmd = {};
  25         struct pdsc *pdsc;
  26         int err;
  27
  28         /* Wrap the client request */
  29         cmd.client_request.opcode = PDS_AQ_CMD_CLIENT_CMD;
  30         cmd.client_request.client_id = cpu_to_le16(pds_vfio->client_id);
  31         memcpy(cmd.client_request.client_cmd, req,
  32                sizeof(cmd.client_request.client_cmd));
  33
  34         pdsc = pdsc_get_pf_struct(pdev);
  35         if (IS_ERR(pdsc))
  36                 return PTR_ERR(pdsc);
  37
  38         err = pdsc_adminq_post(pdsc, &cmd, resp, fast_poll);
  39         if (err && err != -EAGAIN)
  40                 dev_err(pds_vfio_to_dev(pds_vfio),
  41                         "client admin cmd failed: %pe\n", ERR_PTR(err));
  42
  43         return err;
  44 }
  45
  46 int pds_vfio_register_client_cmd(struct pds_vfio_pci_device *pds_vfio)
  47 {
  48         struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
  49         char devname[PDS_DEVNAME_LEN];
  50         struct pdsc *pdsc;
  51         int ci;
  52
  53         snprintf(devname, sizeof(devname), "%s.%d-%u", PDS_VFIO_LM_DEV_NAME,
  54                  pci_domain_nr(pdev->bus),
  55                  PCI_DEVID(pdev->bus->number, pdev->devfn));
  56
  57         pdsc = pdsc_get_pf_struct(pdev);
  58         if (IS_ERR(pdsc))
  59                 return PTR_ERR(pdsc);
  60
  61         ci = pds_client_register(pdsc, devname);
  62         if (ci < 0)
  63                 return ci;
  64
  65         pds_vfio->client_id = ci;
  66
  67         return 0;
  68 }
  69
  70 void pds_vfio_unregister_client_cmd(struct pds_vfio_pci_device *pds_vfio)
  71 {
  72         struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
  73         struct pdsc *pdsc;
  74         int err;
  75
  76         pdsc = pdsc_get_pf_struct(pdev);
  77         if (IS_ERR(pdsc))
  78                 return;
  79
  80         err = pds_client_unregister(pdsc, pds_vfio->client_id);
  81         if (err)
  82                 dev_err(&pdev->dev, "unregister from DSC failed: %pe\n",
  83                         ERR_PTR(err));
  84
  85         pds_vfio->client_id = 0;
  86 }
  87
  88 static int
  89 pds_vfio_suspend_wait_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
  90 {
  91         union pds_core_adminq_cmd cmd = {
  92                 .lm_suspend_status = {
  93                         .opcode = PDS_LM_CMD_SUSPEND_STATUS,
  94                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
  95                         .type = type,
  96                 },
  97         };
  98         struct device *dev = pds_vfio_to_dev(pds_vfio);
  99         union pds_core_adminq_comp comp = {};
 100         unsigned long time_limit;
 101         unsigned long time_start;
 102         unsigned long time_done;
 103         int err;
 104
 105         time_start = jiffies;
 106         time_limit = time_start + HZ * SUSPEND_TIMEOUT_S;
 107         do {
 108                 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
 109                 if (err != -EAGAIN)
 110                         break;
 111
 112                 msleep(SUSPEND_CHECK_INTERVAL_MS);
 113         } while (time_before(jiffies, time_limit));
 114
 115         time_done = jiffies;
 116         dev_dbg(dev, "%s: vf%u: Suspend comp received in %d msecs\n", __func__,
 117                 pds_vfio->vf_id, jiffies_to_msecs(time_done - time_start));
 118
 119         /* Check the results */
 120         if (time_after_eq(time_done, time_limit)) {
 121                 dev_err(dev, "%s: vf%u: Suspend comp timeout\n", __func__,
 122                         pds_vfio->vf_id);
 123                 err = -ETIMEDOUT;
 124         }
 125
 126         return err;
 127 }
 128
 129 int pds_vfio_suspend_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
 130 {
 131         union pds_core_adminq_cmd cmd = {
 132                 .lm_suspend = {
 133                         .opcode = PDS_LM_CMD_SUSPEND,
 134                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 135                         .type = type,
 136                 },
 137         };
 138         struct device *dev = pds_vfio_to_dev(pds_vfio);
 139         union pds_core_adminq_comp comp = {};
 140         int err;
 141
 142         dev_dbg(dev, "vf%u: Suspend device\n", pds_vfio->vf_id);
 143
 144         /*
 145          * The initial suspend request to the firmware starts the device suspend
 146          * operation and the firmware returns success if it's started
 147          * successfully.
 148          */
 149         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
 150         if (err) {
 151                 dev_err(dev, "vf%u: Suspend failed: %pe\n", pds_vfio->vf_id,
 152                         ERR_PTR(err));
 153                 return err;
 154         }
 155
 156         /*
 157          * The subsequent suspend status request(s) check if the firmware has
 158          * completed the device suspend process.
 159          */
 160         return pds_vfio_suspend_wait_device_cmd(pds_vfio, type);
 161 }
 162
 163 int pds_vfio_resume_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)
 164 {
 165         union pds_core_adminq_cmd cmd = {
 166                 .lm_resume = {
 167                         .opcode = PDS_LM_CMD_RESUME,
 168                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 169                         .type = type,
 170                 },
 171         };
 172         struct device *dev = pds_vfio_to_dev(pds_vfio);
 173         union pds_core_adminq_comp comp = {};
 174
 175         dev_dbg(dev, "vf%u: Resume device\n", pds_vfio->vf_id);
 176
 177         return pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);
 178 }
 179
 180 int pds_vfio_get_lm_state_size_cmd(struct pds_vfio_pci_device *pds_vfio, u64 *size)
 181 {
 182         union pds_core_adminq_cmd cmd = {
 183                 .lm_state_size = {
 184                         .opcode = PDS_LM_CMD_STATE_SIZE,
 185                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 186                 },
 187         };
 188         struct device *dev = pds_vfio_to_dev(pds_vfio);
 189         union pds_core_adminq_comp comp = {};
 190         int err;
 191
 192         dev_dbg(dev, "vf%u: Get migration status\n", pds_vfio->vf_id);
 193
 194         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
 195         if (err)
 196                 return err;
 197
 198         *size = le64_to_cpu(comp.lm_state_size.size);
 199         return 0;
 200 }
 201
 202 static int pds_vfio_dma_map_lm_file(struct device *dev,
 203                                     enum dma_data_direction dir,
 204                                     struct pds_vfio_lm_file *lm_file)
 205 {
 206         struct pds_lm_sg_elem *sgl, *sge;
 207         struct scatterlist *sg;
 208         dma_addr_t sgl_addr;
 209         size_t sgl_size;
 210         int err;
 211         int i;
 212
 213         if (!lm_file)
 214                 return -EINVAL;
 215
 216         /* dma map file pages */
 217         err = dma_map_sgtable(dev, &lm_file->sg_table, dir, 0);
 218         if (err)
 219                 return err;
 220
 221         lm_file->num_sge = lm_file->sg_table.nents;
 222
 223         /* alloc sgl */
 224         sgl_size = lm_file->num_sge * sizeof(struct pds_lm_sg_elem);
 225         sgl = kzalloc(sgl_size, GFP_KERNEL);
 226         if (!sgl) {
 227                 err = -ENOMEM;
 228                 goto out_unmap_sgtable;
 229         }
 230
 231         /* fill sgl */
 232         sge = sgl;
 233         for_each_sgtable_dma_sg(&lm_file->sg_table, sg, i) {
 234                 sge->addr = cpu_to_le64(sg_dma_address(sg));
 235                 sge->len = cpu_to_le32(sg_dma_len(sg));
 236                 dev_dbg(dev, "addr = %llx, len = %u\n", sge->addr, sge->len);
 237                 sge++;
 238         }
 239
 240         sgl_addr = dma_map_single(dev, sgl, sgl_size, DMA_TO_DEVICE);
 241         if (dma_mapping_error(dev, sgl_addr)) {
 242                 err = -EIO;
 243                 goto out_free_sgl;
 244         }
 245
 246         lm_file->sgl = sgl;
 247         lm_file->sgl_addr = sgl_addr;
 248
 249         return 0;
 250
 251 out_free_sgl:
 252         kfree(sgl);
 253 out_unmap_sgtable:
 254         lm_file->num_sge = 0;
 255         dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0);
 256         return err;
 257 }
 258
 259 static void pds_vfio_dma_unmap_lm_file(struct device *dev,
 260                                        enum dma_data_direction dir,
 261                                        struct pds_vfio_lm_file *lm_file)
 262 {
 263         if (!lm_file)
 264                 return;
 265
 266         /* free sgl */
 267         if (lm_file->sgl) {
 268                 dma_unmap_single(dev, lm_file->sgl_addr,
 269                                  lm_file->num_sge * sizeof(*lm_file->sgl),
 270                                  DMA_TO_DEVICE);
 271                 kfree(lm_file->sgl);
 272                 lm_file->sgl = NULL;
 273                 lm_file->sgl_addr = DMA_MAPPING_ERROR;
 274                 lm_file->num_sge = 0;
 275         }
 276
 277         /* dma unmap file pages */
 278         dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0);
 279 }
 280
 281 int pds_vfio_get_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio)
 282 {
 283         union pds_core_adminq_cmd cmd = {
 284                 .lm_save = {
 285                         .opcode = PDS_LM_CMD_SAVE,
 286                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 287                 },
 288         };
 289         struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
 290         struct device *pdsc_dev = &pci_physfn(pdev)->dev;
 291         union pds_core_adminq_comp comp = {};
 292         struct pds_vfio_lm_file *lm_file;
 293         int err;
 294
 295         dev_dbg(&pdev->dev, "vf%u: Get migration state\n", pds_vfio->vf_id);
 296
 297         lm_file = pds_vfio->save_file;
 298
 299         err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file);
 300         if (err) {
 301                 dev_err(&pdev->dev, "failed to map save migration file: %pe\n",
 302                         ERR_PTR(err));
 303                 return err;
 304         }
 305
 306         cmd.lm_save.sgl_addr = cpu_to_le64(lm_file->sgl_addr);
 307         cmd.lm_save.num_sge = cpu_to_le32(lm_file->num_sge);
 308
 309         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
 310         if (err)
 311                 dev_err(&pdev->dev, "failed to get migration state: %pe\n",
 312                         ERR_PTR(err));
 313
 314         pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file);
 315
 316         return err;
 317 }
 318
 319 int pds_vfio_set_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio)
 320 {
 321         union pds_core_adminq_cmd cmd = {
 322                 .lm_restore = {
 323                         .opcode = PDS_LM_CMD_RESTORE,
 324                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 325                 },
 326         };
 327         struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);
 328         struct device *pdsc_dev = &pci_physfn(pdev)->dev;
 329         union pds_core_adminq_comp comp = {};
 330         struct pds_vfio_lm_file *lm_file;
 331         int err;
 332
 333         dev_dbg(&pdev->dev, "vf%u: Set migration state\n", pds_vfio->vf_id);
 334
 335         lm_file = pds_vfio->restore_file;
 336
 337         err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file);
 338         if (err) {
 339                 dev_err(&pdev->dev,
 340                         "failed to map restore migration file: %pe\n",
 341                         ERR_PTR(err));
 342                 return err;
 343         }
 344
 345         cmd.lm_restore.sgl_addr = cpu_to_le64(lm_file->sgl_addr);
 346         cmd.lm_restore.num_sge = cpu_to_le32(lm_file->num_sge);
 347
 348         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
 349         if (err)
 350                 dev_err(&pdev->dev, "failed to set migration state: %pe\n",
 351                         ERR_PTR(err));
 352
 353         pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file);
 354
 355         return err;
 356 }
 357
 358 void pds_vfio_send_host_vf_lm_status_cmd(struct pds_vfio_pci_device *pds_vfio,
 359                                          enum pds_lm_host_vf_status vf_status)
 360 {
 361         union pds_core_adminq_cmd cmd = {
 362                 .lm_host_vf_status = {
 363                         .opcode = PDS_LM_CMD_HOST_VF_STATUS,
 364                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 365                         .status = vf_status,
 366                 },
 367         };
 368         struct device *dev = pds_vfio_to_dev(pds_vfio);
 369         union pds_core_adminq_comp comp = {};
 370         int err;
 371
 372         dev_dbg(dev, "vf%u: Set host VF LM status: %u", pds_vfio->vf_id,
 373                 vf_status);
 374         if (vf_status != PDS_LM_STA_IN_PROGRESS &&
 375             vf_status != PDS_LM_STA_NONE) {
 376                 dev_warn(dev, "Invalid host VF migration status, %d\n",
 377                          vf_status);
 378                 return;
 379         }
 380
 381         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
 382         if (err)
 383                 dev_warn(dev, "failed to send host VF migration status: %pe\n",
 384                          ERR_PTR(err));
 385 }
 386
 387 int pds_vfio_dirty_status_cmd(struct pds_vfio_pci_device *pds_vfio,
 388                               u64 regions_dma, u8 *max_regions, u8 *num_regions)
 389 {
 390         union pds_core_adminq_cmd cmd = {
 391                 .lm_dirty_status = {
 392                         .opcode = PDS_LM_CMD_DIRTY_STATUS,
 393                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 394                 },
 395         };
 396         struct device *dev = pds_vfio_to_dev(pds_vfio);
 397         union pds_core_adminq_comp comp = {};
 398         int err;
 399
 400         dev_dbg(dev, "vf%u: Dirty status\n", pds_vfio->vf_id);
 401
 402         cmd.lm_dirty_status.regions_dma = cpu_to_le64(regions_dma);
 403         cmd.lm_dirty_status.max_regions = *max_regions;
 404
 405         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
 406         if (err) {
 407                 dev_err(dev, "failed to get dirty status: %pe\n", ERR_PTR(err));
 408                 return err;
 409         }
 410
 411         /* only support seq_ack approach for now */
 412         if (!(le32_to_cpu(comp.lm_dirty_status.bmp_type_mask) &
 413               BIT(PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK))) {
 414                 dev_err(dev, "Dirty bitmap tracking SEQ_ACK not supported\n");
 415                 return -EOPNOTSUPP;
 416         }
 417
 418         *num_regions = comp.lm_dirty_status.num_regions;
 419         *max_regions = comp.lm_dirty_status.max_regions;
 420
 421         dev_dbg(dev,
 422                 "Page Tracking Status command successful, max_regions: %d, num_regions: %d, bmp_type: %s\n",
 423                 *max_regions, *num_regions, "PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK");
 424
 425         return 0;
 426 }
 427
 428 int pds_vfio_dirty_enable_cmd(struct pds_vfio_pci_device *pds_vfio,
 429                               u64 regions_dma, u8 num_regions)
 430 {
 431         union pds_core_adminq_cmd cmd = {
 432                 .lm_dirty_enable = {
 433                         .opcode = PDS_LM_CMD_DIRTY_ENABLE,
 434                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 435                         .regions_dma = cpu_to_le64(regions_dma),
 436                         .bmp_type = PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK,
 437                         .num_regions = num_regions,
 438                 },
 439         };
 440         struct device *dev = pds_vfio_to_dev(pds_vfio);
 441         union pds_core_adminq_comp comp = {};
 442         int err;
 443
 444         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
 445         if (err) {
 446                 dev_err(dev, "failed dirty tracking enable: %pe\n",
 447                         ERR_PTR(err));
 448                 return err;
 449         }
 450
 451         return 0;
 452 }
 453
 454 int pds_vfio_dirty_disable_cmd(struct pds_vfio_pci_device *pds_vfio)
 455 {
 456         union pds_core_adminq_cmd cmd = {
 457                 .lm_dirty_disable = {
 458                         .opcode = PDS_LM_CMD_DIRTY_DISABLE,
 459                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 460                 },
 461         };
 462         struct device *dev = pds_vfio_to_dev(pds_vfio);
 463         union pds_core_adminq_comp comp = {};
 464         int err;
 465
 466         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
 467         if (err || comp.lm_dirty_status.num_regions != 0) {
 468                 /* in case num_regions is still non-zero after disable */
 469                 err = err ? err : -EIO;
 470                 dev_err(dev,
 471                         "failed dirty tracking disable: %pe, num_regions %d\n",
 472                         ERR_PTR(err), comp.lm_dirty_status.num_regions);
 473                 return err;
 474         }
 475
 476         return 0;
 477 }
 478
 479 int pds_vfio_dirty_seq_ack_cmd(struct pds_vfio_pci_device *pds_vfio,
 480                                u64 sgl_dma, u16 num_sge, u32 offset,
 481                                u32 total_len, bool read_seq)
 482 {
 483         const char *cmd_type_str = read_seq ? "read_seq" : "write_ack";
 484         union pds_core_adminq_cmd cmd = {
 485                 .lm_dirty_seq_ack = {
 486                         .vf_id = cpu_to_le16(pds_vfio->vf_id),
 487                         .len_bytes = cpu_to_le32(total_len),
 488                         .off_bytes = cpu_to_le32(offset),
 489                         .sgl_addr = cpu_to_le64(sgl_dma),
 490                         .num_sge = cpu_to_le16(num_sge),
 491                 },
 492         };
 493         struct device *dev = pds_vfio_to_dev(pds_vfio);
 494         union pds_core_adminq_comp comp = {};
 495         int err;
 496
 497         if (read_seq)
 498                 cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_READ_SEQ;
 499         else
 500                 cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_WRITE_ACK;
 501
 502         err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);
 503         if (err) {
 504                 dev_err(dev, "failed cmd Page Tracking %s: %pe\n", cmd_type_str,
 505                         ERR_PTR(err));
 506                 return err;
 507         }
 508
 509         return 0;
 510 }