pds/dirty.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */
   3
   4 #include <linux/interval_tree.h>
   5 #include <linux/vfio.h>
   6
   7 #include <linux/pds/pds_common.h>
   8 #include <linux/pds/pds_core_if.h>
   9 #include <linux/pds/pds_adminq.h>
  10
  11 #include "vfio_dev.h"
  12 #include "cmds.h"
  13 #include "dirty.h"
  14
  15 #define READ_SEQ true
  16 #define WRITE_ACK false
  17
  18 bool pds_vfio_dirty_is_enabled(struct pds_vfio_pci_device *pds_vfio)
  19 {
  20         return pds_vfio->dirty.is_enabled;
  21 }
  22
  23 void pds_vfio_dirty_set_enabled(struct pds_vfio_pci_device *pds_vfio)
  24 {
  25         pds_vfio->dirty.is_enabled = true;
  26 }
  27
  28 void pds_vfio_dirty_set_disabled(struct pds_vfio_pci_device *pds_vfio)
  29 {
  30         pds_vfio->dirty.is_enabled = false;
  31 }
  32
  33 static void
  34 pds_vfio_print_guest_region_info(struct pds_vfio_pci_device *pds_vfio,
  35                                  u8 max_regions)
  36 {
  37         int len = max_regions * sizeof(struct pds_lm_dirty_region_info);
  38         struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
  39         struct device *pdsc_dev = &pci_physfn(pdev)->dev;
  40         struct pds_lm_dirty_region_info *region_info;
  41         dma_addr_t regions_dma;
  42         u8 num_regions;
  43         int err;
  44
  45         region_info = kcalloc(max_regions,
  46                               sizeof(struct pds_lm_dirty_region_info),
  47                               GFP_KERNEL);
  48         if (!region_info)
  49                 return;
  50
  51         regions_dma =
  52                 dma_map_single(pdsc_dev, region_info, len, DMA_FROM_DEVICE);
  53         if (dma_mapping_error(pdsc_dev, regions_dma))
  54                 goto out_free_region_info;
  55
  56         err = pds_vfio_dirty_status_cmd(pds_vfio, regions_dma, &max_regions,
  57                                         &num_regions);
  58         dma_unmap_single(pdsc_dev, regions_dma, len, DMA_FROM_DEVICE);
  59         if (err)
  60                 goto out_free_region_info;
  61
  62         for (unsigned int i = 0; i < num_regions; i++)
  63                 dev_dbg(&pdev->dev,
  64                         "region_info[%d]: dma_base 0x%llx page_count %u page_size_log2 %u\n",
  65                         i, le64_to_cpu(region_info[i].dma_base),
  66                         le32_to_cpu(region_info[i].page_count),
  67                         region_info[i].page_size_log2);
  68
  69 out_free_region_info:
  70         kfree(region_info);
  71 }
  72
  73 static int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_dirty *dirty,
  74                                         unsigned long bytes)
  75 {
  76         unsigned long *host_seq_bmp, *host_ack_bmp;
  77
  78         host_seq_bmp = vzalloc(bytes);
  79         if (!host_seq_bmp)
  80                 return -ENOMEM;
  81
  82         host_ack_bmp = vzalloc(bytes);
  83         if (!host_ack_bmp) {
  84                 bitmap_free(host_seq_bmp);
  85                 return -ENOMEM;
  86         }
  87
  88         dirty->host_seq.bmp = host_seq_bmp;
  89         dirty->host_ack.bmp = host_ack_bmp;
  90
  91         return 0;
  92 }
  93
  94 static void pds_vfio_dirty_free_bitmaps(struct pds_vfio_dirty *dirty)
  95 {
  96         vfree(dirty->host_seq.bmp);
  97         vfree(dirty->host_ack.bmp);
  98         dirty->host_seq.bmp = NULL;
  99         dirty->host_ack.bmp = NULL;
 100 }
 101
 102 static void __pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device *pds_vfio,
 103                                       struct pds_vfio_bmp_info *bmp_info)
 104 {
 105         struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
 106         struct device *pdsc_dev = &pci_physfn(pdev)->dev;
 107
 108         dma_unmap_single(pdsc_dev, bmp_info->sgl_addr,
 109                          bmp_info->num_sge * sizeof(struct pds_lm_sg_elem),
 110                          DMA_BIDIRECTIONAL);
 111         kfree(bmp_info->sgl);
 112
 113         bmp_info->num_sge = 0;
 114         bmp_info->sgl = NULL;
 115         bmp_info->sgl_addr = 0;
 116 }
 117
 118 static void pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device *pds_vfio)
 119 {
 120         if (pds_vfio->dirty.host_seq.sgl)
 121                 __pds_vfio_dirty_free_sgl(pds_vfio, &pds_vfio->dirty.host_seq);
 122         if (pds_vfio->dirty.host_ack.sgl)
 123                 __pds_vfio_dirty_free_sgl(pds_vfio, &pds_vfio->dirty.host_ack);
 124 }
 125
 126 static int __pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device *pds_vfio,
 127                                       struct pds_vfio_bmp_info *bmp_info,
 128                                       u32 page_count)
 129 {
 130         struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
 131         struct device *pdsc_dev = &pci_physfn(pdev)->dev;
 132         struct pds_lm_sg_elem *sgl;
 133         dma_addr_t sgl_addr;
 134         size_t sgl_size;
 135         u32 max_sge;
 136
 137         max_sge = DIV_ROUND_UP(page_count, PAGE_SIZE * 8);
 138         sgl_size = max_sge * sizeof(struct pds_lm_sg_elem);
 139
 140         sgl = kzalloc(sgl_size, GFP_KERNEL);
 141         if (!sgl)
 142                 return -ENOMEM;
 143
 144         sgl_addr = dma_map_single(pdsc_dev, sgl, sgl_size, DMA_BIDIRECTIONAL);
 145         if (dma_mapping_error(pdsc_dev, sgl_addr)) {
 146                 kfree(sgl);
 147                 return -EIO;
 148         }
 149
 150         bmp_info->sgl = sgl;
 151         bmp_info->num_sge = max_sge;
 152         bmp_info->sgl_addr = sgl_addr;
 153
 154         return 0;
 155 }
 156
 157 static int pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device *pds_vfio,
 158                                     u32 page_count)
 159 {
 160         struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
 161         int err;
 162
 163         err = __pds_vfio_dirty_alloc_sgl(pds_vfio, &dirty->host_seq,
 164                                          page_count);
 165         if (err)
 166                 return err;
 167
 168         err = __pds_vfio_dirty_alloc_sgl(pds_vfio, &dirty->host_ack,
 169                                          page_count);
 170         if (err) {
 171                 __pds_vfio_dirty_free_sgl(pds_vfio, &dirty->host_seq);
 172                 return err;
 173         }
 174
 175         return 0;
 176 }
 177
 178 static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio,
 179                                  struct rb_root_cached *ranges, u32 nnodes,
 180                                  u64 *page_size)
 181 {
 182         struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
 183         struct device *pdsc_dev = &pci_physfn(pdev)->dev;
 184         struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
 185         u64 region_start, region_size, region_page_size;
 186         struct pds_lm_dirty_region_info *region_info;
 187         struct interval_tree_node *node = NULL;
 188         u8 max_regions = 0, num_regions;
 189         dma_addr_t regions_dma = 0;
 190         u32 num_ranges = nnodes;
 191         u32 page_count;
 192         u16 len;
 193         int err;
 194
 195         dev_dbg(&pdev->dev, "vf%u: Start dirty page tracking\n",
 196                 pds_vfio->vf_id);
 197
 198         if (pds_vfio_dirty_is_enabled(pds_vfio))
 199                 return -EINVAL;
 200
 201         /* find if dirty tracking is disabled, i.e. num_regions == 0 */
 202         err = pds_vfio_dirty_status_cmd(pds_vfio, 0, &max_regions,
 203                                         &num_regions);
 204         if (err < 0) {
 205                 dev_err(&pdev->dev, "Failed to get dirty status, err %pe\n",
 206                         ERR_PTR(err));
 207                 return err;
 208         } else if (num_regions) {
 209                 dev_err(&pdev->dev,
 210                         "Dirty tracking already enabled for %d regions\n",
 211                         num_regions);
 212                 return -EEXIST;
 213         } else if (!max_regions) {
 214                 dev_err(&pdev->dev,
 215                         "Device doesn't support dirty tracking, max_regions %d\n",
 216                         max_regions);
 217                 return -EOPNOTSUPP;
 218         }
 219
 220         /*
 221          * Only support 1 region for now. If there are any large gaps in the
 222          * VM's address regions, then this would be a waste of memory as we are
 223          * generating 2 bitmaps (ack/seq) from the min address to the max
 224          * address of the VM's address regions. In the future, if we support
 225          * more than one region in the device/driver we can split the bitmaps
 226          * on the largest address region gaps. We can do this split up to the
 227          * max_regions times returned from the dirty_status command.
 228          */
 229         max_regions = 1;
 230         if (num_ranges > max_regions) {
 231                 vfio_combine_iova_ranges(ranges, nnodes, max_regions);
 232                 num_ranges = max_regions;
 233         }
 234
 235         node = interval_tree_iter_first(ranges, 0, ULONG_MAX);
 236         if (!node)
 237                 return -EINVAL;
 238
 239         region_size = node->last - node->start + 1;
 240         region_start = node->start;
 241         region_page_size = *page_size;
 242
 243         len = sizeof(*region_info);
 244         region_info = kzalloc(len, GFP_KERNEL);
 245         if (!region_info)
 246                 return -ENOMEM;
 247
 248         page_count = DIV_ROUND_UP(region_size, region_page_size);
 249
 250         region_info->dma_base = cpu_to_le64(region_start);
 251         region_info->page_count = cpu_to_le32(page_count);
 252         region_info->page_size_log2 = ilog2(region_page_size);
 253
 254         regions_dma = dma_map_single(pdsc_dev, (void *)region_info, len,
 255                                      DMA_BIDIRECTIONAL);
 256         if (dma_mapping_error(pdsc_dev, regions_dma)) {
 257                 err = -ENOMEM;
 258                 goto out_free_region_info;
 259         }
 260
 261         err = pds_vfio_dirty_enable_cmd(pds_vfio, regions_dma, max_regions);
 262         dma_unmap_single(pdsc_dev, regions_dma, len, DMA_BIDIRECTIONAL);
 263         if (err)
 264                 goto out_free_region_info;
 265
 266         /*
 267          * page_count might be adjusted by the device,
 268          * update it before freeing region_info DMA
 269          */
 270         page_count = le32_to_cpu(region_info->page_count);
 271
 272         dev_dbg(&pdev->dev,
 273                 "region_info: regions_dma 0x%llx dma_base 0x%llx page_count %u page_size_log2 %u\n",
 274                 regions_dma, region_start, page_count,
 275                 (u8)ilog2(region_page_size));
 276
 277         err = pds_vfio_dirty_alloc_bitmaps(dirty, page_count / BITS_PER_BYTE);
 278         if (err) {
 279                 dev_err(&pdev->dev, "Failed to alloc dirty bitmaps: %pe\n",
 280                         ERR_PTR(err));
 281                 goto out_free_region_info;
 282         }
 283
 284         err = pds_vfio_dirty_alloc_sgl(pds_vfio, page_count);
 285         if (err) {
 286                 dev_err(&pdev->dev, "Failed to alloc dirty sg lists: %pe\n",
 287                         ERR_PTR(err));
 288                 goto out_free_bitmaps;
 289         }
 290
 291         dirty->region_start = region_start;
 292         dirty->region_size = region_size;
 293         dirty->region_page_size = region_page_size;
 294         pds_vfio_dirty_set_enabled(pds_vfio);
 295
 296         pds_vfio_print_guest_region_info(pds_vfio, max_regions);
 297
 298         kfree(region_info);
 299
 300         return 0;
 301
 302 out_free_bitmaps:
 303         pds_vfio_dirty_free_bitmaps(dirty);
 304 out_free_region_info:
 305         kfree(region_info);
 306         return err;
 307 }
 308
 309 void pds_vfio_dirty_disable(struct pds_vfio_pci_device *pds_vfio, bool send_cmd)
 310 {
 311         if (pds_vfio_dirty_is_enabled(pds_vfio)) {
 312                 pds_vfio_dirty_set_disabled(pds_vfio);
 313                 if (send_cmd)
 314                         pds_vfio_dirty_disable_cmd(pds_vfio);
 315                 pds_vfio_dirty_free_sgl(pds_vfio);
 316                 pds_vfio_dirty_free_bitmaps(&pds_vfio->dirty);
 317         }
 318
 319         if (send_cmd)
 320                 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE);
 321 }
 322
 323 static int pds_vfio_dirty_seq_ack(struct pds_vfio_pci_device *pds_vfio,
 324                                   struct pds_vfio_bmp_info *bmp_info,
 325                                   u32 offset, u32 bmp_bytes, bool read_seq)
 326 {
 327         const char *bmp_type_str = read_seq ? "read_seq" : "write_ack";
 328         u8 dma_dir = read_seq ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
 329         struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
 330         struct device *pdsc_dev = &pci_physfn(pdev)->dev;
 331         unsigned long long npages;
 332         struct sg_table sg_table;
 333         struct scatterlist *sg;
 334         struct page **pages;
 335         u32 page_offset;
 336         const void *bmp;
 337         size_t size;
 338         u16 num_sge;
 339         int err;
 340         int i;
 341
 342         bmp = (void *)((u64)bmp_info->bmp + offset);
 343         page_offset = offset_in_page(bmp);
 344         bmp -= page_offset;
 345
 346         /*
 347          * Start and end of bitmap section to seq/ack might not be page
 348          * aligned, so use the page_offset to account for that so there
 349          * will be enough pages to represent the bmp_bytes
 350          */
 351         npages = DIV_ROUND_UP_ULL(bmp_bytes + page_offset, PAGE_SIZE);
 352         pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
 353         if (!pages)
 354                 return -ENOMEM;
 355
 356         for (unsigned long long i = 0; i < npages; i++) {
 357                 struct page *page = vmalloc_to_page(bmp);
 358
 359                 if (!page) {
 360                         err = -EFAULT;
 361                         goto out_free_pages;
 362                 }
 363
 364                 pages[i] = page;
 365                 bmp += PAGE_SIZE;
 366         }
 367
 368         err = sg_alloc_table_from_pages(&sg_table, pages, npages, page_offset,
 369                                         bmp_bytes, GFP_KERNEL);
 370         if (err)
 371                 goto out_free_pages;
 372
 373         err = dma_map_sgtable(pdsc_dev, &sg_table, dma_dir, 0);
 374         if (err)
 375                 goto out_free_sg_table;
 376
 377         for_each_sgtable_dma_sg(&sg_table, sg, i) {
 378                 struct pds_lm_sg_elem *sg_elem = &bmp_info->sgl[i];
 379
 380                 sg_elem->addr = cpu_to_le64(sg_dma_address(sg));
 381                 sg_elem->len = cpu_to_le32(sg_dma_len(sg));
 382         }
 383
 384         num_sge = sg_table.nents;
 385         size = num_sge * sizeof(struct pds_lm_sg_elem);
 386         dma_sync_single_for_device(pdsc_dev, bmp_info->sgl_addr, size, dma_dir);
 387         err = pds_vfio_dirty_seq_ack_cmd(pds_vfio, bmp_info->sgl_addr, num_sge,
 388                                          offset, bmp_bytes, read_seq);
 389         if (err)
 390                 dev_err(&pdev->dev,
 391                         "Dirty bitmap %s failed offset %u bmp_bytes %u num_sge %u DMA 0x%llx: %pe\n",
 392                         bmp_type_str, offset, bmp_bytes,
 393                         num_sge, bmp_info->sgl_addr, ERR_PTR(err));
 394         dma_sync_single_for_cpu(pdsc_dev, bmp_info->sgl_addr, size, dma_dir);
 395
 396         dma_unmap_sgtable(pdsc_dev, &sg_table, dma_dir, 0);
 397 out_free_sg_table:
 398         sg_free_table(&sg_table);
 399 out_free_pages:
 400         kfree(pages);
 401
 402         return err;
 403 }
 404
 405 static int pds_vfio_dirty_write_ack(struct pds_vfio_pci_device *pds_vfio,
 406                                     u32 offset, u32 len)
 407 {
 408         return pds_vfio_dirty_seq_ack(pds_vfio, &pds_vfio->dirty.host_ack,
 409                                       offset, len, WRITE_ACK);
 410 }
 411
 412 static int pds_vfio_dirty_read_seq(struct pds_vfio_pci_device *pds_vfio,
 413                                    u32 offset, u32 len)
 414 {
 415         return pds_vfio_dirty_seq_ack(pds_vfio, &pds_vfio->dirty.host_seq,
 416                                       offset, len, READ_SEQ);
 417 }
 418
 419 static int pds_vfio_dirty_process_bitmaps(struct pds_vfio_pci_device *pds_vfio,
 420                                           struct iova_bitmap *dirty_bitmap,
 421                                           u32 bmp_offset, u32 len_bytes)
 422 {
 423         u64 page_size = pds_vfio->dirty.region_page_size;
 424         u64 region_start = pds_vfio->dirty.region_start;
 425         u32 bmp_offset_bit;
 426         __le64 *seq, *ack;
 427         int dword_count;
 428
 429         dword_count = len_bytes / sizeof(u64);
 430         seq = (__le64 *)((u64)pds_vfio->dirty.host_seq.bmp + bmp_offset);
 431         ack = (__le64 *)((u64)pds_vfio->dirty.host_ack.bmp + bmp_offset);
 432         bmp_offset_bit = bmp_offset * 8;
 433
 434         for (int i = 0; i < dword_count; i++) {
 435                 u64 xor = le64_to_cpu(seq[i]) ^ le64_to_cpu(ack[i]);
 436
 437                 /* prepare for next write_ack call */
 438                 ack[i] = seq[i];
 439
 440                 for (u8 bit_i = 0; bit_i < BITS_PER_TYPE(u64); ++bit_i) {
 441                         if (xor & BIT(bit_i)) {
 442                                 u64 abs_bit_i = bmp_offset_bit +
 443                                                 i * BITS_PER_TYPE(u64) + bit_i;
 444                                 u64 addr = abs_bit_i * page_size + region_start;
 445
 446                                 iova_bitmap_set(dirty_bitmap, addr, page_size);
 447                         }
 448                 }
 449         }
 450
 451         return 0;
 452 }
 453
 454 static int pds_vfio_dirty_sync(struct pds_vfio_pci_device *pds_vfio,
 455                                struct iova_bitmap *dirty_bitmap,
 456                                unsigned long iova, unsigned long length)
 457 {
 458         struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
 459         struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
 460         u64 bmp_offset, bmp_bytes;
 461         u64 bitmap_size, pages;
 462         int err;
 463
 464         dev_dbg(dev, "vf%u: Get dirty page bitmap\n", pds_vfio->vf_id);
 465
 466         if (!pds_vfio_dirty_is_enabled(pds_vfio)) {
 467                 dev_err(dev, "vf%u: Sync failed, dirty tracking is disabled\n",
 468                         pds_vfio->vf_id);
 469                 return -EINVAL;
 470         }
 471
 472         pages = DIV_ROUND_UP(length, pds_vfio->dirty.region_page_size);
 473         bitmap_size =
 474                 round_up(pages, sizeof(u64) * BITS_PER_BYTE) / BITS_PER_BYTE;
 475
 476         dev_dbg(dev,
 477                 "vf%u: iova 0x%lx length %lu page_size %llu pages %llu bitmap_size %llu\n",
 478                 pds_vfio->vf_id, iova, length, pds_vfio->dirty.region_page_size,
 479                 pages, bitmap_size);
 480
 481         if (!length || ((iova - dirty->region_start + length) > dirty->region_size)) {
 482                 dev_err(dev, "Invalid iova 0x%lx and/or length 0x%lx to sync\n",
 483                         iova, length);
 484                 return -EINVAL;
 485         }
 486
 487         /* bitmap is modified in 64 bit chunks */
 488         bmp_bytes = ALIGN(DIV_ROUND_UP(length / dirty->region_page_size,
 489                                        sizeof(u64)),
 490                           sizeof(u64));
 491         if (bmp_bytes != bitmap_size) {
 492                 dev_err(dev,
 493                         "Calculated bitmap bytes %llu not equal to bitmap size %llu\n",
 494                         bmp_bytes, bitmap_size);
 495                 return -EINVAL;
 496         }
 497
 498         bmp_offset = DIV_ROUND_UP((iova - dirty->region_start) /
 499                                   dirty->region_page_size, sizeof(u64));
 500
 501         dev_dbg(dev,
 502                 "Syncing dirty bitmap, iova 0x%lx length 0x%lx, bmp_offset %llu bmp_bytes %llu\n",
 503                 iova, length, bmp_offset, bmp_bytes);
 504
 505         err = pds_vfio_dirty_read_seq(pds_vfio, bmp_offset, bmp_bytes);
 506         if (err)
 507                 return err;
 508
 509         err = pds_vfio_dirty_process_bitmaps(pds_vfio, dirty_bitmap, bmp_offset,
 510                                              bmp_bytes);
 511         if (err)
 512                 return err;
 513
 514         err = pds_vfio_dirty_write_ack(pds_vfio, bmp_offset, bmp_bytes);
 515         if (err)
 516                 return err;
 517
 518         return 0;
 519 }
 520
 521 int pds_vfio_dma_logging_report(struct vfio_device *vdev, unsigned long iova,
 522                                 unsigned long length, struct iova_bitmap *dirty)
 523 {
 524         struct pds_vfio_pci_device *pds_vfio =
 525                 container_of(vdev, struct pds_vfio_pci_device,
 526                              vfio_coredev.vdev);
 527         int err;
 528
 529         mutex_lock(&pds_vfio->state_mutex);
 530         err = pds_vfio_dirty_sync(pds_vfio, dirty, iova, length);
 531         pds_vfio_state_mutex_unlock(pds_vfio);
 532
 533         return err;
 534 }
 535
 536 int pds_vfio_dma_logging_start(struct vfio_device *vdev,
 537                                struct rb_root_cached *ranges, u32 nnodes,
 538                                u64 *page_size)
 539 {
 540         struct pds_vfio_pci_device *pds_vfio =
 541                 container_of(vdev, struct pds_vfio_pci_device,
 542                              vfio_coredev.vdev);
 543         int err;
 544
 545         mutex_lock(&pds_vfio->state_mutex);
 546         pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_IN_PROGRESS);
 547         err = pds_vfio_dirty_enable(pds_vfio, ranges, nnodes, page_size);
 548         pds_vfio_state_mutex_unlock(pds_vfio);
 549
 550         return err;
 551 }
 552
 553 int pds_vfio_dma_logging_stop(struct vfio_device *vdev)
 554 {
 555         struct pds_vfio_pci_device *pds_vfio =
 556                 container_of(vdev, struct pds_vfio_pci_device,
 557                              vfio_coredev.vdev);
 558
 559         mutex_lock(&pds_vfio->state_mutex);
 560         pds_vfio_dirty_disable(pds_vfio, true);
 561         pds_vfio_state_mutex_unlock(pds_vfio);
 562
 563         return 0;
 564 }