pds/lm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */
   3
   4 #include <linux/anon_inodes.h>
   5 #include <linux/file.h>
   6 #include <linux/fs.h>
   7 #include <linux/highmem.h>
   8 #include <linux/vfio.h>
   9 #include <linux/vfio_pci_core.h>
  10
  11 #include "vfio_dev.h"
  12 #include "cmds.h"
  13
  14 static struct pds_vfio_lm_file *
  15 pds_vfio_get_lm_file(const struct file_operations *fops, int flags, u64 size)
  16 {
  17         struct pds_vfio_lm_file *lm_file = NULL;
  18         unsigned long long npages;
  19         struct page **pages;
  20         void *page_mem;
  21         const void *p;
  22
  23         if (!size)
  24                 return NULL;
  25
  26         /* Alloc file structure */
  27         lm_file = kzalloc(sizeof(*lm_file), GFP_KERNEL);
  28         if (!lm_file)
  29                 return NULL;
  30
  31         /* Create file */
  32         lm_file->filep =
  33                 anon_inode_getfile("pds_vfio_lm", fops, lm_file, flags);
  34         if (IS_ERR(lm_file->filep))
  35                 goto out_free_file;
  36
  37         stream_open(lm_file->filep->f_inode, lm_file->filep);
  38         mutex_init(&lm_file->lock);
  39
  40         /* prevent file from being released before we are done with it */
  41         get_file(lm_file->filep);
  42
  43         /* Allocate memory for file pages */
  44         npages = DIV_ROUND_UP_ULL(size, PAGE_SIZE);
  45         pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
  46         if (!pages)
  47                 goto out_put_file;
  48
  49         page_mem = kvzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL);
  50         if (!page_mem)
  51                 goto out_free_pages_array;
  52
  53         p = page_mem - offset_in_page(page_mem);
  54         for (unsigned long long i = 0; i < npages; i++) {
  55                 if (is_vmalloc_addr(p))
  56                         pages[i] = vmalloc_to_page(p);
  57                 else
  58                         pages[i] = kmap_to_page((void *)p);
  59                 if (!pages[i])
  60                         goto out_free_page_mem;
  61
  62                 p += PAGE_SIZE;
  63         }
  64
  65         /* Create scatterlist of file pages to use for DMA mapping later */
  66         if (sg_alloc_table_from_pages(&lm_file->sg_table, pages, npages, 0,
  67                                       size, GFP_KERNEL))
  68                 goto out_free_page_mem;
  69
  70         lm_file->size = size;
  71         lm_file->pages = pages;
  72         lm_file->npages = npages;
  73         lm_file->page_mem = page_mem;
  74         lm_file->alloc_size = npages * PAGE_SIZE;
  75
  76         return lm_file;
  77
  78 out_free_page_mem:
  79         kvfree(page_mem);
  80 out_free_pages_array:
  81         kfree(pages);
  82 out_put_file:
  83         fput(lm_file->filep);
  84         mutex_destroy(&lm_file->lock);
  85 out_free_file:
  86         kfree(lm_file);
  87
  88         return NULL;
  89 }
  90
  91 static void pds_vfio_put_lm_file(struct pds_vfio_lm_file *lm_file)
  92 {
  93         mutex_lock(&lm_file->lock);
  94
  95         lm_file->size = 0;
  96         lm_file->alloc_size = 0;
  97
  98         /* Free scatter list of file pages */
  99         sg_free_table(&lm_file->sg_table);
 100
 101         kvfree(lm_file->page_mem);
 102         lm_file->page_mem = NULL;
 103         kfree(lm_file->pages);
 104         lm_file->pages = NULL;
 105
 106         mutex_unlock(&lm_file->lock);
 107
 108         /* allow file to be released since we are done with it */
 109         fput(lm_file->filep);
 110 }
 111
 112 void pds_vfio_put_save_file(struct pds_vfio_pci_device *pds_vfio)
 113 {
 114         if (!pds_vfio->save_file)
 115                 return;
 116
 117         pds_vfio_put_lm_file(pds_vfio->save_file);
 118         pds_vfio->save_file = NULL;
 119 }
 120
 121 void pds_vfio_put_restore_file(struct pds_vfio_pci_device *pds_vfio)
 122 {
 123         if (!pds_vfio->restore_file)
 124                 return;
 125
 126         pds_vfio_put_lm_file(pds_vfio->restore_file);
 127         pds_vfio->restore_file = NULL;
 128 }
 129
 130 static struct page *pds_vfio_get_file_page(struct pds_vfio_lm_file *lm_file,
 131                                            unsigned long offset)
 132 {
 133         unsigned long cur_offset = 0;
 134         struct scatterlist *sg;
 135         unsigned int i;
 136
 137         /* All accesses are sequential */
 138         if (offset < lm_file->last_offset || !lm_file->last_offset_sg) {
 139                 lm_file->last_offset = 0;
 140                 lm_file->last_offset_sg = lm_file->sg_table.sgl;
 141                 lm_file->sg_last_entry = 0;
 142         }
 143
 144         cur_offset = lm_file->last_offset;
 145
 146         for_each_sg(lm_file->last_offset_sg, sg,
 147                     lm_file->sg_table.orig_nents - lm_file->sg_last_entry, i) {
 148                 if (offset < sg->length + cur_offset) {
 149                         lm_file->last_offset_sg = sg;
 150                         lm_file->sg_last_entry += i;
 151                         lm_file->last_offset = cur_offset;
 152                         return nth_page(sg_page(sg),
 153                                         (offset - cur_offset) / PAGE_SIZE);
 154                 }
 155                 cur_offset += sg->length;
 156         }
 157
 158         return NULL;
 159 }
 160
 161 static int pds_vfio_release_file(struct inode *inode, struct file *filp)
 162 {
 163         struct pds_vfio_lm_file *lm_file = filp->private_data;
 164
 165         mutex_lock(&lm_file->lock);
 166         lm_file->filep->f_pos = 0;
 167         lm_file->size = 0;
 168         mutex_unlock(&lm_file->lock);
 169         mutex_destroy(&lm_file->lock);
 170         kfree(lm_file);
 171
 172         return 0;
 173 }
 174
 175 static ssize_t pds_vfio_save_read(struct file *filp, char __user *buf,
 176                                   size_t len, loff_t *pos)
 177 {
 178         struct pds_vfio_lm_file *lm_file = filp->private_data;
 179         ssize_t done = 0;
 180
 181         if (pos)
 182                 return -ESPIPE;
 183         pos = &filp->f_pos;
 184
 185         mutex_lock(&lm_file->lock);
 186         if (*pos > lm_file->size) {
 187                 done = -EINVAL;
 188                 goto out_unlock;
 189         }
 190
 191         len = min_t(size_t, lm_file->size - *pos, len);
 192         while (len) {
 193                 size_t page_offset;
 194                 struct page *page;
 195                 size_t page_len;
 196                 u8 *from_buff;
 197                 int err;
 198
 199                 page_offset = (*pos) % PAGE_SIZE;
 200                 page = pds_vfio_get_file_page(lm_file, *pos - page_offset);
 201                 if (!page) {
 202                         if (done == 0)
 203                                 done = -EINVAL;
 204                         goto out_unlock;
 205                 }
 206
 207                 page_len = min_t(size_t, len, PAGE_SIZE - page_offset);
 208                 from_buff = kmap_local_page(page);
 209                 err = copy_to_user(buf, from_buff + page_offset, page_len);
 210                 kunmap_local(from_buff);
 211                 if (err) {
 212                         done = -EFAULT;
 213                         goto out_unlock;
 214                 }
 215                 *pos += page_len;
 216                 len -= page_len;
 217                 done += page_len;
 218                 buf += page_len;
 219         }
 220
 221 out_unlock:
 222         mutex_unlock(&lm_file->lock);
 223         return done;
 224 }
 225
 226 static const struct file_operations pds_vfio_save_fops = {
 227         .owner = THIS_MODULE,
 228         .read = pds_vfio_save_read,
 229         .release = pds_vfio_release_file,
 230         .llseek = no_llseek,
 231 };
 232
 233 static int pds_vfio_get_save_file(struct pds_vfio_pci_device *pds_vfio)
 234 {
 235         struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
 236         struct pds_vfio_lm_file *lm_file;
 237         u64 size;
 238         int err;
 239
 240         /* Get live migration state size in this state */
 241         err = pds_vfio_get_lm_state_size_cmd(pds_vfio, &size);
 242         if (err) {
 243                 dev_err(dev, "failed to get save status: %pe\n", ERR_PTR(err));
 244                 return err;
 245         }
 246
 247         dev_dbg(dev, "save status, size = %lld\n", size);
 248
 249         if (!size) {
 250                 dev_err(dev, "invalid state size\n");
 251                 return -EIO;
 252         }
 253
 254         lm_file = pds_vfio_get_lm_file(&pds_vfio_save_fops, O_RDONLY, size);
 255         if (!lm_file) {
 256                 dev_err(dev, "failed to create save file\n");
 257                 return -ENOENT;
 258         }
 259
 260         dev_dbg(dev, "size = %lld, alloc_size = %lld, npages = %lld\n",
 261                 lm_file->size, lm_file->alloc_size, lm_file->npages);
 262
 263         pds_vfio->save_file = lm_file;
 264
 265         return 0;
 266 }
 267
 268 static ssize_t pds_vfio_restore_write(struct file *filp, const char __user *buf,
 269                                       size_t len, loff_t *pos)
 270 {
 271         struct pds_vfio_lm_file *lm_file = filp->private_data;
 272         loff_t requested_length;
 273         ssize_t done = 0;
 274
 275         if (pos)
 276                 return -ESPIPE;
 277
 278         pos = &filp->f_pos;
 279
 280         if (*pos < 0 ||
 281             check_add_overflow((loff_t)len, *pos, &requested_length))
 282                 return -EINVAL;
 283
 284         mutex_lock(&lm_file->lock);
 285
 286         while (len) {
 287                 size_t page_offset;
 288                 struct page *page;
 289                 size_t page_len;
 290                 u8 *to_buff;
 291                 int err;
 292
 293                 page_offset = (*pos) % PAGE_SIZE;
 294                 page = pds_vfio_get_file_page(lm_file, *pos - page_offset);
 295                 if (!page) {
 296                         if (done == 0)
 297                                 done = -EINVAL;
 298                         goto out_unlock;
 299                 }
 300
 301                 page_len = min_t(size_t, len, PAGE_SIZE - page_offset);
 302                 to_buff = kmap_local_page(page);
 303                 err = copy_from_user(to_buff + page_offset, buf, page_len);
 304                 kunmap_local(to_buff);
 305                 if (err) {
 306                         done = -EFAULT;
 307                         goto out_unlock;
 308                 }
 309                 *pos += page_len;
 310                 len -= page_len;
 311                 done += page_len;
 312                 buf += page_len;
 313                 lm_file->size += page_len;
 314         }
 315 out_unlock:
 316         mutex_unlock(&lm_file->lock);
 317         return done;
 318 }
 319
 320 static const struct file_operations pds_vfio_restore_fops = {
 321         .owner = THIS_MODULE,
 322         .write = pds_vfio_restore_write,
 323         .release = pds_vfio_release_file,
 324         .llseek = no_llseek,
 325 };
 326
 327 static int pds_vfio_get_restore_file(struct pds_vfio_pci_device *pds_vfio)
 328 {
 329         struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
 330         struct pds_vfio_lm_file *lm_file;
 331         u64 size;
 332
 333         size = sizeof(union pds_lm_dev_state);
 334         dev_dbg(dev, "restore status, size = %lld\n", size);
 335
 336         if (!size) {
 337                 dev_err(dev, "invalid state size");
 338                 return -EIO;
 339         }
 340
 341         lm_file = pds_vfio_get_lm_file(&pds_vfio_restore_fops, O_WRONLY, size);
 342         if (!lm_file) {
 343                 dev_err(dev, "failed to create restore file");
 344                 return -ENOENT;
 345         }
 346         pds_vfio->restore_file = lm_file;
 347
 348         return 0;
 349 }
 350
 351 struct file *
 352 pds_vfio_step_device_state_locked(struct pds_vfio_pci_device *pds_vfio,
 353                                   enum vfio_device_mig_state next)
 354 {
 355         enum vfio_device_mig_state cur = pds_vfio->state;
 356         int err;
 357
 358         if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_STOP_COPY) {
 359                 err = pds_vfio_get_save_file(pds_vfio);
 360                 if (err)
 361                         return ERR_PTR(err);
 362
 363                 err = pds_vfio_get_lm_state_cmd(pds_vfio);
 364                 if (err) {
 365                         pds_vfio_put_save_file(pds_vfio);
 366                         return ERR_PTR(err);
 367                 }
 368
 369                 return pds_vfio->save_file->filep;
 370         }
 371
 372         if (cur == VFIO_DEVICE_STATE_STOP_COPY && next == VFIO_DEVICE_STATE_STOP) {
 373                 pds_vfio_put_save_file(pds_vfio);
 374                 pds_vfio_dirty_disable(pds_vfio, true);
 375                 return NULL;
 376         }
 377
 378         if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RESUMING) {
 379                 err = pds_vfio_get_restore_file(pds_vfio);
 380                 if (err)
 381                         return ERR_PTR(err);
 382
 383                 return pds_vfio->restore_file->filep;
 384         }
 385
 386         if (cur == VFIO_DEVICE_STATE_RESUMING && next == VFIO_DEVICE_STATE_STOP) {
 387                 err = pds_vfio_set_lm_state_cmd(pds_vfio);
 388                 if (err)
 389                         return ERR_PTR(err);
 390
 391                 pds_vfio_put_restore_file(pds_vfio);
 392                 return NULL;
 393         }
 394
 395         if (cur == VFIO_DEVICE_STATE_RUNNING && next == VFIO_DEVICE_STATE_RUNNING_P2P) {
 396                 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio,
 397                                                     PDS_LM_STA_IN_PROGRESS);
 398                 err = pds_vfio_suspend_device_cmd(pds_vfio,
 399                                                   PDS_LM_SUSPEND_RESUME_TYPE_P2P);
 400                 if (err)
 401                         return ERR_PTR(err);
 402
 403                 return NULL;
 404         }
 405
 406         if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_RUNNING) {
 407                 err = pds_vfio_resume_device_cmd(pds_vfio,
 408                                                  PDS_LM_SUSPEND_RESUME_TYPE_FULL);
 409                 if (err)
 410                         return ERR_PTR(err);
 411
 412                 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE);
 413                 return NULL;
 414         }
 415
 416         if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RUNNING_P2P) {
 417                 err = pds_vfio_resume_device_cmd(pds_vfio,
 418                                                  PDS_LM_SUSPEND_RESUME_TYPE_P2P);
 419                 if (err)
 420                         return ERR_PTR(err);
 421
 422                 return NULL;
 423         }
 424
 425         if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_STOP) {
 426                 err = pds_vfio_suspend_device_cmd(pds_vfio,
 427                                                   PDS_LM_SUSPEND_RESUME_TYPE_FULL);
 428                 if (err)
 429                         return ERR_PTR(err);
 430                 return NULL;
 431         }
 432
 433         return ERR_PTR(-EINVAL);
 434 }