drivers/vfio/pci/vfio_pci_rdwr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO PCI I/O Port & MMIO access
   4  *
   5  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   6  *     Author: Alex Williamson <alex.williamson@redhat.com>
   7  *
   8  * Derived from original vfio:
   9  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  10  * Author: Tom Lyon, pugs@cisco.com
  11  */
  12
  13 #include <linux/fs.h>
  14 #include <linux/pci.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/io.h>
  17 #include <linux/vfio.h>
  18 #include <linux/vgaarb.h>
  19
  20 #include "vfio_pci_priv.h"
  21
  22 #ifdef __LITTLE_ENDIAN
  23 #define vfio_ioread64   ioread64
  24 #define vfio_iowrite64  iowrite64
  25 #define vfio_ioread32   ioread32
  26 #define vfio_iowrite32  iowrite32
  27 #define vfio_ioread16   ioread16
  28 #define vfio_iowrite16  iowrite16
  29 #else
  30 #define vfio_ioread64   ioread64be
  31 #define vfio_iowrite64  iowrite64be
  32 #define vfio_ioread32   ioread32be
  33 #define vfio_iowrite32  iowrite32be
  34 #define vfio_ioread16   ioread16be
  35 #define vfio_iowrite16  iowrite16be
  36 #endif
  37 #define vfio_ioread8    ioread8
  38 #define vfio_iowrite8   iowrite8
  39
  40 #define VFIO_IOWRITE(size) \
  41 static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev,            \
  42                         bool test_mem, u##size val, void __iomem *io)   \
  43 {                                                                       \
  44         if (test_mem) {                                                 \
  45                 down_read(&vdev->memory_lock);                          \
  46                 if (!__vfio_pci_memory_enabled(vdev)) {                 \
  47                         up_read(&vdev->memory_lock);                    \
  48                         return -EIO;                                    \
  49                 }                                                       \
  50         }                                                               \
  51                                                                         \
  52         vfio_iowrite##size(val, io);                                    \
  53                                                                         \
  54         if (test_mem)                                                   \
  55                 up_read(&vdev->memory_lock);                            \
  56                                                                         \
  57         return 0;                                                       \
  58 }
  59
  60 VFIO_IOWRITE(8)
  61 VFIO_IOWRITE(16)
  62 VFIO_IOWRITE(32)
  63 #ifdef iowrite64
  64 VFIO_IOWRITE(64)
  65 #endif
  66
  67 #define VFIO_IOREAD(size) \
  68 static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev,             \
  69                         bool test_mem, u##size *val, void __iomem *io)  \
  70 {                                                                       \
  71         if (test_mem) {                                                 \
  72                 down_read(&vdev->memory_lock);                          \
  73                 if (!__vfio_pci_memory_enabled(vdev)) {                 \
  74                         up_read(&vdev->memory_lock);                    \
  75                         return -EIO;                                    \
  76                 }                                                       \
  77         }                                                               \
  78                                                                         \
  79         *val = vfio_ioread##size(io);                                   \
  80                                                                         \
  81         if (test_mem)                                                   \
  82                 up_read(&vdev->memory_lock);                            \
  83                                                                         \
  84         return 0;                                                       \
  85 }
  86
  87 VFIO_IOREAD(8)
  88 VFIO_IOREAD(16)
  89 VFIO_IOREAD(32)
  90
  91 /*
  92  * Read or write from an __iomem region (MMIO or I/O port) with an excluded
  93  * range which is inaccessible.  The excluded range drops writes and fills
  94  * reads with -1.  This is intended for handling MSI-X vector tables and
  95  * leftover space for ROM BARs.
  96  */
  97 static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
  98                         void __iomem *io, char __user *buf,
  99                         loff_t off, size_t count, size_t x_start,
 100                         size_t x_end, bool iswrite)
 101 {
 102         ssize_t done = 0;
 103         int ret;
 104
 105         while (count) {
 106                 size_t fillable, filled;
 107
 108                 if (off < x_start)
 109                         fillable = min(count, (size_t)(x_start - off));
 110                 else if (off >= x_end)
 111                         fillable = count;
 112                 else
 113                         fillable = 0;
 114
 115                 if (fillable >= 4 && !(off % 4)) {
 116                         u32 val;
 117
 118                         if (iswrite) {
 119                                 if (copy_from_user(&val, buf, 4))
 120                                         return -EFAULT;
 121
 122                                 ret = vfio_pci_iowrite32(vdev, test_mem,
 123                                                          val, io + off);
 124                                 if (ret)
 125                                         return ret;
 126                         } else {
 127                                 ret = vfio_pci_ioread32(vdev, test_mem,
 128                                                         &val, io + off);
 129                                 if (ret)
 130                                         return ret;
 131
 132                                 if (copy_to_user(buf, &val, 4))
 133                                         return -EFAULT;
 134                         }
 135
 136                         filled = 4;
 137                 } else if (fillable >= 2 && !(off % 2)) {
 138                         u16 val;
 139
 140                         if (iswrite) {
 141                                 if (copy_from_user(&val, buf, 2))
 142                                         return -EFAULT;
 143
 144                                 ret = vfio_pci_iowrite16(vdev, test_mem,
 145                                                          val, io + off);
 146                                 if (ret)
 147                                         return ret;
 148                         } else {
 149                                 ret = vfio_pci_ioread16(vdev, test_mem,
 150                                                         &val, io + off);
 151                                 if (ret)
 152                                         return ret;
 153
 154                                 if (copy_to_user(buf, &val, 2))
 155                                         return -EFAULT;
 156                         }
 157
 158                         filled = 2;
 159                 } else if (fillable) {
 160                         u8 val;
 161
 162                         if (iswrite) {
 163                                 if (copy_from_user(&val, buf, 1))
 164                                         return -EFAULT;
 165
 166                                 ret = vfio_pci_iowrite8(vdev, test_mem,
 167                                                         val, io + off);
 168                                 if (ret)
 169                                         return ret;
 170                         } else {
 171                                 ret = vfio_pci_ioread8(vdev, test_mem,
 172                                                        &val, io + off);
 173                                 if (ret)
 174                                         return ret;
 175
 176                                 if (copy_to_user(buf, &val, 1))
 177                                         return -EFAULT;
 178                         }
 179
 180                         filled = 1;
 181                 } else {
 182                         /* Fill reads with -1, drop writes */
 183                         filled = min(count, (size_t)(x_end - off));
 184                         if (!iswrite) {
 185                                 u8 val = 0xFF;
 186                                 size_t i;
 187
 188                                 for (i = 0; i < filled; i++)
 189                                         if (copy_to_user(buf + i, &val, 1))
 190                                                 return -EFAULT;
 191                         }
 192                 }
 193
 194                 count -= filled;
 195                 done += filled;
 196                 off += filled;
 197                 buf += filled;
 198         }
 199
 200         return done;
 201 }
 202
 203 static int vfio_pci_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
 204 {
 205         struct pci_dev *pdev = vdev->pdev;
 206         int ret;
 207         void __iomem *io;
 208
 209         if (vdev->barmap[bar])
 210                 return 0;
 211
 212         ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
 213         if (ret)
 214                 return ret;
 215
 216         io = pci_iomap(pdev, bar, 0);
 217         if (!io) {
 218                 pci_release_selected_regions(pdev, 1 << bar);
 219                 return -ENOMEM;
 220         }
 221
 222         vdev->barmap[bar] = io;
 223
 224         return 0;
 225 }
 226
 227 ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
 228                         size_t count, loff_t *ppos, bool iswrite)
 229 {
 230         struct pci_dev *pdev = vdev->pdev;
 231         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 232         int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 233         size_t x_start = 0, x_end = 0;
 234         resource_size_t end;
 235         void __iomem *io;
 236         struct resource *res = &vdev->pdev->resource[bar];
 237         ssize_t done;
 238
 239         if (pci_resource_start(pdev, bar))
 240                 end = pci_resource_len(pdev, bar);
 241         else if (bar == PCI_ROM_RESOURCE &&
 242                  pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW)
 243                 end = 0x20000;
 244         else
 245                 return -EINVAL;
 246
 247         if (pos >= end)
 248                 return -EINVAL;
 249
 250         count = min(count, (size_t)(end - pos));
 251
 252         if (bar == PCI_ROM_RESOURCE) {
 253                 /*
 254                  * The ROM can fill less space than the BAR, so we start the
 255                  * excluded range at the end of the actual ROM.  This makes
 256                  * filling large ROM BARs much faster.
 257                  */
 258                 io = pci_map_rom(pdev, &x_start);
 259                 if (!io) {
 260                         done = -ENOMEM;
 261                         goto out;
 262                 }
 263                 x_end = end;
 264         } else {
 265                 int ret = vfio_pci_setup_barmap(vdev, bar);
 266                 if (ret) {
 267                         done = ret;
 268                         goto out;
 269                 }
 270
 271                 io = vdev->barmap[bar];
 272         }
 273
 274         if (bar == vdev->msix_bar) {
 275                 x_start = vdev->msix_offset;
 276                 x_end = vdev->msix_offset + vdev->msix_size;
 277         }
 278
 279         done = do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos,
 280                         count, x_start, x_end, iswrite);
 281
 282         if (done >= 0)
 283                 *ppos += done;
 284
 285         if (bar == PCI_ROM_RESOURCE)
 286                 pci_unmap_rom(pdev, io);
 287 out:
 288         return done;
 289 }
 290
 291 #ifdef CONFIG_VFIO_PCI_VGA
 292 ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
 293                                size_t count, loff_t *ppos, bool iswrite)
 294 {
 295         int ret;
 296         loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK;
 297         void __iomem *iomem = NULL;
 298         unsigned int rsrc;
 299         bool is_ioport;
 300         ssize_t done;
 301
 302         if (!vdev->has_vga)
 303                 return -EINVAL;
 304
 305         if (pos > 0xbfffful)
 306                 return -EINVAL;
 307
 308         switch ((u32)pos) {
 309         case 0xa0000 ... 0xbffff:
 310                 count = min(count, (size_t)(0xc0000 - pos));
 311                 iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1);
 312                 off = pos - 0xa0000;
 313                 rsrc = VGA_RSRC_LEGACY_MEM;
 314                 is_ioport = false;
 315                 break;
 316         case 0x3b0 ... 0x3bb:
 317                 count = min(count, (size_t)(0x3bc - pos));
 318                 iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1);
 319                 off = pos - 0x3b0;
 320                 rsrc = VGA_RSRC_LEGACY_IO;
 321                 is_ioport = true;
 322                 break;
 323         case 0x3c0 ... 0x3df:
 324                 count = min(count, (size_t)(0x3e0 - pos));
 325                 iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1);
 326                 off = pos - 0x3c0;
 327                 rsrc = VGA_RSRC_LEGACY_IO;
 328                 is_ioport = true;
 329                 break;
 330         default:
 331                 return -EINVAL;
 332         }
 333
 334         if (!iomem)
 335                 return -ENOMEM;
 336
 337         ret = vga_get_interruptible(vdev->pdev, rsrc);
 338         if (ret) {
 339                 is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
 340                 return ret;
 341         }
 342
 343         /*
 344          * VGA MMIO is a legacy, non-BAR resource that hopefully allows
 345          * probing, so we don't currently worry about access in relation
 346          * to the memory enable bit in the command register.
 347          */
 348         done = do_io_rw(vdev, false, iomem, buf, off, count, 0, 0, iswrite);
 349
 350         vga_put(vdev->pdev, rsrc);
 351
 352         is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
 353
 354         if (done >= 0)
 355                 *ppos += done;
 356
 357         return done;
 358 }
 359 #endif
 360
 361 static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
 362                                         bool test_mem)
 363 {
 364         switch (ioeventfd->count) {
 365         case 1:
 366                 vfio_pci_iowrite8(ioeventfd->vdev, test_mem,
 367                                   ioeventfd->data, ioeventfd->addr);
 368                 break;
 369         case 2:
 370                 vfio_pci_iowrite16(ioeventfd->vdev, test_mem,
 371                                    ioeventfd->data, ioeventfd->addr);
 372                 break;
 373         case 4:
 374                 vfio_pci_iowrite32(ioeventfd->vdev, test_mem,
 375                                    ioeventfd->data, ioeventfd->addr);
 376                 break;
 377 #ifdef iowrite64
 378         case 8:
 379                 vfio_pci_iowrite64(ioeventfd->vdev, test_mem,
 380                                    ioeventfd->data, ioeventfd->addr);
 381                 break;
 382 #endif
 383         }
 384 }
 385
 386 static int vfio_pci_ioeventfd_handler(void *opaque, void *unused)
 387 {
 388         struct vfio_pci_ioeventfd *ioeventfd = opaque;
 389         struct vfio_pci_core_device *vdev = ioeventfd->vdev;
 390
 391         if (ioeventfd->test_mem) {
 392                 if (!down_read_trylock(&vdev->memory_lock))
 393                         return 1; /* Lock contended, use thread */
 394                 if (!__vfio_pci_memory_enabled(vdev)) {
 395                         up_read(&vdev->memory_lock);
 396                         return 0;
 397                 }
 398         }
 399
 400         vfio_pci_ioeventfd_do_write(ioeventfd, false);
 401
 402         if (ioeventfd->test_mem)
 403                 up_read(&vdev->memory_lock);
 404
 405         return 0;
 406 }
 407
 408 static void vfio_pci_ioeventfd_thread(void *opaque, void *unused)
 409 {
 410         struct vfio_pci_ioeventfd *ioeventfd = opaque;
 411
 412         vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
 413 }
 414
 415 int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
 416                        uint64_t data, int count, int fd)
 417 {
 418         struct pci_dev *pdev = vdev->pdev;
 419         loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
 420         int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
 421         struct vfio_pci_ioeventfd *ioeventfd;
 422
 423         /* Only support ioeventfds into BARs */
 424         if (bar > VFIO_PCI_BAR5_REGION_INDEX)
 425                 return -EINVAL;
 426
 427         if (pos + count > pci_resource_len(pdev, bar))
 428                 return -EINVAL;
 429
 430         /* Disallow ioeventfds working around MSI-X table writes */
 431         if (bar == vdev->msix_bar &&
 432             !(pos + count <= vdev->msix_offset ||
 433               pos >= vdev->msix_offset + vdev->msix_size))
 434                 return -EINVAL;
 435
 436 #ifndef iowrite64
 437         if (count == 8)
 438                 return -EINVAL;
 439 #endif
 440
 441         ret = vfio_pci_setup_barmap(vdev, bar);
 442         if (ret)
 443                 return ret;
 444
 445         mutex_lock(&vdev->ioeventfds_lock);
 446
 447         list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
 448                 if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
 449                     ioeventfd->data == data && ioeventfd->count == count) {
 450                         if (fd == -1) {
 451                                 vfio_virqfd_disable(&ioeventfd->virqfd);
 452                                 list_del(&ioeventfd->next);
 453                                 vdev->ioeventfds_nr--;
 454                                 kfree(ioeventfd);
 455                                 ret = 0;
 456                         } else
 457                                 ret = -EEXIST;
 458
 459                         goto out_unlock;
 460                 }
 461         }
 462
 463         if (fd < 0) {
 464                 ret = -ENODEV;
 465                 goto out_unlock;
 466         }
 467
 468         if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
 469                 ret = -ENOSPC;
 470                 goto out_unlock;
 471         }
 472
 473         ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL_ACCOUNT);
 474         if (!ioeventfd) {
 475                 ret = -ENOMEM;
 476                 goto out_unlock;
 477         }
 478
 479         ioeventfd->vdev = vdev;
 480         ioeventfd->addr = vdev->barmap[bar] + pos;
 481         ioeventfd->data = data;
 482         ioeventfd->pos = pos;
 483         ioeventfd->bar = bar;
 484         ioeventfd->count = count;
 485         ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM;
 486
 487         ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
 488                                  vfio_pci_ioeventfd_thread, NULL,
 489                                  &ioeventfd->virqfd, fd);
 490         if (ret) {
 491                 kfree(ioeventfd);
 492                 goto out_unlock;
 493         }
 494
 495         list_add(&ioeventfd->next, &vdev->ioeventfds_list);
 496         vdev->ioeventfds_nr++;
 497
 498 out_unlock:
 499         mutex_unlock(&vdev->ioeventfds_lock);
 500
 501         return ret;
 502 }