1 // SPDX-License-Identifier: GPL-2.0-only
3 * VFIO PCI I/O Port & MMIO access
5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
6 * Author: Alex Williamson <alex.williamson@redhat.com>
8 * Derived from original vfio:
9 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
10 * Author: Tom Lyon, pugs@cisco.com
14 #include <linux/pci.h>
15 #include <linux/uaccess.h>
17 #include <linux/vfio.h>
18 #include <linux/vgaarb.h>
20 #include "vfio_pci_private.h"
22 #ifdef __LITTLE_ENDIAN
23 #define vfio_ioread64 ioread64
24 #define vfio_iowrite64 iowrite64
25 #define vfio_ioread32 ioread32
26 #define vfio_iowrite32 iowrite32
27 #define vfio_ioread16 ioread16
28 #define vfio_iowrite16 iowrite16
30 #define vfio_ioread64 ioread64be
31 #define vfio_iowrite64 iowrite64be
32 #define vfio_ioread32 ioread32be
33 #define vfio_iowrite32 iowrite32be
34 #define vfio_ioread16 ioread16be
35 #define vfio_iowrite16 iowrite16be
37 #define vfio_ioread8 ioread8
38 #define vfio_iowrite8 iowrite8
40 #define VFIO_IOWRITE(size) \
41 static int vfio_pci_iowrite##size(struct vfio_pci_device *vdev, \
42 bool test_mem, u##size val, void __iomem *io) \
45 down_read(&vdev->memory_lock); \
46 if (!__vfio_pci_memory_enabled(vdev)) { \
47 up_read(&vdev->memory_lock); \
52 vfio_iowrite##size(val, io); \
55 up_read(&vdev->memory_lock); \
67 #define VFIO_IOREAD(size) \
68 static int vfio_pci_ioread##size(struct vfio_pci_device *vdev, \
69 bool test_mem, u##size *val, void __iomem *io) \
72 down_read(&vdev->memory_lock); \
73 if (!__vfio_pci_memory_enabled(vdev)) { \
74 up_read(&vdev->memory_lock); \
79 *val = vfio_ioread##size(io); \
82 up_read(&vdev->memory_lock); \
92 * Read or write from an __iomem region (MMIO or I/O port) with an excluded
93 * range which is inaccessible. The excluded range drops writes and fills
94 * reads with -1. This is intended for handling MSI-X vector tables and
95 * leftover space for ROM BARs.
97 static ssize_t do_io_rw(struct vfio_pci_device *vdev, bool test_mem,
98 void __iomem *io, char __user *buf,
99 loff_t off, size_t count, size_t x_start,
100 size_t x_end, bool iswrite)
106 size_t fillable, filled;
109 fillable = min(count, (size_t)(x_start - off));
110 else if (off >= x_end)
115 if (fillable >= 4 && !(off % 4)) {
119 if (copy_from_user(&val, buf, 4))
122 ret = vfio_pci_iowrite32(vdev, test_mem,
127 ret = vfio_pci_ioread32(vdev, test_mem,
132 if (copy_to_user(buf, &val, 4))
137 } else if (fillable >= 2 && !(off % 2)) {
141 if (copy_from_user(&val, buf, 2))
144 ret = vfio_pci_iowrite16(vdev, test_mem,
149 ret = vfio_pci_ioread16(vdev, test_mem,
154 if (copy_to_user(buf, &val, 2))
159 } else if (fillable) {
163 if (copy_from_user(&val, buf, 1))
166 ret = vfio_pci_iowrite8(vdev, test_mem,
171 ret = vfio_pci_ioread8(vdev, test_mem,
176 if (copy_to_user(buf, &val, 1))
182 /* Fill reads with -1, drop writes */
183 filled = min(count, (size_t)(x_end - off));
188 for (i = 0; i < filled; i++)
189 if (copy_to_user(buf + i, &val, 1))
203 static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar)
205 struct pci_dev *pdev = vdev->pdev;
209 if (vdev->barmap[bar])
212 ret = pci_request_selected_regions(pdev, 1 << bar, "vfio");
216 io = pci_iomap(pdev, bar, 0);
218 pci_release_selected_regions(pdev, 1 << bar);
222 vdev->barmap[bar] = io;
227 ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
228 size_t count, loff_t *ppos, bool iswrite)
230 struct pci_dev *pdev = vdev->pdev;
231 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
232 int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
233 size_t x_start = 0, x_end = 0;
236 struct resource *res = &vdev->pdev->resource[bar];
239 if (pci_resource_start(pdev, bar))
240 end = pci_resource_len(pdev, bar);
241 else if (bar == PCI_ROM_RESOURCE &&
242 pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW)
250 count = min(count, (size_t)(end - pos));
252 if (bar == PCI_ROM_RESOURCE) {
254 * The ROM can fill less space than the BAR, so we start the
255 * excluded range at the end of the actual ROM. This makes
256 * filling large ROM BARs much faster.
258 io = pci_map_rom(pdev, &x_start);
265 int ret = vfio_pci_setup_barmap(vdev, bar);
271 io = vdev->barmap[bar];
274 if (bar == vdev->msix_bar) {
275 x_start = vdev->msix_offset;
276 x_end = vdev->msix_offset + vdev->msix_size;
279 done = do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos,
280 count, x_start, x_end, iswrite);
285 if (bar == PCI_ROM_RESOURCE)
286 pci_unmap_rom(pdev, io);
291 ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf,
292 size_t count, loff_t *ppos, bool iswrite)
295 loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK;
296 void __iomem *iomem = NULL;
308 case 0xa0000 ... 0xbffff:
309 count = min(count, (size_t)(0xc0000 - pos));
310 iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1);
312 rsrc = VGA_RSRC_LEGACY_MEM;
315 case 0x3b0 ... 0x3bb:
316 count = min(count, (size_t)(0x3bc - pos));
317 iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1);
319 rsrc = VGA_RSRC_LEGACY_IO;
322 case 0x3c0 ... 0x3df:
323 count = min(count, (size_t)(0x3e0 - pos));
324 iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1);
326 rsrc = VGA_RSRC_LEGACY_IO;
336 ret = vga_get_interruptible(vdev->pdev, rsrc);
338 is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
343 * VGA MMIO is a legacy, non-BAR resource that hopefully allows
344 * probing, so we don't currently worry about access in relation
345 * to the memory enable bit in the command register.
347 done = do_io_rw(vdev, false, iomem, buf, off, count, 0, 0, iswrite);
349 vga_put(vdev->pdev, rsrc);
351 is_ioport ? ioport_unmap(iomem) : iounmap(iomem);
359 static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
362 switch (ioeventfd->count) {
364 vfio_pci_iowrite8(ioeventfd->vdev, test_mem,
365 ioeventfd->data, ioeventfd->addr);
368 vfio_pci_iowrite16(ioeventfd->vdev, test_mem,
369 ioeventfd->data, ioeventfd->addr);
372 vfio_pci_iowrite32(ioeventfd->vdev, test_mem,
373 ioeventfd->data, ioeventfd->addr);
377 vfio_pci_iowrite64(ioeventfd->vdev, test_mem,
378 ioeventfd->data, ioeventfd->addr);
384 static int vfio_pci_ioeventfd_handler(void *opaque, void *unused)
386 struct vfio_pci_ioeventfd *ioeventfd = opaque;
387 struct vfio_pci_device *vdev = ioeventfd->vdev;
389 if (ioeventfd->test_mem) {
390 if (!down_read_trylock(&vdev->memory_lock))
391 return 1; /* Lock contended, use thread */
392 if (!__vfio_pci_memory_enabled(vdev)) {
393 up_read(&vdev->memory_lock);
398 vfio_pci_ioeventfd_do_write(ioeventfd, false);
400 if (ioeventfd->test_mem)
401 up_read(&vdev->memory_lock);
406 static void vfio_pci_ioeventfd_thread(void *opaque, void *unused)
408 struct vfio_pci_ioeventfd *ioeventfd = opaque;
410 vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
413 long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
414 uint64_t data, int count, int fd)
416 struct pci_dev *pdev = vdev->pdev;
417 loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
418 int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
419 struct vfio_pci_ioeventfd *ioeventfd;
421 /* Only support ioeventfds into BARs */
422 if (bar > VFIO_PCI_BAR5_REGION_INDEX)
425 if (pos + count > pci_resource_len(pdev, bar))
428 /* Disallow ioeventfds working around MSI-X table writes */
429 if (bar == vdev->msix_bar &&
430 !(pos + count <= vdev->msix_offset ||
431 pos >= vdev->msix_offset + vdev->msix_size))
439 ret = vfio_pci_setup_barmap(vdev, bar);
443 mutex_lock(&vdev->ioeventfds_lock);
445 list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) {
446 if (ioeventfd->pos == pos && ioeventfd->bar == bar &&
447 ioeventfd->data == data && ioeventfd->count == count) {
449 vfio_virqfd_disable(&ioeventfd->virqfd);
450 list_del(&ioeventfd->next);
451 vdev->ioeventfds_nr--;
466 if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) {
471 ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL);
477 ioeventfd->vdev = vdev;
478 ioeventfd->addr = vdev->barmap[bar] + pos;
479 ioeventfd->data = data;
480 ioeventfd->pos = pos;
481 ioeventfd->bar = bar;
482 ioeventfd->count = count;
483 ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM;
485 ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler,
486 vfio_pci_ioeventfd_thread, NULL,
487 &ioeventfd->virqfd, fd);
493 list_add(&ioeventfd->next, &vdev->ioeventfds_list);
494 vdev->ioeventfds_nr++;
497 mutex_unlock(&vdev->ioeventfds_lock);