1 // SPDX-License-Identifier: GPL-2.0
3 * Access to PCI I/O memory from user space programs.
5 * Copyright IBM Corp. 2014
6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
8 #include <linux/kernel.h>
9 #include <linux/syscalls.h>
10 #include <linux/init.h>
12 #include <linux/errno.h>
13 #include <linux/pci.h>
14 #include <asm/pci_io.h>
15 #include <asm/pci_debug.h>
17 static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
23 } data = {offset, cc, status};
25 zpci_err_hex(&data, sizeof(data));
28 static inline int __pcistb_mio_inuser(
29 void __iomem *ioaddr, const void __user *src,
36 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
40 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
41 : [cc] "+d" (cc), [len] "+d" (len)
42 : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
44 *status = len >> 24 & 0xff;
48 static inline int __pcistg_mio_inuser(
49 void __iomem *ioaddr, const void __user *src,
52 register u64 addr asm("2") = (u64 __force) ioaddr;
53 register u64 len asm("3") = ulen;
60 * copy 0 < @len <= 8 bytes from @src into the right most bytes of
61 * a register, then store it to PCI at @ioaddr while in secondary
62 * address space. pcistg then uses the user mappings.
66 "0: llgc %[tmp],0(%[src])\n"
67 "4: sllg %[val],%[val],8\n"
69 " ogr %[val],%[tmp]\n"
71 "1: .insn rre,0xb9d40000,%[val],%[ioaddr]\n"
75 EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
77 [src] "+a" (src), [cnt] "+d" (cnt),
78 [val] "+d" (val), [tmp] "=d" (tmp),
79 [len] "+d" (len), [cc] "+d" (cc),
82 *status = len >> 24 & 0xff;
84 /* did we read everything from user memory? */
91 static inline int __memcpy_toio_inuser(void __iomem *dst,
92 const void __user *src, size_t n)
101 old_fs = enable_sacf_uaccess();
103 size = zpci_get_max_io_size((u64 __force) dst,
104 (u64 __force) src, n,
105 ZPCI_MAX_WRITE_SIZE);
106 if (size > 8) /* main path */
107 rc = __pcistb_mio_inuser(dst, src, size, &status);
109 rc = __pcistg_mio_inuser(dst, src, size, &status);
116 disable_sacf_uaccess(old_fs);
118 zpci_err_mmio(rc, status, (__force u64) dst);
122 static long get_pfn(unsigned long user_addr, unsigned long access,
125 struct vm_area_struct *vma;
128 down_read(¤t->mm->mmap_sem);
130 vma = find_vma(current->mm, user_addr);
131 if (!vma || user_addr < vma->vm_start)
134 if (!(vma->vm_flags & access))
136 ret = follow_pfn(vma, user_addr, pfn);
138 up_read(¤t->mm->mmap_sem);
142 SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
143 const void __user *, user_buffer, size_t, length)
146 void __iomem *io_addr;
151 if (!zpci_is_enabled())
154 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
158 * Only support read access to MIO capable devices on a MIO enabled
159 * system. Otherwise we would have to check for every address if it is
160 * a special ZPCI_ADDR and we would have to do a get_pfn() which we
161 * don't need for MIO capable devices.
163 if (static_branch_likely(&have_mio)) {
164 ret = __memcpy_toio_inuser((void __iomem *) mmio_addr,
171 buf = kmalloc(length, GFP_KERNEL);
177 ret = get_pfn(mmio_addr, VM_WRITE, &pfn);
180 io_addr = (void __iomem *)((pfn << PAGE_SHIFT) |
181 (mmio_addr & ~PAGE_MASK));
184 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
187 if (copy_from_user(buf, user_buffer, length))
190 ret = zpci_memcpy_toio(io_addr, buf, length);
192 if (buf != local_buf)
197 static inline int __pcilg_mio_inuser(
198 void __user *dst, const void __iomem *ioaddr,
199 u64 ulen, u8 *status)
201 register u64 addr asm("2") = (u64 __force) ioaddr;
202 register u64 len asm("3") = ulen;
204 int shift = ulen * 8;
209 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
210 * user space) into a register using pcilg then store these bytes at
215 "0: .insn rre,0xb9d60000,%[val],%[ioaddr]\n"
220 "2: ahi %[shift],-8\n"
221 " srlg %[tmp],%[val],0(%[shift])\n"
222 "3: stc %[tmp],0(%[dst])\n"
226 EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b)
228 [cc] "+d" (cc), [val] "=d" (val), [len] "+d" (len),
229 [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
235 /* did we write everything to the user space buffer? */
239 *status = len >> 24 & 0xff;
243 static inline int __memcpy_fromio_inuser(void __user *dst,
244 const void __iomem *src,
251 old_fs = enable_sacf_uaccess();
253 size = zpci_get_max_io_size((u64 __force) src,
254 (u64 __force) dst, n,
256 rc = __pcilg_mio_inuser(dst, src, size, &status);
263 disable_sacf_uaccess(old_fs);
265 zpci_err_mmio(rc, status, (__force u64) dst);
269 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
270 void __user *, user_buffer, size_t, length)
273 void __iomem *io_addr;
278 if (!zpci_is_enabled())
281 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
285 * Only support write access to MIO capable devices on a MIO enabled
286 * system. Otherwise we would have to check for every address if it is
287 * a special ZPCI_ADDR and we would have to do a get_pfn() which we
288 * don't need for MIO capable devices.
290 if (static_branch_likely(&have_mio)) {
291 ret = __memcpy_fromio_inuser(
292 user_buffer, (const void __iomem *)mmio_addr,
298 buf = kmalloc(length, GFP_KERNEL);
305 ret = get_pfn(mmio_addr, VM_READ, &pfn);
308 io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK));
310 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
314 ret = zpci_memcpy_fromio(buf, io_addr, length);
317 if (copy_to_user(user_buffer, buf, length))
321 if (buf != local_buf)