1 // SPDX-License-Identifier: GPL-2.0
3 * Access to PCI I/O memory from user space programs.
5 * Copyright IBM Corp. 2014
6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
8 #include <linux/kernel.h>
9 #include <linux/syscalls.h>
10 #include <linux/init.h>
12 #include <linux/errno.h>
13 #include <linux/pci.h>
14 #include <asm/asm-extable.h>
15 #include <asm/pci_io.h>
16 #include <asm/pci_debug.h>
18 static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
24 } data = {offset, cc, status};
26 zpci_err_hex(&data, sizeof(data));
29 static inline int __pcistb_mio_inuser(
30 void __iomem *ioaddr, const void __user *src,
37 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
41 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
42 : [cc] "+d" (cc), [len] "+d" (len)
43 : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
45 *status = len >> 24 & 0xff;
49 static inline int __pcistg_mio_inuser(
50 void __iomem *ioaddr, const void __user *src,
53 union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
60 * copy 0 < @len <= 8 bytes from @src into the right most bytes of
61 * a register, then store it to PCI at @ioaddr while in secondary
62 * address space. pcistg then uses the user mappings.
66 "0: llgc %[tmp],0(%[src])\n"
67 "4: sllg %[val],%[val],8\n"
69 " ogr %[val],%[tmp]\n"
71 "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n"
75 EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
77 [src] "+a" (src), [cnt] "+d" (cnt),
78 [val] "+d" (val), [tmp] "=d" (tmp),
79 [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair)
81 *status = ioaddr_len.odd >> 24 & 0xff;
83 /* did we read everything from user memory? */
90 static inline int __memcpy_toio_inuser(void __iomem *dst,
91 const void __user *src, size_t n)
100 size = zpci_get_max_io_size((u64 __force) dst,
101 (u64 __force) src, n,
102 ZPCI_MAX_WRITE_SIZE);
103 if (size > 8) /* main path */
104 rc = __pcistb_mio_inuser(dst, src, size, &status);
106 rc = __pcistg_mio_inuser(dst, src, size, &status);
114 zpci_err_mmio(rc, status, (__force u64) dst);
118 SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
119 const void __user *, user_buffer, size_t, length)
122 void __iomem *io_addr;
124 struct vm_area_struct *vma;
129 if (!zpci_is_enabled())
132 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
136 * We only support write access to MIO capable devices if we are on
137 * a MIO enabled system. Otherwise we would have to check for every
138 * address if it is a special ZPCI_ADDR and would have to do
139 * a pfn lookup which we don't need for MIO capable devices. Currently
140 * ISM devices are the only devices without MIO support and there is no
141 * known need for accessing these from userspace.
143 if (static_branch_likely(&have_mio)) {
144 ret = __memcpy_toio_inuser((void __iomem *) mmio_addr,
151 buf = kmalloc(length, GFP_KERNEL);
158 if (copy_from_user(buf, user_buffer, length))
161 mmap_read_lock(current->mm);
163 vma = vma_lookup(current->mm, mmio_addr);
165 goto out_unlock_mmap;
166 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
167 goto out_unlock_mmap;
169 if (!(vma->vm_flags & VM_WRITE))
170 goto out_unlock_mmap;
172 ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
174 goto out_unlock_mmap;
176 io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
177 (mmio_addr & ~PAGE_MASK));
179 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
182 ret = zpci_memcpy_toio(io_addr, buf, length);
184 pte_unmap_unlock(ptep, ptl);
186 mmap_read_unlock(current->mm);
188 if (buf != local_buf)
193 static inline int __pcilg_mio_inuser(
194 void __user *dst, const void __iomem *ioaddr,
195 u64 ulen, u8 *status)
197 union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
199 int shift = ulen * 8;
204 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
205 * user space) into a register using pcilg then store these bytes at
210 "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n"
215 "2: ahi %[shift],-8\n"
216 " srlg %[tmp],%[val],0(%[shift])\n"
217 "3: stc %[tmp],0(%[dst])\n"
221 EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b)
223 [ioaddr_len] "+&d" (ioaddr_len.pair),
224 [cc] "+d" (cc), [val] "=d" (val),
225 [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
229 /* did we write everything to the user space buffer? */
233 *status = ioaddr_len.odd >> 24 & 0xff;
237 static inline int __memcpy_fromio_inuser(void __user *dst,
238 const void __iomem *src,
245 size = zpci_get_max_io_size((u64 __force) src,
246 (u64 __force) dst, n,
248 rc = __pcilg_mio_inuser(dst, src, size, &status);
256 zpci_err_mmio(rc, status, (__force u64) dst);
260 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
261 void __user *, user_buffer, size_t, length)
264 void __iomem *io_addr;
266 struct vm_area_struct *vma;
271 if (!zpci_is_enabled())
274 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
278 * We only support read access to MIO capable devices if we are on
279 * a MIO enabled system. Otherwise we would have to check for every
280 * address if it is a special ZPCI_ADDR and would have to do
281 * a pfn lookup which we don't need for MIO capable devices. Currently
282 * ISM devices are the only devices without MIO support and there is no
283 * known need for accessing these from userspace.
285 if (static_branch_likely(&have_mio)) {
286 ret = __memcpy_fromio_inuser(
287 user_buffer, (const void __iomem *)mmio_addr,
293 buf = kmalloc(length, GFP_KERNEL);
300 mmap_read_lock(current->mm);
302 vma = vma_lookup(current->mm, mmio_addr);
304 goto out_unlock_mmap;
305 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
306 goto out_unlock_mmap;
308 if (!(vma->vm_flags & VM_WRITE))
309 goto out_unlock_mmap;
311 ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
313 goto out_unlock_mmap;
315 io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
316 (mmio_addr & ~PAGE_MASK));
318 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
322 ret = zpci_memcpy_fromio(buf, io_addr, length);
325 pte_unmap_unlock(ptep, ptl);
327 mmap_read_unlock(current->mm);
329 if (!ret && copy_to_user(user_buffer, buf, length))
332 if (buf != local_buf)