1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * VAS user space API for its accelerators (Only NX-GZIP is supported now)
4 * Copyright (C) 2019 Haren Myneni, IBM Corp
7 #define pr_fmt(fmt) "vas-api: " fmt
9 #include <linux/kernel.h>
10 #include <linux/device.h>
11 #include <linux/cdev.h>
13 #include <linux/slab.h>
14 #include <linux/uaccess.h>
15 #include <linux/kthread.h>
16 #include <linux/sched/signal.h>
17 #include <linux/mmu_context.h>
20 #include <uapi/asm/vas-api.h>
23 * The driver creates the device node that can be used as follows:
26 * fd = open("/dev/crypto/nx-gzip", O_RDWR);
27 * rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr);
28 * paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL).
29 * vas_copy(&crb, 0, 1);
30 * vas_paste(paste_addr, 0, 1);
31 * close(fd) or exit process to close window.
33 * where "vas_copy" and "vas_paste" are defined in copy-paste.h.
34 * copy/paste returns to the user space directly. So refer NX hardware
35 * documentation for exact copy/paste usage and completion / error
40 * Wrapper object for the nx-gzip device - there is just one instance of
41 * this node for the whole system.
43 static struct coproc_dev {
45 struct device *device;
49 enum vas_cop_type cop_type;
50 const struct vas_user_win_ops *vops;
53 struct coproc_instance {
54 struct coproc_dev *coproc;
55 struct vas_window *txwin;
58 static char *coproc_devnode(const struct device *dev, umode_t *mode)
60 return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev));
64 * Take reference to pid and mm
66 int get_vas_user_win_ref(struct vas_user_win_ref *task_ref)
69 * Window opened by a child thread may not be closed when
70 * it exits. So take reference to its pid and release it
71 * when the window is free by parent thread.
72 * Acquire a reference to the task's pid to make sure
73 * pid will not be re-used - needed only for multithread
76 task_ref->pid = get_task_pid(current, PIDTYPE_PID);
78 * Acquire a reference to the task's mm.
80 task_ref->mm = get_task_mm(current);
82 put_pid(task_ref->pid);
83 pr_err("pid(%d): mm_struct is not found\n",
91 * Process closes window during exit. In the case of
92 * multithread application, the child thread can open
93 * window and can exit without closing it. So takes tgid
94 * reference until window closed to make sure tgid is not
97 task_ref->tgid = find_get_pid(task_tgid_vnr(current));
103 * Successful return must release the task reference with
106 static bool ref_get_pid_and_task(struct vas_user_win_ref *task_ref,
107 struct task_struct **tskp, struct pid **pidp)
109 struct task_struct *tsk;
113 tsk = get_pid_task(pid, PIDTYPE_PID);
115 pid = task_ref->tgid;
116 tsk = get_pid_task(pid, PIDTYPE_PID);
118 * Parent thread (tgid) will be closing window when it
119 * exits. So should not get here.
121 if (WARN_ON_ONCE(!tsk))
125 /* Return if the task is exiting. */
126 if (tsk->flags & PF_EXITING) {
127 put_task_struct(tsk);
138 * Update the CSB to indicate a translation error.
140 * User space will be polling on CSB after the request is issued.
141 * If NX can handle the request without any issues, it updates CSB.
142 * Whereas if NX encounters page fault, the kernel will handle the
143 * fault and update CSB with translation error.
145 * If we are unable to update the CSB means copy_to_user failed due to
146 * invalid csb_addr, send a signal to the process.
148 void vas_update_csb(struct coprocessor_request_block *crb,
149 struct vas_user_win_ref *task_ref)
151 struct coprocessor_status_block csb;
152 struct kernel_siginfo info;
153 struct task_struct *tsk;
154 void __user *csb_addr;
159 * NX user space windows can not be opened for task->mm=NULL
160 * and faults will not be generated for kernel requests.
162 if (WARN_ON_ONCE(!task_ref->mm))
165 csb_addr = (void __user *)be64_to_cpu(crb->csb_addr);
167 memset(&csb, 0, sizeof(csb));
168 csb.cc = CSB_CC_FAULT_ADDRESS;
169 csb.ce = CSB_CE_TERMINATION;
174 * NX operates and returns in BE format as defined CRB struct.
175 * So saves fault_storage_addr in BE as NX pastes in FIFO and
176 * expects user space to convert to CPU format.
178 csb.address = crb->stamp.nx.fault_storage_addr;
182 * Process closes send window after all pending NX requests are
183 * completed. In multi-thread applications, a child thread can
184 * open a window and can exit without closing it. May be some
185 * requests are pending or this window can be used by other
186 * threads later. We should handle faults if NX encounters
187 * pages faults on these requests. Update CSB with translation
188 * error and fault address. If csb_addr passed by user space is
189 * invalid, send SEGV signal to pid saved in window. If the
190 * child thread is not running, send the signal to tgid.
191 * Parent thread (tgid) will close this window upon its exit.
193 * pid and mm references are taken when window is opened by
194 * process (pid). So tgid is used only when child thread opens
195 * a window and exits without closing it.
198 if (!ref_get_pid_and_task(task_ref, &tsk, &pid))
201 kthread_use_mm(task_ref->mm);
202 rc = copy_to_user(csb_addr, &csb, sizeof(csb));
204 * User space polls on csb.flags (first byte). So add barrier
205 * then copy first byte with csb flags update.
209 /* Make sure update to csb.flags is visible now */
211 rc = copy_to_user(csb_addr, &csb, sizeof(u8));
213 kthread_unuse_mm(task_ref->mm);
214 put_task_struct(tsk);
221 pr_debug("Invalid CSB address 0x%p signalling pid(%d)\n",
222 csb_addr, pid_vnr(pid));
224 clear_siginfo(&info);
225 info.si_signo = SIGSEGV;
226 info.si_errno = EFAULT;
227 info.si_code = SEGV_MAPERR;
228 info.si_addr = csb_addr;
230 * process will be polling on csb.flags after request is sent to
231 * NX. So generally CSB update should not fail except when an
232 * application passes invalid csb_addr. So an error message will
233 * be displayed and leave it to user space whether to ignore or
234 * handle this signal.
237 rc = kill_pid_info(SIGSEGV, &info, pid);
240 pr_devel("pid %d kill_proc_info() rc %d\n", pid_vnr(pid), rc);
243 void vas_dump_crb(struct coprocessor_request_block *crb)
245 struct data_descriptor_entry *dde;
246 struct nx_fault_stamp *nx;
249 pr_devel("SrcDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n",
250 be64_to_cpu(dde->address), be32_to_cpu(dde->length),
251 dde->count, dde->index, dde->flags);
254 pr_devel("TgtDDE: addr 0x%llx, len %d, count %d, idx %d, flags %d\n",
255 be64_to_cpu(dde->address), be32_to_cpu(dde->length),
256 dde->count, dde->index, dde->flags);
259 pr_devel("NX Stamp: PSWID 0x%x, FSA 0x%llx, flags 0x%x, FS 0x%x\n",
260 be32_to_cpu(nx->pswid),
261 be64_to_cpu(crb->stamp.nx.fault_storage_addr),
262 nx->flags, nx->fault_status);
265 static int coproc_open(struct inode *inode, struct file *fp)
267 struct coproc_instance *cp_inst;
269 cp_inst = kzalloc(sizeof(*cp_inst), GFP_KERNEL);
273 cp_inst->coproc = container_of(inode->i_cdev, struct coproc_dev,
275 fp->private_data = cp_inst;
280 static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg)
282 void __user *uptr = (void __user *)arg;
283 struct vas_tx_win_open_attr uattr;
284 struct coproc_instance *cp_inst;
285 struct vas_window *txwin;
288 cp_inst = fp->private_data;
291 * One window for file descriptor
296 rc = copy_from_user(&uattr, uptr, sizeof(uattr));
298 pr_err("copy_from_user() returns %d\n", rc);
302 if (uattr.version != 1) {
303 pr_err("Invalid window open API version\n");
307 if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->open_win) {
308 pr_err("VAS API is not registered\n");
312 txwin = cp_inst->coproc->vops->open_win(uattr.vas_id, uattr.flags,
313 cp_inst->coproc->cop_type);
315 pr_err_ratelimited("VAS window open failed rc=%ld\n",
317 return PTR_ERR(txwin);
320 mutex_init(&txwin->task_ref.mmap_mutex);
321 cp_inst->txwin = txwin;
326 static int coproc_release(struct inode *inode, struct file *fp)
328 struct coproc_instance *cp_inst = fp->private_data;
331 if (cp_inst->txwin) {
332 if (cp_inst->coproc->vops &&
333 cp_inst->coproc->vops->close_win) {
334 rc = cp_inst->coproc->vops->close_win(cp_inst->txwin);
338 cp_inst->txwin = NULL;
342 fp->private_data = NULL;
345 * We don't know here if user has other receive windows
346 * open, so we can't really call clear_thread_tidr().
347 * So, once the process calls set_thread_tidr(), the
348 * TIDR value sticks around until process exits, resulting
349 * in an extra copy in restore_sprs().
356 * If the executed instruction that caused the fault was a paste, then
357 * clear regs CR0[EQ], advance NIP, and return 0. Else return error code.
359 static int do_fail_paste(void)
361 struct pt_regs *regs = current->thread.regs;
364 if (WARN_ON_ONCE(!regs))
367 if (WARN_ON_ONCE(!user_mode(regs)))
371 * If we couldn't translate the instruction, the driver should
372 * return success without handling the fault, it will be retried
373 * or the instruction fetch will fault.
375 if (get_user(instword, (u32 __user *)(regs->nip)))
379 * Not a paste instruction, driver may fail the fault.
381 if ((instword & PPC_INST_PASTE_MASK) != PPC_INST_PASTE)
384 regs->ccr &= ~0xe0000000; /* Clear CR0[0-2] to fail paste */
385 regs_add_return_ip(regs, 4); /* Emulate the paste */
391 * This fault handler is invoked when the core generates page fault on
392 * the paste address. Happens if the kernel closes window in hypervisor
393 * (on pseries) due to lost credit or the paste address is not mapped.
395 static vm_fault_t vas_mmap_fault(struct vm_fault *vmf)
397 struct vm_area_struct *vma = vmf->vma;
398 struct file *fp = vma->vm_file;
399 struct coproc_instance *cp_inst = fp->private_data;
400 struct vas_window *txwin;
406 * window is not opened. Shouldn't expect this error.
408 if (!cp_inst || !cp_inst->txwin) {
409 pr_err("Unexpected fault on paste address with TX window closed\n");
410 return VM_FAULT_SIGBUS;
413 txwin = cp_inst->txwin;
415 * When the LPAR lost credits due to core removal or during
416 * migration, invalidate the existing mapping for the current
417 * paste addresses and set windows in-active (zap_vma_pages in
418 * reconfig_close_windows()).
419 * New mapping will be done later after migration or new credits
420 * available. So continue to receive faults if the user space
423 if (txwin->task_ref.vma != vmf->vma) {
424 pr_err("No previous mapping with paste address\n");
425 return VM_FAULT_SIGBUS;
428 mutex_lock(&txwin->task_ref.mmap_mutex);
430 * The window may be inactive due to lost credit (Ex: core
431 * removal with DLPAR). If the window is active again when
432 * the credit is available, map the new paste address at the
433 * window virtual address.
435 if (txwin->status == VAS_WIN_ACTIVE) {
436 paste_addr = cp_inst->coproc->vops->paste_addr(txwin);
438 fault = vmf_insert_pfn(vma, vma->vm_start,
439 (paste_addr >> PAGE_SHIFT));
440 mutex_unlock(&txwin->task_ref.mmap_mutex);
444 mutex_unlock(&txwin->task_ref.mmap_mutex);
447 * Received this fault due to closing the actual window.
448 * It can happen during migration or lost credits.
449 * Since no mapping, return the paste instruction failure
452 ret = do_fail_paste();
454 * The user space can retry several times until success (needed
455 * for migration) or should fallback to SW compression or
456 * manage with the existing open windows if available.
457 * Looking at sysfs interface, it can determine whether these
458 * failures are coming during migration or core removal:
459 * nr_used_credits > nr_total_credits when lost credits
461 if (!ret || (ret == -EAGAIN))
462 return VM_FAULT_NOPAGE;
464 return VM_FAULT_SIGBUS;
467 static const struct vm_operations_struct vas_vm_ops = {
468 .fault = vas_mmap_fault,
471 static int coproc_mmap(struct file *fp, struct vm_area_struct *vma)
473 struct coproc_instance *cp_inst = fp->private_data;
474 struct vas_window *txwin;
480 txwin = cp_inst->txwin;
482 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
483 pr_debug("size 0x%zx, PAGE_SIZE 0x%zx\n",
484 (vma->vm_end - vma->vm_start), PAGE_SIZE);
488 /* Ensure instance has an open send window */
490 pr_err("No send window open?\n");
494 if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->paste_addr) {
495 pr_err("VAS API is not registered\n");
500 * The initial mmap is done after the window is opened
501 * with ioctl. But before mmap(), this window can be closed in
502 * the hypervisor due to lost credit (core removal on pseries).
503 * So if the window is not active, return mmap() failure with
504 * -EACCES and expects the user space reissue mmap() when it
505 * is active again or open new window when the credit is available.
506 * mmap_mutex protects the paste address mmap() with DLPAR
507 * close/open event and allows mmap() only when the window is
510 mutex_lock(&txwin->task_ref.mmap_mutex);
511 if (txwin->status != VAS_WIN_ACTIVE) {
512 pr_err("Window is not active\n");
517 paste_addr = cp_inst->coproc->vops->paste_addr(txwin);
519 pr_err("Window paste address failed\n");
524 pfn = paste_addr >> PAGE_SHIFT;
526 /* flags, page_prot from cxl_mmap(), except we want cachable */
527 vm_flags_set(vma, VM_IO | VM_PFNMAP);
528 vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
530 prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY);
532 rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
533 vma->vm_end - vma->vm_start, prot);
535 pr_devel("paste addr %llx at %lx, rc %d\n", paste_addr,
538 txwin->task_ref.vma = vma;
539 vma->vm_ops = &vas_vm_ops;
542 mutex_unlock(&txwin->task_ref.mmap_mutex);
546 static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
549 case VAS_TX_WIN_OPEN:
550 return coproc_ioc_tx_win_open(fp, arg);
556 static struct file_operations coproc_fops = {
558 .release = coproc_release,
560 .unlocked_ioctl = coproc_ioctl,
564 * Supporting only nx-gzip coprocessor type now, but this API code
565 * extended to other coprocessor types later.
567 int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type,
569 const struct vas_user_win_ops *vops)
574 rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, name);
576 pr_err("Unable to allocate coproc major number: %i\n", rc);
580 pr_devel("%s device allocated, dev [%i,%i]\n", name,
581 MAJOR(coproc_device.devt), MINOR(coproc_device.devt));
583 coproc_device.class = class_create(name);
584 if (IS_ERR(coproc_device.class)) {
585 rc = PTR_ERR(coproc_device.class);
586 pr_err("Unable to create %s class %d\n", name, rc);
589 coproc_device.class->devnode = coproc_devnode;
590 coproc_device.cop_type = cop_type;
591 coproc_device.vops = vops;
593 coproc_fops.owner = mod;
594 cdev_init(&coproc_device.cdev, &coproc_fops);
596 devno = MKDEV(MAJOR(coproc_device.devt), 0);
597 rc = cdev_add(&coproc_device.cdev, devno, 1);
599 pr_err("cdev_add() failed %d\n", rc);
603 coproc_device.device = device_create(coproc_device.class, NULL,
604 devno, NULL, name, MINOR(devno));
605 if (IS_ERR(coproc_device.device)) {
606 rc = PTR_ERR(coproc_device.device);
607 pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc);
611 pr_devel("Added dev [%d,%d]\n", MAJOR(devno), MINOR(devno));
616 cdev_del(&coproc_device.cdev);
618 class_destroy(coproc_device.class);
620 unregister_chrdev_region(coproc_device.devt, 1);
624 void vas_unregister_coproc_api(void)
628 cdev_del(&coproc_device.cdev);
629 devno = MKDEV(MAJOR(coproc_device.devt), 0);
630 device_destroy(coproc_device.class, devno);
632 class_destroy(coproc_device.class);
633 unregister_chrdev_region(coproc_device.devt, 1);