2 * IBM Accelerator Family 'GenWQE'
4 * (C) Copyright IBM Corp. 2013
6 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
7 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
8 * Author: Michael Jung <mijung@gmx.net>
9 * Author: Michael Ruettger <michael@ibmra.de>
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License (version 2 only)
13 * as published by the Free Software Foundation.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
22 * Character device representation of the GenWQE device. This allows
23 * user-space applications to communicate with the card.
26 #include <linux/kernel.h>
27 #include <linux/types.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/string.h>
32 #include <linux/sched/signal.h>
33 #include <linux/wait.h>
34 #include <linux/delay.h>
35 #include <linux/atomic.h>
37 #include "card_base.h"
38 #include "card_ddcb.h"
40 static int genwqe_open_files(struct genwqe_dev *cd)
45 spin_lock_irqsave(&cd->file_lock, flags);
46 rc = list_empty(&cd->file_list);
47 spin_unlock_irqrestore(&cd->file_lock, flags);
51 static void genwqe_add_file(struct genwqe_dev *cd, struct genwqe_file *cfile)
55 cfile->opener = get_pid(task_tgid(current));
56 spin_lock_irqsave(&cd->file_lock, flags);
57 list_add(&cfile->list, &cd->file_list);
58 spin_unlock_irqrestore(&cd->file_lock, flags);
61 static int genwqe_del_file(struct genwqe_dev *cd, struct genwqe_file *cfile)
65 spin_lock_irqsave(&cd->file_lock, flags);
66 list_del(&cfile->list);
67 spin_unlock_irqrestore(&cd->file_lock, flags);
68 put_pid(cfile->opener);
73 static void genwqe_add_pin(struct genwqe_file *cfile, struct dma_mapping *m)
77 spin_lock_irqsave(&cfile->pin_lock, flags);
78 list_add(&m->pin_list, &cfile->pin_list);
79 spin_unlock_irqrestore(&cfile->pin_lock, flags);
82 static int genwqe_del_pin(struct genwqe_file *cfile, struct dma_mapping *m)
86 spin_lock_irqsave(&cfile->pin_lock, flags);
87 list_del(&m->pin_list);
88 spin_unlock_irqrestore(&cfile->pin_lock, flags);
94 * genwqe_search_pin() - Search for the mapping for a userspace address
95 * @cfile: Descriptor of opened file
96 * @u_addr: User virtual address
97 * @size: Size of buffer
98 * @dma_addr: DMA address to be updated
100 * Return: Pointer to the corresponding mapping NULL if not found
102 static struct dma_mapping *genwqe_search_pin(struct genwqe_file *cfile,
103 unsigned long u_addr,
108 struct dma_mapping *m;
110 spin_lock_irqsave(&cfile->pin_lock, flags);
112 list_for_each_entry(m, &cfile->pin_list, pin_list) {
113 if ((((u64)m->u_vaddr) <= (u_addr)) &&
114 (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {
117 *virt_addr = m->k_vaddr +
118 (u_addr - (u64)m->u_vaddr);
120 spin_unlock_irqrestore(&cfile->pin_lock, flags);
124 spin_unlock_irqrestore(&cfile->pin_lock, flags);
128 static void __genwqe_add_mapping(struct genwqe_file *cfile,
129 struct dma_mapping *dma_map)
133 spin_lock_irqsave(&cfile->map_lock, flags);
134 list_add(&dma_map->card_list, &cfile->map_list);
135 spin_unlock_irqrestore(&cfile->map_lock, flags);
138 static void __genwqe_del_mapping(struct genwqe_file *cfile,
139 struct dma_mapping *dma_map)
143 spin_lock_irqsave(&cfile->map_lock, flags);
144 list_del(&dma_map->card_list);
145 spin_unlock_irqrestore(&cfile->map_lock, flags);
150 * __genwqe_search_mapping() - Search for the mapping for a userspace address
151 * @cfile: descriptor of opened file
152 * @u_addr: user virtual address
153 * @size: size of buffer
154 * @dma_addr: DMA address to be updated
155 * Return: Pointer to the corresponding mapping NULL if not found
157 static struct dma_mapping *__genwqe_search_mapping(struct genwqe_file *cfile,
158 unsigned long u_addr,
160 dma_addr_t *dma_addr,
164 struct dma_mapping *m;
165 struct pci_dev *pci_dev = cfile->cd->pci_dev;
167 spin_lock_irqsave(&cfile->map_lock, flags);
168 list_for_each_entry(m, &cfile->map_list, card_list) {
170 if ((((u64)m->u_vaddr) <= (u_addr)) &&
171 (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {
173 /* match found: current is as expected and
176 *dma_addr = m->dma_addr +
177 (u_addr - (u64)m->u_vaddr);
180 *virt_addr = m->k_vaddr +
181 (u_addr - (u64)m->u_vaddr);
183 spin_unlock_irqrestore(&cfile->map_lock, flags);
187 spin_unlock_irqrestore(&cfile->map_lock, flags);
189 dev_err(&pci_dev->dev,
190 "[%s] Entry not found: u_addr=%lx, size=%x\n",
191 __func__, u_addr, size);
196 static void genwqe_remove_mappings(struct genwqe_file *cfile)
199 struct list_head *node, *next;
200 struct dma_mapping *dma_map;
201 struct genwqe_dev *cd = cfile->cd;
202 struct pci_dev *pci_dev = cfile->cd->pci_dev;
204 list_for_each_safe(node, next, &cfile->map_list) {
205 dma_map = list_entry(node, struct dma_mapping, card_list);
207 list_del_init(&dma_map->card_list);
210 * This is really a bug, because those things should
211 * have been already tidied up.
213 * GENWQE_MAPPING_RAW should have been removed via mmunmap().
214 * GENWQE_MAPPING_SGL_TEMP should be removed by tidy up code.
216 dev_err(&pci_dev->dev,
217 "[%s] %d. cleanup mapping: u_vaddr=%p u_kaddr=%016lx dma_addr=%lx\n",
218 __func__, i++, dma_map->u_vaddr,
219 (unsigned long)dma_map->k_vaddr,
220 (unsigned long)dma_map->dma_addr);
222 if (dma_map->type == GENWQE_MAPPING_RAW) {
223 /* we allocated this dynamically */
224 __genwqe_free_consistent(cd, dma_map->size,
228 } else if (dma_map->type == GENWQE_MAPPING_SGL_TEMP) {
229 /* we use dma_map statically from the request */
230 genwqe_user_vunmap(cd, dma_map, NULL);
235 static void genwqe_remove_pinnings(struct genwqe_file *cfile)
237 struct list_head *node, *next;
238 struct dma_mapping *dma_map;
239 struct genwqe_dev *cd = cfile->cd;
241 list_for_each_safe(node, next, &cfile->pin_list) {
242 dma_map = list_entry(node, struct dma_mapping, pin_list);
245 * This is not a bug, because a killed processed might
246 * not call the unpin ioctl, which is supposed to free
249 * Pinnings are dymically allocated and need to be
252 list_del_init(&dma_map->pin_list);
253 genwqe_user_vunmap(cd, dma_map, NULL);
259 * genwqe_kill_fasync() - Send signal to all processes with open GenWQE files
261 * E.g. genwqe_send_signal(cd, SIGIO);
263 static int genwqe_kill_fasync(struct genwqe_dev *cd, int sig)
265 unsigned int files = 0;
267 struct genwqe_file *cfile;
269 spin_lock_irqsave(&cd->file_lock, flags);
270 list_for_each_entry(cfile, &cd->file_list, list) {
271 if (cfile->async_queue)
272 kill_fasync(&cfile->async_queue, sig, POLL_HUP);
275 spin_unlock_irqrestore(&cd->file_lock, flags);
279 static int genwqe_terminate(struct genwqe_dev *cd)
281 unsigned int files = 0;
283 struct genwqe_file *cfile;
285 spin_lock_irqsave(&cd->file_lock, flags);
286 list_for_each_entry(cfile, &cd->file_list, list) {
287 kill_pid(cfile->opener, SIGKILL, 1);
290 spin_unlock_irqrestore(&cd->file_lock, flags);
295 * genwqe_open() - file open
296 * @inode: file system information
299 * This function is executed whenever an application calls
300 * open("/dev/genwqe",..).
302 * Return: 0 if successful or <0 if errors
304 static int genwqe_open(struct inode *inode, struct file *filp)
306 struct genwqe_dev *cd;
307 struct genwqe_file *cfile;
308 struct pci_dev *pci_dev;
310 cfile = kzalloc(sizeof(*cfile), GFP_KERNEL);
314 cd = container_of(inode->i_cdev, struct genwqe_dev, cdev_genwqe);
315 pci_dev = cd->pci_dev;
318 cfile->client = NULL;
320 spin_lock_init(&cfile->map_lock); /* list of raw memory allocations */
321 INIT_LIST_HEAD(&cfile->map_list);
323 spin_lock_init(&cfile->pin_lock); /* list of user pinned memory */
324 INIT_LIST_HEAD(&cfile->pin_list);
326 filp->private_data = cfile;
328 genwqe_add_file(cd, cfile);
333 * genwqe_fasync() - Setup process to receive SIGIO.
334 * @fd: file descriptor
338 * Sending a signal is working as following:
340 * if (cdev->async_queue)
341 * kill_fasync(&cdev->async_queue, SIGIO, POLL_IN);
343 * Some devices also implement asynchronous notification to indicate
344 * when the device can be written; in this case, of course,
345 * kill_fasync must be called with a mode of POLL_OUT.
347 static int genwqe_fasync(int fd, struct file *filp, int mode)
349 struct genwqe_file *cdev = (struct genwqe_file *)filp->private_data;
351 return fasync_helper(fd, filp, mode, &cdev->async_queue);
356 * genwqe_release() - file close
357 * @inode: file system information
360 * This function is executed whenever an application calls 'close(fd_genwqe)'
364 static int genwqe_release(struct inode *inode, struct file *filp)
366 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
367 struct genwqe_dev *cd = cfile->cd;
369 /* there must be no entries in these lists! */
370 genwqe_remove_mappings(cfile);
371 genwqe_remove_pinnings(cfile);
373 /* remove this filp from the asynchronously notified filp's */
374 genwqe_fasync(-1, filp, 0);
377 * For this to work we must not release cd when this cfile is
378 * not yet released, otherwise the list entry is invalid,
379 * because the list itself gets reinstantiated!
381 genwqe_del_file(cd, cfile);
386 static void genwqe_vma_open(struct vm_area_struct *vma)
392 * genwqe_vma_close() - Called each time when vma is unmapped
394 * Free memory which got allocated by GenWQE mmap().
396 static void genwqe_vma_close(struct vm_area_struct *vma)
398 unsigned long vsize = vma->vm_end - vma->vm_start;
399 struct inode *inode = file_inode(vma->vm_file);
400 struct dma_mapping *dma_map;
401 struct genwqe_dev *cd = container_of(inode->i_cdev, struct genwqe_dev,
403 struct pci_dev *pci_dev = cd->pci_dev;
404 dma_addr_t d_addr = 0;
405 struct genwqe_file *cfile = vma->vm_private_data;
407 dma_map = __genwqe_search_mapping(cfile, vma->vm_start, vsize,
409 if (dma_map == NULL) {
410 dev_err(&pci_dev->dev,
411 " [%s] err: mapping not found: v=%lx, p=%lx s=%lx\n",
412 __func__, vma->vm_start, vma->vm_pgoff << PAGE_SHIFT,
416 __genwqe_del_mapping(cfile, dma_map);
417 __genwqe_free_consistent(cd, dma_map->size, dma_map->k_vaddr,
422 static const struct vm_operations_struct genwqe_vma_ops = {
423 .open = genwqe_vma_open,
424 .close = genwqe_vma_close,
428 * genwqe_mmap() - Provide contignous buffers to userspace
430 * We use mmap() to allocate contignous buffers used for DMA
431 * transfers. After the buffer is allocated we remap it to user-space
432 * and remember a reference to our dma_mapping data structure, where
433 * we store the associated DMA address and allocated size.
435 * When we receive a DDCB execution request with the ATS bits set to
436 * plain buffer, we lookup our dma_mapping list to find the
437 * corresponding DMA address for the associated user-space address.
439 static int genwqe_mmap(struct file *filp, struct vm_area_struct *vma)
442 unsigned long pfn, vsize = vma->vm_end - vma->vm_start;
443 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
444 struct genwqe_dev *cd = cfile->cd;
445 struct dma_mapping *dma_map;
450 if (get_order(vsize) > MAX_ORDER)
453 dma_map = kzalloc(sizeof(struct dma_mapping), GFP_KERNEL);
457 genwqe_mapping_init(dma_map, GENWQE_MAPPING_RAW);
458 dma_map->u_vaddr = (void *)vma->vm_start;
459 dma_map->size = vsize;
460 dma_map->nr_pages = DIV_ROUND_UP(vsize, PAGE_SIZE);
461 dma_map->k_vaddr = __genwqe_alloc_consistent(cd, vsize,
463 if (dma_map->k_vaddr == NULL) {
468 if (capable(CAP_SYS_ADMIN) && (vsize > sizeof(dma_addr_t)))
469 *(dma_addr_t *)dma_map->k_vaddr = dma_map->dma_addr;
471 pfn = virt_to_phys(dma_map->k_vaddr) >> PAGE_SHIFT;
472 rc = remap_pfn_range(vma,
482 vma->vm_private_data = cfile;
483 vma->vm_ops = &genwqe_vma_ops;
484 __genwqe_add_mapping(cfile, dma_map);
489 __genwqe_free_consistent(cd, dma_map->size,
498 * do_flash_update() - Excute flash update (write image or CVPD)
500 * @load: details about image load
502 * Return: 0 if successful
505 #define FLASH_BLOCK 0x40000 /* we use 256k blocks */
507 static int do_flash_update(struct genwqe_file *cfile,
508 struct genwqe_bitstream *load)
519 struct genwqe_dev *cd = cfile->cd;
520 struct file *filp = cfile->filp;
521 struct pci_dev *pci_dev = cd->pci_dev;
523 if ((load->size & 0x3) != 0)
526 if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0)
529 /* FIXME Bits have changed for new service layer! */
530 switch ((char)load->partition) {
533 break; /* download/erase_first/part_0 */
536 break; /* download/erase_first/part_1 */
539 break; /* download/erase_first/vpd */
544 buf = (u8 __user *)load->data_addr;
545 xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);
549 blocks_to_flash = load->size / FLASH_BLOCK;
551 struct genwqe_ddcb_cmd *req;
554 * We must be 4 byte aligned. Buffer must be 0 appened
555 * to have defined values when calculating CRC.
557 tocopy = min_t(size_t, load->size, FLASH_BLOCK);
559 rc = copy_from_user(xbuf, buf, tocopy);
564 crc = genwqe_crc32(xbuf, tocopy, 0xffffffff);
566 dev_dbg(&pci_dev->dev,
567 "[%s] DMA: %lx CRC: %08x SZ: %ld %d\n",
568 __func__, (unsigned long)dma_addr, crc, tocopy,
571 /* prepare DDCB for SLU process */
572 req = ddcb_requ_alloc();
578 req->cmd = SLCMD_MOVE_FLASH;
579 req->cmdopts = cmdopts;
581 /* prepare invariant values */
582 if (genwqe_get_slu_id(cd) <= 0x2) {
583 *(__be64 *)&req->__asiv[0] = cpu_to_be64(dma_addr);
584 *(__be64 *)&req->__asiv[8] = cpu_to_be64(tocopy);
585 *(__be64 *)&req->__asiv[16] = cpu_to_be64(flash);
586 *(__be32 *)&req->__asiv[24] = cpu_to_be32(0);
587 req->__asiv[24] = load->uid;
588 *(__be32 *)&req->__asiv[28] = cpu_to_be32(crc);
590 /* for simulation only */
591 *(__be64 *)&req->__asiv[88] = cpu_to_be64(load->slu_id);
592 *(__be64 *)&req->__asiv[96] = cpu_to_be64(load->app_id);
593 req->asiv_length = 32; /* bytes included in crc calc */
594 } else { /* setup DDCB for ATS architecture */
595 *(__be64 *)&req->asiv[0] = cpu_to_be64(dma_addr);
596 *(__be32 *)&req->asiv[8] = cpu_to_be32(tocopy);
597 *(__be32 *)&req->asiv[12] = cpu_to_be32(0); /* resvd */
598 *(__be64 *)&req->asiv[16] = cpu_to_be64(flash);
599 *(__be32 *)&req->asiv[24] = cpu_to_be32(load->uid<<24);
600 *(__be32 *)&req->asiv[28] = cpu_to_be32(crc);
602 /* for simulation only */
603 *(__be64 *)&req->asiv[80] = cpu_to_be64(load->slu_id);
604 *(__be64 *)&req->asiv[88] = cpu_to_be64(load->app_id);
607 req->ats = 0x4ULL << 44;
608 req->asiv_length = 40; /* bytes included in crc calc */
612 /* For Genwqe5 we get back the calculated CRC */
613 *(u64 *)&req->asv[0] = 0ULL; /* 0x80 */
615 rc = __genwqe_execute_raw_ddcb(cd, req, filp->f_flags);
617 load->retc = req->retc;
618 load->attn = req->attn;
619 load->progress = req->progress;
626 if (req->retc != DDCB_RETC_COMPLETE) {
632 load->size -= tocopy;
640 __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);
644 static int do_flash_read(struct genwqe_file *cfile,
645 struct genwqe_bitstream *load)
647 int rc, blocks_to_flash;
654 struct genwqe_dev *cd = cfile->cd;
655 struct file *filp = cfile->filp;
656 struct pci_dev *pci_dev = cd->pci_dev;
657 struct genwqe_ddcb_cmd *cmd;
659 if ((load->size & 0x3) != 0)
662 if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0)
665 /* FIXME Bits have changed for new service layer! */
666 switch ((char)load->partition) {
669 break; /* upload/part_0 */
672 break; /* upload/part_1 */
675 break; /* upload/vpd */
680 buf = (u8 __user *)load->data_addr;
681 xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);
685 blocks_to_flash = load->size / FLASH_BLOCK;
688 * We must be 4 byte aligned. Buffer must be 0 appened
689 * to have defined values when calculating CRC.
691 tocopy = min_t(size_t, load->size, FLASH_BLOCK);
693 dev_dbg(&pci_dev->dev,
694 "[%s] DMA: %lx SZ: %ld %d\n",
695 __func__, (unsigned long)dma_addr, tocopy,
698 /* prepare DDCB for SLU process */
699 cmd = ddcb_requ_alloc();
704 cmd->cmd = SLCMD_MOVE_FLASH;
705 cmd->cmdopts = cmdopts;
707 /* prepare invariant values */
708 if (genwqe_get_slu_id(cd) <= 0x2) {
709 *(__be64 *)&cmd->__asiv[0] = cpu_to_be64(dma_addr);
710 *(__be64 *)&cmd->__asiv[8] = cpu_to_be64(tocopy);
711 *(__be64 *)&cmd->__asiv[16] = cpu_to_be64(flash);
712 *(__be32 *)&cmd->__asiv[24] = cpu_to_be32(0);
713 cmd->__asiv[24] = load->uid;
714 *(__be32 *)&cmd->__asiv[28] = cpu_to_be32(0) /* CRC */;
715 cmd->asiv_length = 32; /* bytes included in crc calc */
716 } else { /* setup DDCB for ATS architecture */
717 *(__be64 *)&cmd->asiv[0] = cpu_to_be64(dma_addr);
718 *(__be32 *)&cmd->asiv[8] = cpu_to_be32(tocopy);
719 *(__be32 *)&cmd->asiv[12] = cpu_to_be32(0); /* resvd */
720 *(__be64 *)&cmd->asiv[16] = cpu_to_be64(flash);
721 *(__be32 *)&cmd->asiv[24] = cpu_to_be32(load->uid<<24);
722 *(__be32 *)&cmd->asiv[28] = cpu_to_be32(0); /* CRC */
725 cmd->ats = 0x5ULL << 44;
726 cmd->asiv_length = 40; /* bytes included in crc calc */
730 /* we only get back the calculated CRC */
731 *(u64 *)&cmd->asv[0] = 0ULL; /* 0x80 */
733 rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);
735 load->retc = cmd->retc;
736 load->attn = cmd->attn;
737 load->progress = cmd->progress;
739 if ((rc < 0) && (rc != -EBADMSG)) {
744 rc = copy_to_user(buf, xbuf, tocopy);
751 /* We know that we can get retc 0x104 with CRC err */
752 if (((cmd->retc == DDCB_RETC_FAULT) &&
753 (cmd->attn != 0x02)) || /* Normally ignore CRC error */
754 ((cmd->retc == DDCB_RETC_COMPLETE) &&
755 (cmd->attn != 0x00))) { /* Everything was fine */
761 load->size -= tocopy;
770 __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);
774 static int genwqe_pin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
777 struct genwqe_dev *cd = cfile->cd;
778 struct pci_dev *pci_dev = cfile->cd->pci_dev;
779 struct dma_mapping *dma_map;
780 unsigned long map_addr;
781 unsigned long map_size;
783 if ((m->addr == 0x0) || (m->size == 0))
785 if (m->size > ULONG_MAX - PAGE_SIZE - (m->addr & ~PAGE_MASK))
788 map_addr = (m->addr & PAGE_MASK);
789 map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);
791 dma_map = kzalloc(sizeof(struct dma_mapping), GFP_KERNEL);
795 genwqe_mapping_init(dma_map, GENWQE_MAPPING_SGL_PINNED);
796 rc = genwqe_user_vmap(cd, dma_map, (void *)map_addr, map_size, NULL);
798 dev_err(&pci_dev->dev,
799 "[%s] genwqe_user_vmap rc=%d\n", __func__, rc);
804 genwqe_add_pin(cfile, dma_map);
808 static int genwqe_unpin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
810 struct genwqe_dev *cd = cfile->cd;
811 struct dma_mapping *dma_map;
812 unsigned long map_addr;
813 unsigned long map_size;
818 map_addr = (m->addr & PAGE_MASK);
819 map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);
821 dma_map = genwqe_search_pin(cfile, map_addr, map_size, NULL);
825 genwqe_del_pin(cfile, dma_map);
826 genwqe_user_vunmap(cd, dma_map, NULL);
832 * ddcb_cmd_cleanup() - Remove dynamically created fixup entries
834 * Only if there are any. Pinnings are not removed.
836 static int ddcb_cmd_cleanup(struct genwqe_file *cfile, struct ddcb_requ *req)
839 struct dma_mapping *dma_map;
840 struct genwqe_dev *cd = cfile->cd;
842 for (i = 0; i < DDCB_FIXUPS; i++) {
843 dma_map = &req->dma_mappings[i];
845 if (dma_mapping_used(dma_map)) {
846 __genwqe_del_mapping(cfile, dma_map);
847 genwqe_user_vunmap(cd, dma_map, req);
849 if (req->sgls[i].sgl != NULL)
850 genwqe_free_sync_sgl(cd, &req->sgls[i]);
856 * ddcb_cmd_fixups() - Establish DMA fixups/sglists for user memory references
858 * Before the DDCB gets executed we need to handle the fixups. We
859 * replace the user-space addresses with DMA addresses or do
860 * additional setup work e.g. generating a scatter-gather list which
861 * is used to describe the memory referred to in the fixup.
863 static int ddcb_cmd_fixups(struct genwqe_file *cfile, struct ddcb_requ *req)
866 unsigned int asiv_offs, i;
867 struct genwqe_dev *cd = cfile->cd;
868 struct genwqe_ddcb_cmd *cmd = &req->cmd;
869 struct dma_mapping *m;
870 const char *type = "UNKNOWN";
872 for (i = 0, asiv_offs = 0x00; asiv_offs <= 0x58;
873 i++, asiv_offs += 0x08) {
880 ats_flags = ATS_GET_FLAGS(cmd->ats, asiv_offs);
885 break; /* nothing to do here */
887 case ATS_TYPE_FLAT_RDWR:
888 case ATS_TYPE_FLAT_RD: {
889 u_addr = be64_to_cpu(*((__be64 *)&cmd->
891 u_size = be32_to_cpu(*((__be32 *)&cmd->
892 asiv[asiv_offs + 0x08]));
895 * No data available. Ignore u_addr in this
896 * case and set addr to 0. Hardware must not
900 *((__be64 *)&cmd->asiv[asiv_offs]) =
905 m = __genwqe_search_mapping(cfile, u_addr, u_size,
912 *((__be64 *)&cmd->asiv[asiv_offs]) =
917 case ATS_TYPE_SGL_RDWR:
918 case ATS_TYPE_SGL_RD: {
921 u_addr = be64_to_cpu(*((__be64 *)
922 &cmd->asiv[asiv_offs]));
923 u_size = be32_to_cpu(*((__be32 *)
924 &cmd->asiv[asiv_offs + 0x08]));
927 * No data available. Ignore u_addr in this
928 * case and set addr to 0. Hardware must not
929 * fetch the empty sgl.
932 *((__be64 *)&cmd->asiv[asiv_offs]) =
937 m = genwqe_search_pin(cfile, u_addr, u_size, NULL);
940 page_offs = (u_addr -
941 (u64)m->u_vaddr)/PAGE_SIZE;
944 m = &req->dma_mappings[i];
946 genwqe_mapping_init(m,
947 GENWQE_MAPPING_SGL_TEMP);
948 rc = genwqe_user_vmap(cd, m, (void *)u_addr,
953 __genwqe_add_mapping(cfile, m);
957 /* create genwqe style scatter gather list */
958 rc = genwqe_alloc_sync_sgl(cd, &req->sgls[i],
959 (void __user *)u_addr,
964 genwqe_setup_sgl(cd, &req->sgls[i],
965 &m->dma_list[page_offs]);
967 *((__be64 *)&cmd->asiv[asiv_offs]) =
968 cpu_to_be64(req->sgls[i].sgl_dma_addr);
980 ddcb_cmd_cleanup(cfile, req);
985 * genwqe_execute_ddcb() - Execute DDCB using userspace address fixups
987 * The code will build up the translation tables or lookup the
988 * contignous memory allocation table to find the right translations
991 static int genwqe_execute_ddcb(struct genwqe_file *cfile,
992 struct genwqe_ddcb_cmd *cmd)
995 struct genwqe_dev *cd = cfile->cd;
996 struct file *filp = cfile->filp;
997 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
999 rc = ddcb_cmd_fixups(cfile, req);
1003 rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);
1004 ddcb_cmd_cleanup(cfile, req);
1008 static int do_execute_ddcb(struct genwqe_file *cfile,
1009 unsigned long arg, int raw)
1012 struct genwqe_ddcb_cmd *cmd;
1013 struct ddcb_requ *req;
1014 struct genwqe_dev *cd = cfile->cd;
1015 struct file *filp = cfile->filp;
1017 cmd = ddcb_requ_alloc();
1021 req = container_of(cmd, struct ddcb_requ, cmd);
1023 if (copy_from_user(cmd, (void __user *)arg, sizeof(*cmd))) {
1024 ddcb_requ_free(cmd);
1029 rc = genwqe_execute_ddcb(cfile, cmd);
1031 rc = __genwqe_execute_raw_ddcb(cd, cmd, filp->f_flags);
1033 /* Copy back only the modifed fields. Do not copy ASIV
1034 back since the copy got modified by the driver. */
1035 if (copy_to_user((void __user *)arg, cmd,
1036 sizeof(*cmd) - DDCB_ASIV_LENGTH)) {
1037 ddcb_requ_free(cmd);
1041 ddcb_requ_free(cmd);
1046 * genwqe_ioctl() - IO control
1047 * @filp: file handle
1048 * @cmd: command identifier (passed from user)
1049 * @arg: argument (passed from user)
1053 static long genwqe_ioctl(struct file *filp, unsigned int cmd,
1057 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
1058 struct genwqe_dev *cd = cfile->cd;
1059 struct pci_dev *pci_dev = cd->pci_dev;
1060 struct genwqe_reg_io __user *io;
1064 /* Return -EIO if card hit EEH */
1065 if (pci_channel_offline(pci_dev))
1068 if (_IOC_TYPE(cmd) != GENWQE_IOC_CODE)
1073 case GENWQE_GET_CARD_STATE:
1074 put_user(cd->card_state, (enum genwqe_card_state __user *)arg);
1077 /* Register access */
1078 case GENWQE_READ_REG64: {
1079 io = (struct genwqe_reg_io __user *)arg;
1081 if (get_user(reg_offs, &io->num))
1084 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))
1087 val = __genwqe_readq(cd, reg_offs);
1088 put_user(val, &io->val64);
1092 case GENWQE_WRITE_REG64: {
1093 io = (struct genwqe_reg_io __user *)arg;
1095 if (!capable(CAP_SYS_ADMIN))
1098 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1101 if (get_user(reg_offs, &io->num))
1104 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))
1107 if (get_user(val, &io->val64))
1110 __genwqe_writeq(cd, reg_offs, val);
1114 case GENWQE_READ_REG32: {
1115 io = (struct genwqe_reg_io __user *)arg;
1117 if (get_user(reg_offs, &io->num))
1120 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))
1123 val = __genwqe_readl(cd, reg_offs);
1124 put_user(val, &io->val64);
1128 case GENWQE_WRITE_REG32: {
1129 io = (struct genwqe_reg_io __user *)arg;
1131 if (!capable(CAP_SYS_ADMIN))
1134 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1137 if (get_user(reg_offs, &io->num))
1140 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))
1143 if (get_user(val, &io->val64))
1146 __genwqe_writel(cd, reg_offs, val);
1150 /* Flash update/reading */
1151 case GENWQE_SLU_UPDATE: {
1152 struct genwqe_bitstream load;
1154 if (!genwqe_is_privileged(cd))
1157 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1160 if (copy_from_user(&load, (void __user *)arg,
1164 rc = do_flash_update(cfile, &load);
1166 if (copy_to_user((void __user *)arg, &load, sizeof(load)))
1172 case GENWQE_SLU_READ: {
1173 struct genwqe_bitstream load;
1175 if (!genwqe_is_privileged(cd))
1178 if (genwqe_flash_readback_fails(cd))
1179 return -ENOSPC; /* known to fail for old versions */
1181 if (copy_from_user(&load, (void __user *)arg, sizeof(load)))
1184 rc = do_flash_read(cfile, &load);
1186 if (copy_to_user((void __user *)arg, &load, sizeof(load)))
1192 /* memory pinning and unpinning */
1193 case GENWQE_PIN_MEM: {
1194 struct genwqe_mem m;
1196 if (copy_from_user(&m, (void __user *)arg, sizeof(m)))
1199 return genwqe_pin_mem(cfile, &m);
1202 case GENWQE_UNPIN_MEM: {
1203 struct genwqe_mem m;
1205 if (copy_from_user(&m, (void __user *)arg, sizeof(m)))
1208 return genwqe_unpin_mem(cfile, &m);
1211 /* launch an DDCB and wait for completion */
1212 case GENWQE_EXECUTE_DDCB:
1213 return do_execute_ddcb(cfile, arg, 0);
1215 case GENWQE_EXECUTE_RAW_DDCB: {
1217 if (!capable(CAP_SYS_ADMIN))
1220 return do_execute_ddcb(cfile, arg, 1);
1230 #if defined(CONFIG_COMPAT)
1232 * genwqe_compat_ioctl() - Compatibility ioctl
1234 * Called whenever a 32-bit process running under a 64-bit kernel
1235 * performs an ioctl on /dev/genwqe<n>_card.
1237 * @filp: file pointer.
1239 * @arg: user argument.
1240 * Return: zero on success or negative number on failure.
1242 static long genwqe_compat_ioctl(struct file *filp, unsigned int cmd,
1245 return genwqe_ioctl(filp, cmd, arg);
1247 #endif /* defined(CONFIG_COMPAT) */
1249 static const struct file_operations genwqe_fops = {
1250 .owner = THIS_MODULE,
1251 .open = genwqe_open,
1252 .fasync = genwqe_fasync,
1253 .mmap = genwqe_mmap,
1254 .unlocked_ioctl = genwqe_ioctl,
1255 #if defined(CONFIG_COMPAT)
1256 .compat_ioctl = genwqe_compat_ioctl,
1258 .release = genwqe_release,
1261 static int genwqe_device_initialized(struct genwqe_dev *cd)
1263 return cd->dev != NULL;
1267 * genwqe_device_create() - Create and configure genwqe char device
1268 * @cd: genwqe device descriptor
1270 * This function must be called before we create any more genwqe
1271 * character devices, because it is allocating the major and minor
1272 * number which are supposed to be used by the client drivers.
1274 int genwqe_device_create(struct genwqe_dev *cd)
1277 struct pci_dev *pci_dev = cd->pci_dev;
1280 * Here starts the individual setup per client. It must
1281 * initialize its own cdev data structure with its own fops.
1282 * The appropriate devnum needs to be created. The ranges must
1285 rc = alloc_chrdev_region(&cd->devnum_genwqe, 0,
1286 GENWQE_MAX_MINOR, GENWQE_DEVNAME);
1288 dev_err(&pci_dev->dev, "err: alloc_chrdev_region failed\n");
1292 cdev_init(&cd->cdev_genwqe, &genwqe_fops);
1293 cd->cdev_genwqe.owner = THIS_MODULE;
1295 rc = cdev_add(&cd->cdev_genwqe, cd->devnum_genwqe, 1);
1297 dev_err(&pci_dev->dev, "err: cdev_add failed\n");
1302 * Finally the device in /dev/... must be created. The rule is
1303 * to use card%d_clientname for each created device.
1305 cd->dev = device_create_with_groups(cd->class_genwqe,
1307 cd->devnum_genwqe, cd,
1308 genwqe_attribute_groups,
1309 GENWQE_DEVNAME "%u_card",
1311 if (IS_ERR(cd->dev)) {
1312 rc = PTR_ERR(cd->dev);
1316 rc = genwqe_init_debugfs(cd);
1323 device_destroy(cd->class_genwqe, cd->devnum_genwqe);
1325 cdev_del(&cd->cdev_genwqe);
1327 unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);
1333 static int genwqe_inform_and_stop_processes(struct genwqe_dev *cd)
1337 struct pci_dev *pci_dev = cd->pci_dev;
1339 if (!genwqe_open_files(cd))
1342 dev_warn(&pci_dev->dev, "[%s] send SIGIO and wait ...\n", __func__);
1344 rc = genwqe_kill_fasync(cd, SIGIO);
1346 /* give kill_timeout seconds to close file descriptors ... */
1347 for (i = 0; (i < genwqe_kill_timeout) &&
1348 genwqe_open_files(cd); i++) {
1349 dev_info(&pci_dev->dev, " %d sec ...", i);
1355 /* if no open files we can safely continue, else ... */
1356 if (!genwqe_open_files(cd))
1359 dev_warn(&pci_dev->dev,
1360 "[%s] send SIGKILL and wait ...\n", __func__);
1362 rc = genwqe_terminate(cd);
1364 /* Give kill_timout more seconds to end processes */
1365 for (i = 0; (i < genwqe_kill_timeout) &&
1366 genwqe_open_files(cd); i++) {
1367 dev_warn(&pci_dev->dev, " %d sec ...", i);
1378 * genwqe_device_remove() - Remove genwqe's char device
1380 * This function must be called after the client devices are removed
1381 * because it will free the major/minor number range for the genwqe
1384 * This function must be robust enough to be called twice.
1386 int genwqe_device_remove(struct genwqe_dev *cd)
1389 struct pci_dev *pci_dev = cd->pci_dev;
1391 if (!genwqe_device_initialized(cd))
1394 genwqe_inform_and_stop_processes(cd);
1397 * We currently do wait until all filedescriptors are
1398 * closed. This leads to a problem when we abort the
1399 * application which will decrease this reference from
1400 * 1/unused to 0/illegal and not from 2/used 1/empty.
1402 rc = kref_read(&cd->cdev_genwqe.kobj.kref);
1404 dev_err(&pci_dev->dev,
1405 "[%s] err: cdev_genwqe...refcount=%d\n", __func__, rc);
1406 panic("Fatal err: cannot free resources with pending references!");
1409 genqwe_exit_debugfs(cd);
1410 device_destroy(cd->class_genwqe, cd->devnum_genwqe);
1411 cdev_del(&cd->cdev_genwqe);
1412 unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);