1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/netdevice.h>
38 #include <linux/etherdevice.h>
40 #include <linux/tcp.h>
41 #include <linux/if_vlan.h>
42 #include <net/addrconf.h>
45 #include "i40iw_register.h"
46 #include <net/netevent.h>
47 #define CLIENT_IW_INTERFACE_VERSION_MAJOR 0
48 #define CLIENT_IW_INTERFACE_VERSION_MINOR 01
49 #define CLIENT_IW_INTERFACE_VERSION_BUILD 00
51 #define DRV_VERSION_MAJOR 0
52 #define DRV_VERSION_MINOR 5
53 #define DRV_VERSION_BUILD 123
54 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
55 __stringify(DRV_VERSION_MINOR) "." __stringify(DRV_VERSION_BUILD)
58 module_param(debug, int, 0644);
59 MODULE_PARM_DESC(debug, "debug flags: 0=disabled (default), 0x7fffffff=all");
61 static int resource_profile;
62 module_param(resource_profile, int, 0644);
63 MODULE_PARM_DESC(resource_profile,
64 "Resource Profile: 0=no VF RDMA support (default), 1=Weighted VF, 2=Even Distribution");
66 static int max_rdma_vfs = 32;
67 module_param(max_rdma_vfs, int, 0644);
68 MODULE_PARM_DESC(max_rdma_vfs, "Maximum VF count: 0-32 32=default");
69 static int mpa_version = 2;
70 module_param(mpa_version, int, 0644);
71 MODULE_PARM_DESC(mpa_version, "MPA version to be used in MPA Req/Resp 1 or 2");
73 MODULE_AUTHOR("Intel Corporation, <e1000-rdma@lists.sourceforge.net>");
74 MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver");
75 MODULE_LICENSE("Dual BSD/GPL");
76 MODULE_VERSION(DRV_VERSION);
78 static struct i40e_client i40iw_client;
79 static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw";
81 static LIST_HEAD(i40iw_handlers);
82 static spinlock_t i40iw_handler_lock;
84 static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
85 u32 vf_id, u8 *msg, u16 len);
87 static struct notifier_block i40iw_inetaddr_notifier = {
88 .notifier_call = i40iw_inetaddr_event
91 static struct notifier_block i40iw_inetaddr6_notifier = {
92 .notifier_call = i40iw_inet6addr_event
95 static struct notifier_block i40iw_net_notifier = {
96 .notifier_call = i40iw_net_event
99 static atomic_t i40iw_notifiers_registered;
102 * i40iw_find_i40e_handler - find a handler given a client info
103 * @ldev: pointer to a client info
105 static struct i40iw_handler *i40iw_find_i40e_handler(struct i40e_info *ldev)
107 struct i40iw_handler *hdl;
110 spin_lock_irqsave(&i40iw_handler_lock, flags);
111 list_for_each_entry(hdl, &i40iw_handlers, list) {
112 if (hdl->ldev.netdev == ldev->netdev) {
113 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
117 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
122 * i40iw_find_netdev - find a handler given a netdev
123 * @netdev: pointer to net_device
125 struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev)
127 struct i40iw_handler *hdl;
130 spin_lock_irqsave(&i40iw_handler_lock, flags);
131 list_for_each_entry(hdl, &i40iw_handlers, list) {
132 if (hdl->ldev.netdev == netdev) {
133 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
137 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
142 * i40iw_add_handler - add a handler to the list
143 * @hdl: handler to be added to the handler list
145 static void i40iw_add_handler(struct i40iw_handler *hdl)
149 spin_lock_irqsave(&i40iw_handler_lock, flags);
150 list_add(&hdl->list, &i40iw_handlers);
151 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
155 * i40iw_del_handler - delete a handler from the list
156 * @hdl: handler to be deleted from the handler list
158 static int i40iw_del_handler(struct i40iw_handler *hdl)
162 spin_lock_irqsave(&i40iw_handler_lock, flags);
163 list_del(&hdl->list);
164 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
169 * i40iw_enable_intr - set up device interrupts
170 * @dev: hardware control device structure
171 * @msix_id: id of the interrupt to be enabled
173 static void i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id)
177 val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
178 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
179 (3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
181 i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_id - 1), val);
183 i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_id - 1), val);
187 * i40iw_dpc - tasklet for aeq and ceq 0
188 * @data: iwarp device
190 static void i40iw_dpc(unsigned long data)
192 struct i40iw_device *iwdev = (struct i40iw_device *)data;
194 if (iwdev->msix_shared)
195 i40iw_process_ceq(iwdev, iwdev->ceqlist);
196 i40iw_process_aeq(iwdev);
197 i40iw_enable_intr(&iwdev->sc_dev, iwdev->iw_msixtbl[0].idx);
201 * i40iw_ceq_dpc - dpc handler for CEQ
202 * @data: data points to CEQ
204 static void i40iw_ceq_dpc(unsigned long data)
206 struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
207 struct i40iw_device *iwdev = iwceq->iwdev;
209 i40iw_process_ceq(iwdev, iwceq);
210 i40iw_enable_intr(&iwdev->sc_dev, iwceq->msix_idx);
214 * i40iw_irq_handler - interrupt handler for aeq and ceq0
215 * @irq: Interrupt request number
216 * @data: iwarp device
218 static irqreturn_t i40iw_irq_handler(int irq, void *data)
220 struct i40iw_device *iwdev = (struct i40iw_device *)data;
222 tasklet_schedule(&iwdev->dpc_tasklet);
227 * i40iw_destroy_cqp - destroy control qp
228 * @iwdev: iwarp device
229 * @create_done: 1 if cqp create poll was success
231 * Issue destroy cqp request and
232 * free the resources associated with the cqp
234 static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp)
236 enum i40iw_status_code status = 0;
237 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
238 struct i40iw_cqp *cqp = &iwdev->cqp;
240 if (free_hwcqp && dev->cqp_ops->cqp_destroy)
241 status = dev->cqp_ops->cqp_destroy(dev->cqp);
243 i40iw_pr_err("destroy cqp failed");
245 i40iw_free_dma_mem(dev->hw, &cqp->sq);
246 kfree(cqp->scratch_array);
247 iwdev->cqp.scratch_array = NULL;
249 kfree(cqp->cqp_requests);
250 cqp->cqp_requests = NULL;
254 * i40iw_disable_irqs - disable device interrupts
255 * @dev: hardware control device structure
256 * @msic_vec: msix vector to disable irq
257 * @dev_id: parameter to pass to free_irq (used during irq setup)
259 * The function is called when destroying aeq/ceq
261 static void i40iw_disable_irq(struct i40iw_sc_dev *dev,
262 struct i40iw_msix_vector *msix_vec,
266 i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0);
268 i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_vec->idx - 1), 0);
269 free_irq(msix_vec->irq, dev_id);
273 * i40iw_destroy_aeq - destroy aeq
274 * @iwdev: iwarp device
275 * @reset: true if called before reset
277 * Issue a destroy aeq request and
278 * free the resources associated with the aeq
279 * The function is called during driver unload
281 static void i40iw_destroy_aeq(struct i40iw_device *iwdev, bool reset)
283 enum i40iw_status_code status = I40IW_ERR_NOT_READY;
284 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
285 struct i40iw_aeq *aeq = &iwdev->aeq;
287 if (!iwdev->msix_shared)
288 i40iw_disable_irq(dev, iwdev->iw_msixtbl, (void *)iwdev);
292 if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1))
293 status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq);
295 i40iw_pr_err("destroy aeq failed %d\n", status);
298 i40iw_free_dma_mem(dev->hw, &aeq->mem);
302 * i40iw_destroy_ceq - destroy ceq
303 * @iwdev: iwarp device
304 * @iwceq: ceq to be destroyed
305 * @reset: true if called before reset
307 * Issue a destroy ceq request and
308 * free the resources associated with the ceq
310 static void i40iw_destroy_ceq(struct i40iw_device *iwdev,
311 struct i40iw_ceq *iwceq,
314 enum i40iw_status_code status;
315 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
320 status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1);
322 i40iw_pr_err("ceq destroy command failed %d\n", status);
326 status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq);
328 i40iw_pr_err("ceq destroy completion failed %d\n", status);
330 i40iw_free_dma_mem(dev->hw, &iwceq->mem);
334 * i40iw_dele_ceqs - destroy all ceq's
335 * @iwdev: iwarp device
336 * @reset: true if called before reset
338 * Go through all of the device ceq's and for each ceq
339 * disable the ceq interrupt and destroy the ceq
341 static void i40iw_dele_ceqs(struct i40iw_device *iwdev, bool reset)
344 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
345 struct i40iw_ceq *iwceq = iwdev->ceqlist;
346 struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
348 if (iwdev->msix_shared) {
349 i40iw_disable_irq(dev, msix_vec, (void *)iwdev);
350 i40iw_destroy_ceq(iwdev, iwceq, reset);
355 for (msix_vec++; i < iwdev->ceqs_count; i++, msix_vec++, iwceq++) {
356 i40iw_disable_irq(dev, msix_vec, (void *)iwceq);
357 i40iw_destroy_ceq(iwdev, iwceq, reset);
362 * i40iw_destroy_ccq - destroy control cq
363 * @iwdev: iwarp device
364 * @reset: true if called before reset
366 * Issue destroy ccq request and
367 * free the resources associated with the ccq
369 static void i40iw_destroy_ccq(struct i40iw_device *iwdev, bool reset)
371 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
372 struct i40iw_ccq *ccq = &iwdev->ccq;
373 enum i40iw_status_code status = 0;
376 status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, true);
378 i40iw_pr_err("ccq destroy failed %d\n", status);
379 i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
382 /* types of hmc objects */
383 static enum i40iw_hmc_rsrc_type iw_hmc_obj_types[] = {
388 I40IW_HMC_IW_APBVT_ENTRY,
398 * i40iw_close_hmc_objects_type - delete hmc objects of a given type
399 * @iwdev: iwarp device
400 * @obj_type: the hmc object type to be deleted
401 * @is_pf: true if the function is PF otherwise false
402 * @reset: true if called before reset
404 static void i40iw_close_hmc_objects_type(struct i40iw_sc_dev *dev,
405 enum i40iw_hmc_rsrc_type obj_type,
406 struct i40iw_hmc_info *hmc_info,
410 struct i40iw_hmc_del_obj_info info;
412 memset(&info, 0, sizeof(info));
413 info.hmc_info = hmc_info;
414 info.rsrc_type = obj_type;
415 info.count = hmc_info->hmc_obj[obj_type].cnt;
417 if (dev->hmc_ops->del_hmc_object(dev, &info, reset))
418 i40iw_pr_err("del obj of type %d failed\n", obj_type);
422 * i40iw_del_hmc_objects - remove all device hmc objects
424 * @hmc_info: hmc_info to free
425 * @is_pf: true if hmc_info belongs to PF, not vf nor allocated
426 * by PF on behalf of VF
427 * @reset: true if called before reset
429 static void i40iw_del_hmc_objects(struct i40iw_sc_dev *dev,
430 struct i40iw_hmc_info *hmc_info,
436 for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++)
437 i40iw_close_hmc_objects_type(dev, iw_hmc_obj_types[i], hmc_info, is_pf, reset);
441 * i40iw_ceq_handler - interrupt handler for ceq
444 static irqreturn_t i40iw_ceq_handler(int irq, void *data)
446 struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
448 if (iwceq->irq != irq)
449 i40iw_pr_err("expected irq = %d received irq = %d\n", iwceq->irq, irq);
450 tasklet_schedule(&iwceq->dpc_tasklet);
455 * i40iw_create_hmc_obj_type - create hmc object of a given type
456 * @dev: hardware control device structure
457 * @info: information for the hmc object to create
459 static enum i40iw_status_code i40iw_create_hmc_obj_type(struct i40iw_sc_dev *dev,
460 struct i40iw_hmc_create_obj_info *info)
462 return dev->hmc_ops->create_hmc_object(dev, info);
466 * i40iw_create_hmc_objs - create all hmc objects for the device
467 * @iwdev: iwarp device
468 * @is_pf: true if the function is PF otherwise false
470 * Create the device hmc objects and allocate hmc pages
471 * Return 0 if successful, otherwise clean up and return error
473 static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev,
476 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
477 struct i40iw_hmc_create_obj_info info;
478 enum i40iw_status_code status;
481 memset(&info, 0, sizeof(info));
482 info.hmc_info = dev->hmc_info;
484 info.entry_type = iwdev->sd_type;
485 for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
486 info.rsrc_type = iw_hmc_obj_types[i];
487 info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
488 status = i40iw_create_hmc_obj_type(dev, &info);
490 i40iw_pr_err("create obj type %d status = %d\n",
491 iw_hmc_obj_types[i], status);
496 return (dev->cqp_misc_ops->static_hmc_pages_allocated(dev->cqp, 0,
502 /* destroy the hmc objects of a given type */
503 i40iw_close_hmc_objects_type(dev,
513 * i40iw_obj_aligned_mem - get aligned memory from device allocated memory
514 * @iwdev: iwarp device
515 * @memptr: points to the memory addresses
516 * @size: size of memory needed
517 * @mask: mask for the aligned memory
519 * Get aligned memory of the requested size and
520 * update the memptr to point to the new aligned memory
521 * Return 0 if successful, otherwise return no memory error
523 enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,
524 struct i40iw_dma_mem *memptr,
528 unsigned long va, newva;
531 va = (unsigned long)iwdev->obj_next.va;
534 newva = ALIGN(va, (mask + 1));
536 memptr->va = (u8 *)va + extra;
537 memptr->pa = iwdev->obj_next.pa + extra;
539 if ((memptr->va + size) > (iwdev->obj_mem.va + iwdev->obj_mem.size))
540 return I40IW_ERR_NO_MEMORY;
542 iwdev->obj_next.va = memptr->va + size;
543 iwdev->obj_next.pa = memptr->pa + size;
548 * i40iw_create_cqp - create control qp
549 * @iwdev: iwarp device
551 * Return 0, if the cqp and all the resources associated with it
552 * are successfully created, otherwise return error
554 static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)
556 enum i40iw_status_code status;
557 u32 sqsize = I40IW_CQP_SW_SQSIZE_2048;
558 struct i40iw_dma_mem mem;
559 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
560 struct i40iw_cqp_init_info cqp_init_info;
561 struct i40iw_cqp *cqp = &iwdev->cqp;
562 u16 maj_err, min_err;
565 cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
566 if (!cqp->cqp_requests)
567 return I40IW_ERR_NO_MEMORY;
568 cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
569 if (!cqp->scratch_array) {
570 kfree(cqp->cqp_requests);
571 return I40IW_ERR_NO_MEMORY;
573 dev->cqp = &cqp->sc_cqp;
575 memset(&cqp_init_info, 0, sizeof(cqp_init_info));
576 status = i40iw_allocate_dma_mem(dev->hw, &cqp->sq,
577 (sizeof(struct i40iw_cqp_sq_wqe) * sqsize),
578 I40IW_CQP_ALIGNMENT);
581 status = i40iw_obj_aligned_mem(iwdev, &mem, sizeof(struct i40iw_cqp_ctx),
582 I40IW_HOST_CTX_ALIGNMENT_MASK);
585 dev->cqp->host_ctx_pa = mem.pa;
586 dev->cqp->host_ctx = mem.va;
587 /* populate the cqp init info */
588 cqp_init_info.dev = dev;
589 cqp_init_info.sq_size = sqsize;
590 cqp_init_info.sq = cqp->sq.va;
591 cqp_init_info.sq_pa = cqp->sq.pa;
592 cqp_init_info.host_ctx_pa = mem.pa;
593 cqp_init_info.host_ctx = mem.va;
594 cqp_init_info.hmc_profile = iwdev->resource_profile;
595 cqp_init_info.enabled_vf_count = iwdev->max_rdma_vfs;
596 cqp_init_info.scratch_array = cqp->scratch_array;
597 status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info);
599 i40iw_pr_err("cqp init status %d\n", status);
602 status = dev->cqp_ops->cqp_create(dev->cqp, true, &maj_err, &min_err);
604 i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",
605 status, maj_err, min_err);
608 spin_lock_init(&cqp->req_lock);
609 INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
610 INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
611 /* init the waitq of the cqp_requests and add them to the list */
612 for (i = 0; i < I40IW_CQP_SW_SQSIZE_2048; i++) {
613 init_waitqueue_head(&cqp->cqp_requests[i].waitq);
614 list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
618 /* clean up the created resources */
619 i40iw_destroy_cqp(iwdev, false);
624 * i40iw_create_ccq - create control cq
625 * @iwdev: iwarp device
627 * Return 0, if the ccq and the resources associated with it
628 * are successfully created, otherwise return error
630 static enum i40iw_status_code i40iw_create_ccq(struct i40iw_device *iwdev)
632 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
633 struct i40iw_dma_mem mem;
634 enum i40iw_status_code status;
635 struct i40iw_ccq_init_info info;
636 struct i40iw_ccq *ccq = &iwdev->ccq;
638 memset(&info, 0, sizeof(info));
639 dev->ccq = &ccq->sc_cq;
642 ccq->shadow_area.size = sizeof(struct i40iw_cq_shadow_area);
643 ccq->mem_cq.size = sizeof(struct i40iw_cqe) * IW_CCQ_SIZE;
644 status = i40iw_allocate_dma_mem(dev->hw, &ccq->mem_cq,
645 ccq->mem_cq.size, I40IW_CQ0_ALIGNMENT);
648 status = i40iw_obj_aligned_mem(iwdev, &mem, ccq->shadow_area.size,
649 I40IW_SHADOWAREA_MASK);
652 ccq->sc_cq.back_cq = (void *)ccq;
653 /* populate the ccq init info */
654 info.cq_base = ccq->mem_cq.va;
655 info.cq_pa = ccq->mem_cq.pa;
656 info.num_elem = IW_CCQ_SIZE;
657 info.shadow_area = mem.va;
658 info.shadow_area_pa = mem.pa;
659 info.ceqe_mask = false;
660 info.ceq_id_valid = true;
661 info.shadow_read_threshold = 16;
662 status = dev->ccq_ops->ccq_init(dev->ccq, &info);
664 status = dev->ccq_ops->ccq_create(dev->ccq, 0, true, true);
667 i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
672 * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq
673 * @iwdev: iwarp device
674 * @msix_vec: interrupt vector information
675 * @iwceq: ceq associated with the vector
676 * @ceq_id: the id number of the iwceq
678 * Allocate interrupt resources and enable irq handling
679 * Return 0 if successful, otherwise return error
681 static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iwdev,
682 struct i40iw_ceq *iwceq,
684 struct i40iw_msix_vector *msix_vec)
686 enum i40iw_status_code status;
688 if (iwdev->msix_shared && !ceq_id) {
689 tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
690 status = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "AEQCEQ", iwdev);
692 tasklet_init(&iwceq->dpc_tasklet, i40iw_ceq_dpc, (unsigned long)iwceq);
693 status = request_irq(msix_vec->irq, i40iw_ceq_handler, 0, "CEQ", iwceq);
697 i40iw_pr_err("ceq irq config fail\n");
698 return I40IW_ERR_CONFIG;
700 msix_vec->ceq_id = ceq_id;
701 msix_vec->cpu_affinity = 0;
707 * i40iw_create_ceq - create completion event queue
708 * @iwdev: iwarp device
709 * @iwceq: pointer to the ceq resources to be created
710 * @ceq_id: the id number of the iwceq
712 * Return 0, if the ceq and the resources associated with it
713 * are successfully created, otherwise return error
715 static enum i40iw_status_code i40iw_create_ceq(struct i40iw_device *iwdev,
716 struct i40iw_ceq *iwceq,
719 enum i40iw_status_code status;
720 struct i40iw_ceq_init_info info;
721 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
724 memset(&info, 0, sizeof(info));
725 info.ceq_id = ceq_id;
726 iwceq->iwdev = iwdev;
727 iwceq->mem.size = sizeof(struct i40iw_ceqe) *
728 iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
729 status = i40iw_allocate_dma_mem(dev->hw, &iwceq->mem, iwceq->mem.size,
730 I40IW_CEQ_ALIGNMENT);
733 info.ceq_id = ceq_id;
734 info.ceqe_base = iwceq->mem.va;
735 info.ceqe_pa = iwceq->mem.pa;
737 info.elem_cnt = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
738 iwceq->sc_ceq.ceq_id = ceq_id;
740 scratch = (uintptr_t)&iwdev->cqp.sc_cqp;
741 status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info);
743 status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq, scratch);
747 i40iw_free_dma_mem(dev->hw, &iwceq->mem);
751 void i40iw_request_reset(struct i40iw_device *iwdev)
753 struct i40e_info *ldev = iwdev->ldev;
755 ldev->ops->request_reset(ldev, iwdev->client, 1);
759 * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources
760 * @iwdev: iwarp device
761 * @ldev: i40e lan device
763 * Allocate a list for all device completion event queues
764 * Create the ceq's and configure their msix interrupt vectors
765 * Return 0, if at least one ceq is successfully set up, otherwise return error
767 static enum i40iw_status_code i40iw_setup_ceqs(struct i40iw_device *iwdev,
768 struct i40e_info *ldev)
772 struct i40iw_ceq *iwceq;
773 struct i40iw_msix_vector *msix_vec;
774 enum i40iw_status_code status = 0;
777 if (ldev && ldev->ops && ldev->ops->setup_qvlist) {
778 status = ldev->ops->setup_qvlist(ldev, &i40iw_client,
783 status = I40IW_ERR_BAD_PTR;
787 num_ceqs = min(iwdev->msix_count, iwdev->sc_dev.hmc_fpm_misc.max_ceqs);
788 iwdev->ceqlist = kcalloc(num_ceqs, sizeof(*iwdev->ceqlist), GFP_KERNEL);
789 if (!iwdev->ceqlist) {
790 status = I40IW_ERR_NO_MEMORY;
793 i = (iwdev->msix_shared) ? 0 : 1;
794 for (ceq_id = 0; i < num_ceqs; i++, ceq_id++) {
795 iwceq = &iwdev->ceqlist[ceq_id];
796 status = i40iw_create_ceq(iwdev, iwceq, ceq_id);
798 i40iw_pr_err("create ceq status = %d\n", status);
802 msix_vec = &iwdev->iw_msixtbl[i];
803 iwceq->irq = msix_vec->irq;
804 iwceq->msix_idx = msix_vec->idx;
805 status = i40iw_configure_ceq_vector(iwdev, iwceq, ceq_id, msix_vec);
807 i40iw_destroy_ceq(iwdev, iwceq, false);
810 i40iw_enable_intr(&iwdev->sc_dev, msix_vec->idx);
816 if (!iwdev->ceqs_count) {
817 kfree(iwdev->ceqlist);
818 iwdev->ceqlist = NULL;
827 * i40iw_configure_aeq_vector - set up the msix vector for aeq
828 * @iwdev: iwarp device
830 * Allocate interrupt resources and enable irq handling
831 * Return 0 if successful, otherwise return error
833 static enum i40iw_status_code i40iw_configure_aeq_vector(struct i40iw_device *iwdev)
835 struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
838 if (!iwdev->msix_shared) {
839 tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
840 ret = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "i40iw", iwdev);
843 i40iw_pr_err("aeq irq config fail\n");
844 return I40IW_ERR_CONFIG;
851 * i40iw_create_aeq - create async event queue
852 * @iwdev: iwarp device
854 * Return 0, if the aeq and the resources associated with it
855 * are successfully created, otherwise return error
857 static enum i40iw_status_code i40iw_create_aeq(struct i40iw_device *iwdev)
859 enum i40iw_status_code status;
860 struct i40iw_aeq_init_info info;
861 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
862 struct i40iw_aeq *aeq = &iwdev->aeq;
866 aeq_size = 2 * iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt +
867 iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
868 memset(&info, 0, sizeof(info));
869 aeq->mem.size = sizeof(struct i40iw_sc_aeqe) * aeq_size;
870 status = i40iw_allocate_dma_mem(dev->hw, &aeq->mem, aeq->mem.size,
871 I40IW_AEQ_ALIGNMENT);
875 info.aeqe_base = aeq->mem.va;
876 info.aeq_elem_pa = aeq->mem.pa;
877 info.elem_cnt = aeq_size;
879 status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info);
882 status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1);
884 status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq);
887 i40iw_free_dma_mem(dev->hw, &aeq->mem);
892 * i40iw_setup_aeq - set up the device aeq
893 * @iwdev: iwarp device
895 * Create the aeq and configure its msix interrupt vector
896 * Return 0 if successful, otherwise return error
898 static enum i40iw_status_code i40iw_setup_aeq(struct i40iw_device *iwdev)
900 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
901 enum i40iw_status_code status;
903 status = i40iw_create_aeq(iwdev);
907 status = i40iw_configure_aeq_vector(iwdev);
909 i40iw_destroy_aeq(iwdev, false);
913 if (!iwdev->msix_shared)
914 i40iw_enable_intr(dev, iwdev->iw_msixtbl[0].idx);
919 * i40iw_initialize_ilq - create iwarp local queue for cm
920 * @iwdev: iwarp device
922 * Return 0 if successful, otherwise return error
924 static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev)
926 struct i40iw_puda_rsrc_info info;
927 enum i40iw_status_code status;
929 info.type = I40IW_PUDA_RSRC_TYPE_ILQ;
936 info.buf_size = 1024;
937 info.tx_buf_cnt = 16384;
938 info.mss = iwdev->mss;
939 info.receive = i40iw_receive_ilq;
940 info.xmit_complete = i40iw_free_sqbuf;
941 status = i40iw_puda_create_rsrc(&iwdev->sc_dev, &info);
943 i40iw_pr_err("ilq create fail\n");
948 * i40iw_initialize_ieq - create iwarp exception queue
949 * @iwdev: iwarp device
951 * Return 0 if successful, otherwise return error
953 static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev)
955 struct i40iw_puda_rsrc_info info;
956 enum i40iw_status_code status;
958 info.type = I40IW_PUDA_RSRC_TYPE_IEQ;
960 info.qp_id = iwdev->sc_dev.exception_lan_queue;
965 info.buf_size = 2048;
966 info.mss = iwdev->mss;
967 info.tx_buf_cnt = 16384;
968 status = i40iw_puda_create_rsrc(&iwdev->sc_dev, &info);
970 i40iw_pr_err("ieq create fail\n");
975 * i40iw_hmc_setup - create hmc objects for the device
976 * @iwdev: iwarp device
978 * Set up the device private memory space for the number and size of
979 * the hmc objects and create the objects
980 * Return 0 if successful, otherwise return error
982 static enum i40iw_status_code i40iw_hmc_setup(struct i40iw_device *iwdev)
984 enum i40iw_status_code status;
986 iwdev->sd_type = I40IW_SD_TYPE_DIRECT;
987 status = i40iw_config_fpm_values(&iwdev->sc_dev, IW_CFG_FPM_QP_COUNT);
990 status = i40iw_create_hmc_objs(iwdev, true);
993 iwdev->init_state = HMC_OBJS_CREATED;
999 * i40iw_del_init_mem - deallocate memory resources
1000 * @iwdev: iwarp device
1002 static void i40iw_del_init_mem(struct i40iw_device *iwdev)
1004 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1006 i40iw_free_dma_mem(&iwdev->hw, &iwdev->obj_mem);
1007 kfree(dev->hmc_info->sd_table.sd_entry);
1008 dev->hmc_info->sd_table.sd_entry = NULL;
1009 kfree(iwdev->mem_resources);
1010 iwdev->mem_resources = NULL;
1011 kfree(iwdev->ceqlist);
1012 iwdev->ceqlist = NULL;
1013 kfree(iwdev->iw_msixtbl);
1014 iwdev->iw_msixtbl = NULL;
1015 kfree(iwdev->hmc_info_mem);
1016 iwdev->hmc_info_mem = NULL;
1020 * i40iw_del_macip_entry - remove a mac ip address entry from the hw table
1021 * @iwdev: iwarp device
1022 * @idx: the index of the mac ip address to delete
1024 static void i40iw_del_macip_entry(struct i40iw_device *iwdev, u8 idx)
1026 struct i40iw_cqp *iwcqp = &iwdev->cqp;
1027 struct i40iw_cqp_request *cqp_request;
1028 struct cqp_commands_info *cqp_info;
1029 enum i40iw_status_code status = 0;
1031 cqp_request = i40iw_get_cqp_request(iwcqp, true);
1033 i40iw_pr_err("cqp_request memory failed\n");
1036 cqp_info = &cqp_request->info;
1037 cqp_info->cqp_cmd = OP_DELETE_LOCAL_MAC_IPADDR_ENTRY;
1038 cqp_info->post_sq = 1;
1039 cqp_info->in.u.del_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1040 cqp_info->in.u.del_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1041 cqp_info->in.u.del_local_mac_ipaddr_entry.entry_idx = idx;
1042 cqp_info->in.u.del_local_mac_ipaddr_entry.ignore_ref_count = 0;
1043 status = i40iw_handle_cqp_op(iwdev, cqp_request);
1045 i40iw_pr_err("CQP-OP Del MAC Ip entry fail");
1049 * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table
1050 * @iwdev: iwarp device
1051 * @mac_addr: pointer to mac address
1052 * @idx: the index of the mac ip address to add
1054 static enum i40iw_status_code i40iw_add_mac_ipaddr_entry(struct i40iw_device *iwdev,
1058 struct i40iw_local_mac_ipaddr_entry_info *info;
1059 struct i40iw_cqp *iwcqp = &iwdev->cqp;
1060 struct i40iw_cqp_request *cqp_request;
1061 struct cqp_commands_info *cqp_info;
1062 enum i40iw_status_code status = 0;
1064 cqp_request = i40iw_get_cqp_request(iwcqp, true);
1066 i40iw_pr_err("cqp_request memory failed\n");
1067 return I40IW_ERR_NO_MEMORY;
1070 cqp_info = &cqp_request->info;
1072 cqp_info->post_sq = 1;
1073 info = &cqp_info->in.u.add_local_mac_ipaddr_entry.info;
1074 ether_addr_copy(info->mac_addr, mac_addr);
1075 info->entry_idx = idx;
1076 cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1077 cqp_info->cqp_cmd = OP_ADD_LOCAL_MAC_IPADDR_ENTRY;
1078 cqp_info->in.u.add_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1079 cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1080 status = i40iw_handle_cqp_op(iwdev, cqp_request);
1082 i40iw_pr_err("CQP-OP Add MAC Ip entry fail");
1087 * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry
1088 * @iwdev: iwarp device
1089 * @mac_ip_tbl_idx: the index of the new mac ip address
1091 * Allocate a mac ip address entry and update the mac_ip_tbl_idx
1092 * to hold the index of the newly created mac ip address
1093 * Return 0 if successful, otherwise return error
1095 static enum i40iw_status_code i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device *iwdev,
1096 u16 *mac_ip_tbl_idx)
1098 struct i40iw_cqp *iwcqp = &iwdev->cqp;
1099 struct i40iw_cqp_request *cqp_request;
1100 struct cqp_commands_info *cqp_info;
1101 enum i40iw_status_code status = 0;
1103 cqp_request = i40iw_get_cqp_request(iwcqp, true);
1105 i40iw_pr_err("cqp_request memory failed\n");
1106 return I40IW_ERR_NO_MEMORY;
1109 /* increment refcount, because we need the cqp request ret value */
1110 atomic_inc(&cqp_request->refcount);
1112 cqp_info = &cqp_request->info;
1113 cqp_info->cqp_cmd = OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY;
1114 cqp_info->post_sq = 1;
1115 cqp_info->in.u.alloc_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1116 cqp_info->in.u.alloc_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1117 status = i40iw_handle_cqp_op(iwdev, cqp_request);
1119 *mac_ip_tbl_idx = cqp_request->compl_info.op_ret_val;
1121 i40iw_pr_err("CQP-OP Alloc MAC Ip entry fail");
1122 /* decrement refcount and free the cqp request, if no longer used */
1123 i40iw_put_cqp_request(iwcqp, cqp_request);
1128 * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry
1129 * @iwdev: iwarp device
1130 * @macaddr: pointer to mac address
1132 * Allocate a mac ip address entry and add it to the hw table
1133 * Return 0 if successful, otherwise return error
1135 static enum i40iw_status_code i40iw_alloc_set_mac_ipaddr(struct i40iw_device *iwdev,
1138 enum i40iw_status_code status;
1140 status = i40iw_alloc_local_mac_ipaddr_entry(iwdev, &iwdev->mac_ip_table_idx);
1142 status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr,
1143 (u8)iwdev->mac_ip_table_idx);
1145 i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
1151 * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table
1152 * @iwdev: iwarp device
1154 static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev)
1156 struct net_device *ip_dev;
1157 struct inet6_dev *idev;
1158 struct inet6_ifaddr *ifp;
1159 u32 local_ipaddr6[4];
1162 for_each_netdev_rcu(&init_net, ip_dev) {
1163 if ((((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF) &&
1164 (rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev)) ||
1165 (ip_dev == iwdev->netdev)) && (ip_dev->flags & IFF_UP)) {
1166 idev = __in6_dev_get(ip_dev);
1168 i40iw_pr_err("ipv6 inet device not found\n");
1171 list_for_each_entry(ifp, &idev->addr_list, if_list) {
1172 i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &ifp->addr,
1173 rdma_vlan_dev_vlan_id(ip_dev), ip_dev->dev_addr);
1174 i40iw_copy_ip_ntohl(local_ipaddr6,
1175 ifp->addr.in6_u.u6_addr32);
1176 i40iw_manage_arp_cache(iwdev,
1188 * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table
1189 * @iwdev: iwarp device
1191 static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev)
1193 struct net_device *dev;
1194 struct in_device *idev;
1195 bool got_lock = true;
1198 if (!rtnl_trylock())
1201 for_each_netdev(&init_net, dev) {
1202 if ((((rdma_vlan_dev_vlan_id(dev) < 0xFFFF) &&
1203 (rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) ||
1204 (dev == iwdev->netdev)) && (dev->flags & IFF_UP)) {
1205 idev = in_dev_get(dev);
1207 i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
1208 "IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_address,
1209 rdma_vlan_dev_vlan_id(dev), dev->dev_addr);
1211 ip_addr = ntohl(ifa->ifa_address);
1212 i40iw_manage_arp_cache(iwdev,
1227 * i40iw_add_mac_ip - add mac and ip addresses
1228 * @iwdev: iwarp device
1230 * Create and add a mac ip address entry to the hw table and
1231 * ipv4/ipv6 addresses to the arp cache
1232 * Return 0 if successful, otherwise return error
1234 static enum i40iw_status_code i40iw_add_mac_ip(struct i40iw_device *iwdev)
1236 struct net_device *netdev = iwdev->netdev;
1237 enum i40iw_status_code status;
1239 status = i40iw_alloc_set_mac_ipaddr(iwdev, (u8 *)netdev->dev_addr);
1242 i40iw_add_ipv4_addr(iwdev);
1243 i40iw_add_ipv6_addr(iwdev);
1248 * i40iw_wait_pe_ready - Check if firmware is ready
1249 * @hw: provides access to registers
1251 static void i40iw_wait_pe_ready(struct i40iw_hw *hw)
1260 statusfw = i40iw_rd32(hw, I40E_GLPE_FWLDSTATUS);
1261 i40iw_pr_info("[%04d] fm load status[x%04X]\n", __LINE__, statusfw);
1262 statuscpu0 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS0);
1263 i40iw_pr_info("[%04d] CSR_CQP status[x%04X]\n", __LINE__, statuscpu0);
1264 statuscpu1 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS1);
1265 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS1 status[x%04X]\n",
1266 __LINE__, statuscpu1);
1267 statuscpu2 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS2);
1268 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS2 status[x%04X]\n",
1269 __LINE__, statuscpu2);
1270 if ((statuscpu0 == 0x80) && (statuscpu1 == 0x80) && (statuscpu2 == 0x80))
1271 break; /* SUCCESS */
1274 } while (retrycount < 14);
1275 i40iw_wr32(hw, 0xb4040, 0x4C104C5);
1279 * i40iw_initialize_dev - initialize device
1280 * @iwdev: iwarp device
1281 * @ldev: lan device information
1283 * Allocate memory for the hmc objects and initialize iwdev
1284 * Return 0 if successful, otherwise clean up the resources
1287 static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev,
1288 struct i40e_info *ldev)
1290 enum i40iw_status_code status;
1291 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1292 struct i40iw_device_init_info info;
1293 struct i40iw_dma_mem mem;
1296 memset(&info, 0, sizeof(info));
1297 size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) +
1298 (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX);
1299 iwdev->hmc_info_mem = kzalloc(size, GFP_KERNEL);
1300 if (!iwdev->hmc_info_mem) {
1301 i40iw_pr_err("memory alloc fail\n");
1302 return I40IW_ERR_NO_MEMORY;
1304 iwdev->pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev->hmc_info_mem;
1305 dev->hmc_info = &iwdev->hw.hmc;
1306 dev->hmc_info->hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev->pble_rsrc + 1);
1307 status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_QUERY_FPM_BUF_SIZE,
1308 I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);
1311 info.fpm_query_buf_pa = mem.pa;
1312 info.fpm_query_buf = mem.va;
1313 status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_COMMIT_FPM_BUF_SIZE,
1314 I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK);
1317 info.fpm_commit_buf_pa = mem.pa;
1318 info.fpm_commit_buf = mem.va;
1319 info.hmc_fn_id = ldev->fid;
1320 info.is_pf = (ldev->ftype) ? false : true;
1321 info.bar0 = ldev->hw_addr;
1322 info.hw = &iwdev->hw;
1323 info.debug_mask = debug;
1324 info.qs_handle = ldev->params.qos.prio_qos[0].qs_handle;
1325 info.exception_lan_queue = 1;
1326 info.vchnl_send = i40iw_virtchnl_send;
1327 status = i40iw_device_init(&iwdev->sc_dev, &info);
1330 kfree(iwdev->hmc_info_mem);
1331 iwdev->hmc_info_mem = NULL;
1337 * i40iw_register_notifiers - register tcp ip notifiers
1339 static void i40iw_register_notifiers(void)
1341 if (atomic_inc_return(&i40iw_notifiers_registered) == 1) {
1342 register_inetaddr_notifier(&i40iw_inetaddr_notifier);
1343 register_inet6addr_notifier(&i40iw_inetaddr6_notifier);
1344 register_netevent_notifier(&i40iw_net_notifier);
1349 * i40iw_save_msix_info - copy msix vector information to iwarp device
1350 * @iwdev: iwarp device
1351 * @ldev: lan device information
1353 * Allocate iwdev msix table and copy the ldev msix info to the table
1354 * Return 0 if successful, otherwise return error
1356 static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev,
1357 struct i40e_info *ldev)
1359 struct i40e_qvlist_info *iw_qvlist;
1360 struct i40e_qv_info *iw_qvinfo;
1365 iwdev->msix_count = ldev->msix_count;
1367 size = sizeof(struct i40iw_msix_vector) * iwdev->msix_count;
1368 size += sizeof(struct i40e_qvlist_info);
1369 size += sizeof(struct i40e_qv_info) * iwdev->msix_count - 1;
1370 iwdev->iw_msixtbl = kzalloc(size, GFP_KERNEL);
1372 if (!iwdev->iw_msixtbl)
1373 return I40IW_ERR_NO_MEMORY;
1374 iwdev->iw_qvlist = (struct i40e_qvlist_info *)(&iwdev->iw_msixtbl[iwdev->msix_count]);
1375 iw_qvlist = iwdev->iw_qvlist;
1376 iw_qvinfo = iw_qvlist->qv_info;
1377 iw_qvlist->num_vectors = iwdev->msix_count;
1378 if (iwdev->msix_count <= num_online_cpus())
1379 iwdev->msix_shared = true;
1380 for (i = 0, ceq_idx = 0; i < iwdev->msix_count; i++, iw_qvinfo++) {
1381 iwdev->iw_msixtbl[i].idx = ldev->msix_entries[i].entry;
1382 iwdev->iw_msixtbl[i].irq = ldev->msix_entries[i].vector;
1384 iw_qvinfo->aeq_idx = 0;
1385 if (iwdev->msix_shared)
1386 iw_qvinfo->ceq_idx = ceq_idx++;
1388 iw_qvinfo->ceq_idx = I40E_QUEUE_INVALID_IDX;
1390 iw_qvinfo->aeq_idx = I40E_QUEUE_INVALID_IDX;
1391 iw_qvinfo->ceq_idx = ceq_idx++;
1393 iw_qvinfo->itr_idx = 3;
1394 iw_qvinfo->v_idx = iwdev->iw_msixtbl[i].idx;
1400 * i40iw_deinit_device - clean up the device resources
1401 * @iwdev: iwarp device
1402 * @reset: true if called before reset
1403 * @del_hdl: true if delete hdl entry
1405 * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses,
1406 * destroy the device queues and free the pble and the hmc objects
1408 static void i40iw_deinit_device(struct i40iw_device *iwdev, bool reset, bool del_hdl)
1410 struct i40e_info *ldev = iwdev->ldev;
1412 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1414 i40iw_pr_info("state = %d\n", iwdev->init_state);
1416 switch (iwdev->init_state) {
1417 case RDMA_DEV_REGISTERED:
1418 iwdev->iw_status = 0;
1419 i40iw_port_ibevent(iwdev);
1420 i40iw_destroy_rdma_device(iwdev->iwibdev);
1422 case IP_ADDR_REGISTERED:
1424 i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
1427 if (!atomic_dec_return(&i40iw_notifiers_registered)) {
1428 unregister_netevent_notifier(&i40iw_net_notifier);
1429 unregister_inetaddr_notifier(&i40iw_inetaddr_notifier);
1430 unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier);
1434 i40iw_dele_ceqs(iwdev, reset);
1437 i40iw_destroy_aeq(iwdev, reset);
1440 i40iw_puda_dele_resources(dev, I40IW_PUDA_RSRC_TYPE_IEQ, reset);
1443 i40iw_puda_dele_resources(dev, I40IW_PUDA_RSRC_TYPE_ILQ, reset);
1446 i40iw_destroy_ccq(iwdev, reset);
1448 case PBLE_CHUNK_MEM:
1449 i40iw_destroy_pble_pool(dev, iwdev->pble_rsrc);
1451 case HMC_OBJS_CREATED:
1452 i40iw_del_hmc_objects(dev, dev->hmc_info, true, reset);
1455 i40iw_destroy_cqp(iwdev, !reset);
1458 i40iw_cleanup_cm_core(&iwdev->cm_core);
1460 i40iw_hw_stats_del_timer(dev);
1462 i40iw_del_init_mem(iwdev);
1467 i40iw_pr_err("bad init_state = %d\n", iwdev->init_state);
1472 i40iw_del_handler(i40iw_find_i40e_handler(ldev));
1477 * i40iw_setup_init_state - set up the initial device struct
1478 * @hdl: handler for iwarp device - one per instance
1479 * @ldev: lan device information
1480 * @client: iwarp client information, provided during registration
1482 * Initialize the iwarp device and its hdl information
1483 * using the ldev and client information
1484 * Return 0 if successful, otherwise return error
1486 static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl,
1487 struct i40e_info *ldev,
1488 struct i40e_client *client)
1490 struct i40iw_device *iwdev = &hdl->device;
1491 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1492 enum i40iw_status_code status;
1494 memcpy(&hdl->ldev, ldev, sizeof(*ldev));
1495 if (resource_profile == 1)
1496 resource_profile = 2;
1498 iwdev->mpa_version = mpa_version;
1499 iwdev->resource_profile = (resource_profile < I40IW_HMC_PROFILE_EQUAL) ?
1500 (u8)resource_profile + I40IW_HMC_PROFILE_DEFAULT :
1501 I40IW_HMC_PROFILE_DEFAULT;
1502 iwdev->max_rdma_vfs =
1503 (iwdev->resource_profile != I40IW_HMC_PROFILE_DEFAULT) ? max_rdma_vfs : 0;
1504 iwdev->max_enabled_vfs = iwdev->max_rdma_vfs;
1505 iwdev->netdev = ldev->netdev;
1506 hdl->client = client;
1507 iwdev->mss = (!ldev->params.mtu) ? I40IW_DEFAULT_MSS : ldev->params.mtu - I40IW_MTU_TO_MSS;
1509 iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_DB_ADDR_OFFSET;
1511 iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_VF_DB_ADDR_OFFSET;
1513 status = i40iw_save_msix_info(iwdev, ldev);
1516 iwdev->hw.dev_context = (void *)ldev->pcidev;
1517 iwdev->hw.hw_addr = ldev->hw_addr;
1518 status = i40iw_allocate_dma_mem(&iwdev->hw,
1519 &iwdev->obj_mem, 8192, 4096);
1522 iwdev->obj_next = iwdev->obj_mem;
1524 init_waitqueue_head(&iwdev->vchnl_waitq);
1525 init_waitqueue_head(&dev->vf_reqs);
1527 status = i40iw_initialize_dev(iwdev, ldev);
1530 kfree(iwdev->iw_msixtbl);
1531 i40iw_free_dma_mem(dev->hw, &iwdev->obj_mem);
1532 iwdev->iw_msixtbl = NULL;
1538 * i40iw_open - client interface operation open for iwarp/uda device
1539 * @ldev: lan device information
1540 * @client: iwarp client information, provided during registration
1542 * Called by the lan driver during the processing of client register
1543 * Create device resources, set up queues, pble and hmc objects and
1544 * register the device with the ib verbs interface
1545 * Return 0 if successful, otherwise return error
1547 static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
1549 struct i40iw_device *iwdev;
1550 struct i40iw_sc_dev *dev;
1551 enum i40iw_status_code status;
1552 struct i40iw_handler *hdl;
1554 hdl = i40iw_find_netdev(ldev->netdev);
1558 hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
1561 iwdev = &hdl->device;
1563 dev = &iwdev->sc_dev;
1564 i40iw_setup_cm_core(iwdev);
1566 dev->back_dev = (void *)iwdev;
1567 iwdev->ldev = &hdl->ldev;
1568 iwdev->client = client;
1569 mutex_init(&iwdev->pbl_mutex);
1570 i40iw_add_handler(hdl);
1573 status = i40iw_setup_init_state(hdl, ldev, client);
1576 iwdev->init_state = INITIAL_STATE;
1578 i40iw_wait_pe_ready(dev->hw);
1579 status = i40iw_create_cqp(iwdev);
1582 iwdev->init_state = CQP_CREATED;
1583 status = i40iw_hmc_setup(iwdev);
1586 status = i40iw_create_ccq(iwdev);
1589 iwdev->init_state = CCQ_CREATED;
1590 status = i40iw_initialize_ilq(iwdev);
1593 iwdev->init_state = ILQ_CREATED;
1594 status = i40iw_initialize_ieq(iwdev);
1597 iwdev->init_state = IEQ_CREATED;
1598 status = i40iw_setup_aeq(iwdev);
1601 iwdev->init_state = AEQ_CREATED;
1602 status = i40iw_setup_ceqs(iwdev, ldev);
1605 iwdev->init_state = CEQ_CREATED;
1606 status = i40iw_initialize_hw_resources(iwdev);
1609 dev->ccq_ops->ccq_arm(dev->ccq);
1610 status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc);
1613 iwdev->virtchnl_wq = alloc_ordered_workqueue("iwvch", WQ_MEM_RECLAIM);
1614 i40iw_register_notifiers();
1615 iwdev->init_state = INET_NOTIFIER;
1616 status = i40iw_add_mac_ip(iwdev);
1619 iwdev->init_state = IP_ADDR_REGISTERED;
1620 if (i40iw_register_rdma_device(iwdev)) {
1621 i40iw_pr_err("register rdma device fail\n");
1625 iwdev->init_state = RDMA_DEV_REGISTERED;
1626 iwdev->iw_status = 1;
1627 i40iw_port_ibevent(iwdev);
1628 i40iw_pr_info("i40iw_open completed\n");
1632 i40iw_pr_err("status = %d last completion = %d\n", status, iwdev->init_state);
1633 i40iw_deinit_device(iwdev, false, false);
1638 * i40iw_l2param_change : handle qs handles for qos and mss change
1639 * @ldev: lan device information
1640 * @client: client for paramater change
1641 * @params: new parameters from L2
1643 static void i40iw_l2param_change(struct i40e_info *ldev,
1644 struct i40e_client *client,
1645 struct i40e_params *params)
1647 struct i40iw_handler *hdl;
1648 struct i40iw_device *iwdev;
1650 hdl = i40iw_find_i40e_handler(ldev);
1654 iwdev = &hdl->device;
1656 iwdev->mss = params->mtu - I40IW_MTU_TO_MSS;
1660 * i40iw_close - client interface operation close for iwarp/uda device
1661 * @ldev: lan device information
1662 * @client: client to close
1664 * Called by the lan driver during the processing of client unregister
1665 * Destroy and clean up the driver resources
1667 static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool reset)
1669 struct i40iw_device *iwdev;
1670 struct i40iw_handler *hdl;
1672 hdl = i40iw_find_i40e_handler(ldev);
1676 iwdev = &hdl->device;
1677 destroy_workqueue(iwdev->virtchnl_wq);
1678 i40iw_deinit_device(iwdev, reset, true);
1682 * i40iw_vf_reset - process VF reset
1683 * @ldev: lan device information
1684 * @client: client interface instance
1685 * @vf_id: virtual function id
1687 * Called when a VF is reset by the PF
1688 * Destroy and clean up the VF resources
1690 static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u32 vf_id)
1692 struct i40iw_handler *hdl;
1693 struct i40iw_sc_dev *dev;
1694 struct i40iw_hmc_fcn_info hmc_fcn_info;
1695 struct i40iw_virt_mem vf_dev_mem;
1696 struct i40iw_vfdev *tmp_vfdev;
1698 unsigned long flags;
1700 hdl = i40iw_find_i40e_handler(ldev);
1704 dev = &hdl->device.sc_dev;
1706 for (i = 0; i < I40IW_MAX_PE_ENABLED_VF_COUNT; i++) {
1707 if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id != vf_id))
1709 /* free all resources allocated on behalf of vf */
1710 tmp_vfdev = dev->vf_dev[i];
1711 spin_lock_irqsave(&dev->dev_pestat.stats_lock, flags);
1712 dev->vf_dev[i] = NULL;
1713 spin_unlock_irqrestore(&dev->dev_pestat.stats_lock, flags);
1714 i40iw_del_hmc_objects(dev, &tmp_vfdev->hmc_info, false, false);
1715 /* remove vf hmc function */
1716 memset(&hmc_fcn_info, 0, sizeof(hmc_fcn_info));
1717 hmc_fcn_info.vf_id = vf_id;
1718 hmc_fcn_info.iw_vf_idx = tmp_vfdev->iw_vf_idx;
1719 hmc_fcn_info.free_fcn = true;
1720 i40iw_cqp_manage_hmc_fcn_cmd(dev, &hmc_fcn_info);
1722 vf_dev_mem.va = tmp_vfdev;
1723 vf_dev_mem.size = sizeof(struct i40iw_vfdev) +
1724 sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX;
1725 i40iw_free_virt_mem(dev->hw, &vf_dev_mem);
1731 * i40iw_vf_enable - enable a number of VFs
1732 * @ldev: lan device information
1733 * @client: client interface instance
1734 * @num_vfs: number of VFs for the PF
1736 * Called when the number of VFs changes
1738 static void i40iw_vf_enable(struct i40e_info *ldev,
1739 struct i40e_client *client,
1742 struct i40iw_handler *hdl;
1744 hdl = i40iw_find_i40e_handler(ldev);
1748 if (num_vfs > I40IW_MAX_PE_ENABLED_VF_COUNT)
1749 hdl->device.max_enabled_vfs = I40IW_MAX_PE_ENABLED_VF_COUNT;
1751 hdl->device.max_enabled_vfs = num_vfs;
1755 * i40iw_vf_capable - check if VF capable
1756 * @ldev: lan device information
1757 * @client: client interface instance
1758 * @vf_id: virtual function id
1760 * Return 1 if a VF slot is available or if VF is already RDMA enabled
1761 * Return 0 otherwise
1763 static int i40iw_vf_capable(struct i40e_info *ldev,
1764 struct i40e_client *client,
1767 struct i40iw_handler *hdl;
1768 struct i40iw_sc_dev *dev;
1771 hdl = i40iw_find_i40e_handler(ldev);
1775 dev = &hdl->device.sc_dev;
1777 for (i = 0; i < hdl->device.max_enabled_vfs; i++) {
1778 if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id == vf_id))
1786 * i40iw_virtchnl_receive - receive a message through the virtual channel
1787 * @ldev: lan device information
1788 * @client: client interface instance
1789 * @vf_id: virtual function id associated with the message
1790 * @msg: message buffer pointer
1791 * @len: length of the message
1793 * Invoke virtual channel receive operation for the given msg
1794 * Return 0 if successful, otherwise return error
1796 static int i40iw_virtchnl_receive(struct i40e_info *ldev,
1797 struct i40e_client *client,
1802 struct i40iw_handler *hdl;
1803 struct i40iw_sc_dev *dev;
1804 struct i40iw_device *iwdev;
1805 int ret_code = I40IW_NOT_SUPPORTED;
1808 return I40IW_ERR_PARAM;
1810 hdl = i40iw_find_i40e_handler(ldev);
1812 return I40IW_ERR_PARAM;
1814 dev = &hdl->device.sc_dev;
1815 iwdev = dev->back_dev;
1817 if (dev->vchnl_if.vchnl_recv) {
1818 ret_code = dev->vchnl_if.vchnl_recv(dev, vf_id, msg, len);
1820 atomic_dec(&iwdev->vchnl_msgs);
1821 wake_up(&iwdev->vchnl_waitq);
1828 * i40iw_vf_clear_to_send - wait to send virtual channel message
1829 * @dev: iwarp device *
1830 * Wait for until virtual channel is clear
1831 * before sending the next message
1833 * Returns false if error
1834 * Returns true if clear to send
1836 bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev)
1838 struct i40iw_device *iwdev;
1841 iwdev = dev->back_dev;
1843 if (!wq_has_sleeper(&dev->vf_reqs) &&
1844 (atomic_read(&iwdev->vchnl_msgs) == 0))
1845 return true; /* virtual channel is clear */
1848 add_wait_queue_exclusive(&dev->vf_reqs, &wait);
1850 if (!wait_event_timeout(dev->vf_reqs,
1851 (atomic_read(&iwdev->vchnl_msgs) == 0),
1852 I40IW_VCHNL_EVENT_TIMEOUT))
1853 dev->vchnl_up = false;
1855 remove_wait_queue(&dev->vf_reqs, &wait);
1857 return dev->vchnl_up;
1861 * i40iw_virtchnl_send - send a message through the virtual channel
1862 * @dev: iwarp device
1863 * @vf_id: virtual function id associated with the message
1864 * @msg: virtual channel message buffer pointer
1865 * @len: length of the message
1867 * Invoke virtual channel send operation for the given msg
1868 * Return 0 if successful, otherwise return error
1870 static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
1875 struct i40iw_device *iwdev;
1876 struct i40e_info *ldev;
1878 if (!dev || !dev->back_dev)
1879 return I40IW_ERR_BAD_PTR;
1881 iwdev = dev->back_dev;
1884 if (ldev && ldev->ops && ldev->ops->virtchnl_send)
1885 return ldev->ops->virtchnl_send(ldev, &i40iw_client, vf_id, msg, len);
1886 return I40IW_ERR_BAD_PTR;
1889 /* client interface functions */
1890 static const struct i40e_client_ops i40e_ops = {
1892 .close = i40iw_close,
1893 .l2_param_change = i40iw_l2param_change,
1894 .virtchnl_receive = i40iw_virtchnl_receive,
1895 .vf_reset = i40iw_vf_reset,
1896 .vf_enable = i40iw_vf_enable,
1897 .vf_capable = i40iw_vf_capable
1901 * i40iw_init_module - driver initialization function
1903 * First function to call when the driver is loaded
1904 * Register the driver as i40e client and port mapper client
1906 static int __init i40iw_init_module(void)
1910 memset(&i40iw_client, 0, sizeof(i40iw_client));
1911 i40iw_client.version.major = CLIENT_IW_INTERFACE_VERSION_MAJOR;
1912 i40iw_client.version.minor = CLIENT_IW_INTERFACE_VERSION_MINOR;
1913 i40iw_client.version.build = CLIENT_IW_INTERFACE_VERSION_BUILD;
1914 i40iw_client.ops = &i40e_ops;
1915 memcpy(i40iw_client.name, i40iw_client_name, I40E_CLIENT_STR_LENGTH);
1916 i40iw_client.type = I40E_CLIENT_IWARP;
1917 spin_lock_init(&i40iw_handler_lock);
1918 ret = i40e_register_client(&i40iw_client);
1923 * i40iw_exit_module - driver exit clean up function
1925 * The function is called just before the driver is unloaded
1926 * Unregister the driver as i40e client and port mapper client
1928 static void __exit i40iw_exit_module(void)
1930 i40e_unregister_client(&i40iw_client);
1933 module_init(i40iw_init_module);
1934 module_exit(i40iw_exit_module);