1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/netdevice.h>
38 #include <linux/etherdevice.h>
40 #include <linux/tcp.h>
41 #include <linux/if_vlan.h>
42 #include <net/addrconf.h>
45 #include "i40iw_register.h"
46 #include <net/netevent.h>
47 #define CLIENT_IW_INTERFACE_VERSION_MAJOR 0
48 #define CLIENT_IW_INTERFACE_VERSION_MINOR 01
49 #define CLIENT_IW_INTERFACE_VERSION_BUILD 00
51 #define DRV_VERSION_MAJOR 0
52 #define DRV_VERSION_MINOR 5
53 #define DRV_VERSION_BUILD 123
54 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
55 __stringify(DRV_VERSION_MINOR) "." __stringify(DRV_VERSION_BUILD)
58 module_param(debug, int, 0644);
59 MODULE_PARM_DESC(debug, "debug flags: 0=disabled (default), 0x7fffffff=all");
61 static int resource_profile;
62 module_param(resource_profile, int, 0644);
63 MODULE_PARM_DESC(resource_profile,
64 "Resource Profile: 0=no VF RDMA support (default), 1=Weighted VF, 2=Even Distribution");
66 static int max_rdma_vfs = 32;
67 module_param(max_rdma_vfs, int, 0644);
68 MODULE_PARM_DESC(max_rdma_vfs, "Maximum VF count: 0-32 32=default");
69 static int mpa_version = 2;
70 module_param(mpa_version, int, 0644);
71 MODULE_PARM_DESC(mpa_version, "MPA version to be used in MPA Req/Resp 1 or 2");
73 MODULE_AUTHOR("Intel Corporation, <e1000-rdma@lists.sourceforge.net>");
74 MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver");
75 MODULE_LICENSE("Dual BSD/GPL");
77 static struct i40e_client i40iw_client;
78 static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw";
80 static LIST_HEAD(i40iw_handlers);
81 static spinlock_t i40iw_handler_lock;
83 static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
84 u32 vf_id, u8 *msg, u16 len);
86 static struct notifier_block i40iw_inetaddr_notifier = {
87 .notifier_call = i40iw_inetaddr_event
90 static struct notifier_block i40iw_inetaddr6_notifier = {
91 .notifier_call = i40iw_inet6addr_event
94 static struct notifier_block i40iw_net_notifier = {
95 .notifier_call = i40iw_net_event
99 * i40iw_find_i40e_handler - find a handler given a client info
100 * @ldev: pointer to a client info
102 static struct i40iw_handler *i40iw_find_i40e_handler(struct i40e_info *ldev)
104 struct i40iw_handler *hdl;
107 spin_lock_irqsave(&i40iw_handler_lock, flags);
108 list_for_each_entry(hdl, &i40iw_handlers, list) {
109 if (hdl->ldev.netdev == ldev->netdev) {
110 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
114 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
119 * i40iw_find_netdev - find a handler given a netdev
120 * @netdev: pointer to net_device
122 struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev)
124 struct i40iw_handler *hdl;
127 spin_lock_irqsave(&i40iw_handler_lock, flags);
128 list_for_each_entry(hdl, &i40iw_handlers, list) {
129 if (hdl->ldev.netdev == netdev) {
130 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
134 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
139 * i40iw_add_handler - add a handler to the list
140 * @hdl: handler to be added to the handler list
142 static void i40iw_add_handler(struct i40iw_handler *hdl)
146 spin_lock_irqsave(&i40iw_handler_lock, flags);
147 list_add(&hdl->list, &i40iw_handlers);
148 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
152 * i40iw_del_handler - delete a handler from the list
153 * @hdl: handler to be deleted from the handler list
155 static int i40iw_del_handler(struct i40iw_handler *hdl)
159 spin_lock_irqsave(&i40iw_handler_lock, flags);
160 list_del(&hdl->list);
161 spin_unlock_irqrestore(&i40iw_handler_lock, flags);
166 * i40iw_enable_intr - set up device interrupts
167 * @dev: hardware control device structure
168 * @msix_id: id of the interrupt to be enabled
170 static void i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id)
174 val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
175 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
176 (3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
178 i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_id - 1), val);
180 i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_id - 1), val);
184 * i40iw_dpc - tasklet for aeq and ceq 0
185 * @data: iwarp device
187 static void i40iw_dpc(unsigned long data)
189 struct i40iw_device *iwdev = (struct i40iw_device *)data;
191 if (iwdev->msix_shared)
192 i40iw_process_ceq(iwdev, iwdev->ceqlist);
193 i40iw_process_aeq(iwdev);
194 i40iw_enable_intr(&iwdev->sc_dev, iwdev->iw_msixtbl[0].idx);
198 * i40iw_ceq_dpc - dpc handler for CEQ
199 * @data: data points to CEQ
201 static void i40iw_ceq_dpc(unsigned long data)
203 struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
204 struct i40iw_device *iwdev = iwceq->iwdev;
206 i40iw_process_ceq(iwdev, iwceq);
207 i40iw_enable_intr(&iwdev->sc_dev, iwceq->msix_idx);
211 * i40iw_irq_handler - interrupt handler for aeq and ceq0
212 * @irq: Interrupt request number
213 * @data: iwarp device
215 static irqreturn_t i40iw_irq_handler(int irq, void *data)
217 struct i40iw_device *iwdev = (struct i40iw_device *)data;
219 tasklet_schedule(&iwdev->dpc_tasklet);
224 * i40iw_destroy_cqp - destroy control qp
225 * @iwdev: iwarp device
226 * @create_done: 1 if cqp create poll was success
228 * Issue destroy cqp request and
229 * free the resources associated with the cqp
231 static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp)
233 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
234 struct i40iw_cqp *cqp = &iwdev->cqp;
237 dev->cqp_ops->cqp_destroy(dev->cqp);
239 i40iw_cleanup_pending_cqp_op(iwdev);
241 i40iw_free_dma_mem(dev->hw, &cqp->sq);
242 kfree(cqp->scratch_array);
243 iwdev->cqp.scratch_array = NULL;
245 kfree(cqp->cqp_requests);
246 cqp->cqp_requests = NULL;
250 * i40iw_disable_irqs - disable device interrupts
251 * @dev: hardware control device structure
252 * @msic_vec: msix vector to disable irq
253 * @dev_id: parameter to pass to free_irq (used during irq setup)
255 * The function is called when destroying aeq/ceq
257 static void i40iw_disable_irq(struct i40iw_sc_dev *dev,
258 struct i40iw_msix_vector *msix_vec,
262 i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0);
264 i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_vec->idx - 1), 0);
265 irq_set_affinity_hint(msix_vec->irq, NULL);
266 free_irq(msix_vec->irq, dev_id);
270 * i40iw_destroy_aeq - destroy aeq
271 * @iwdev: iwarp device
273 * Issue a destroy aeq request and
274 * free the resources associated with the aeq
275 * The function is called during driver unload
277 static void i40iw_destroy_aeq(struct i40iw_device *iwdev)
279 enum i40iw_status_code status = I40IW_ERR_NOT_READY;
280 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
281 struct i40iw_aeq *aeq = &iwdev->aeq;
283 if (!iwdev->msix_shared)
284 i40iw_disable_irq(dev, iwdev->iw_msixtbl, (void *)iwdev);
288 if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1))
289 status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq);
291 i40iw_pr_err("destroy aeq failed %d\n", status);
294 i40iw_free_dma_mem(dev->hw, &aeq->mem);
298 * i40iw_destroy_ceq - destroy ceq
299 * @iwdev: iwarp device
300 * @iwceq: ceq to be destroyed
302 * Issue a destroy ceq request and
303 * free the resources associated with the ceq
305 static void i40iw_destroy_ceq(struct i40iw_device *iwdev,
306 struct i40iw_ceq *iwceq)
308 enum i40iw_status_code status;
309 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
314 status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1);
316 i40iw_pr_err("ceq destroy command failed %d\n", status);
320 status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq);
322 i40iw_pr_err("ceq destroy completion failed %d\n", status);
324 i40iw_free_dma_mem(dev->hw, &iwceq->mem);
328 * i40iw_dele_ceqs - destroy all ceq's
329 * @iwdev: iwarp device
331 * Go through all of the device ceq's and for each ceq
332 * disable the ceq interrupt and destroy the ceq
334 static void i40iw_dele_ceqs(struct i40iw_device *iwdev)
337 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
338 struct i40iw_ceq *iwceq = iwdev->ceqlist;
339 struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
341 if (iwdev->msix_shared) {
342 i40iw_disable_irq(dev, msix_vec, (void *)iwdev);
343 i40iw_destroy_ceq(iwdev, iwceq);
348 for (msix_vec++; i < iwdev->ceqs_count; i++, msix_vec++, iwceq++) {
349 i40iw_disable_irq(dev, msix_vec, (void *)iwceq);
350 i40iw_destroy_ceq(iwdev, iwceq);
355 * i40iw_destroy_ccq - destroy control cq
356 * @iwdev: iwarp device
358 * Issue destroy ccq request and
359 * free the resources associated with the ccq
361 static void i40iw_destroy_ccq(struct i40iw_device *iwdev)
363 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
364 struct i40iw_ccq *ccq = &iwdev->ccq;
365 enum i40iw_status_code status = 0;
368 status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, true);
370 i40iw_pr_err("ccq destroy failed %d\n", status);
371 i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
374 /* types of hmc objects */
375 static enum i40iw_hmc_rsrc_type iw_hmc_obj_types[] = {
380 I40IW_HMC_IW_APBVT_ENTRY,
390 * i40iw_close_hmc_objects_type - delete hmc objects of a given type
391 * @iwdev: iwarp device
392 * @obj_type: the hmc object type to be deleted
393 * @is_pf: true if the function is PF otherwise false
394 * @reset: true if called before reset
396 static void i40iw_close_hmc_objects_type(struct i40iw_sc_dev *dev,
397 enum i40iw_hmc_rsrc_type obj_type,
398 struct i40iw_hmc_info *hmc_info,
402 struct i40iw_hmc_del_obj_info info;
404 memset(&info, 0, sizeof(info));
405 info.hmc_info = hmc_info;
406 info.rsrc_type = obj_type;
407 info.count = hmc_info->hmc_obj[obj_type].cnt;
409 if (dev->hmc_ops->del_hmc_object(dev, &info, reset))
410 i40iw_pr_err("del obj of type %d failed\n", obj_type);
414 * i40iw_del_hmc_objects - remove all device hmc objects
416 * @hmc_info: hmc_info to free
417 * @is_pf: true if hmc_info belongs to PF, not vf nor allocated
418 * by PF on behalf of VF
419 * @reset: true if called before reset
421 static void i40iw_del_hmc_objects(struct i40iw_sc_dev *dev,
422 struct i40iw_hmc_info *hmc_info,
428 for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++)
429 i40iw_close_hmc_objects_type(dev, iw_hmc_obj_types[i], hmc_info, is_pf, reset);
433 * i40iw_ceq_handler - interrupt handler for ceq
436 static irqreturn_t i40iw_ceq_handler(int irq, void *data)
438 struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
440 if (iwceq->irq != irq)
441 i40iw_pr_err("expected irq = %d received irq = %d\n", iwceq->irq, irq);
442 tasklet_schedule(&iwceq->dpc_tasklet);
447 * i40iw_create_hmc_obj_type - create hmc object of a given type
448 * @dev: hardware control device structure
449 * @info: information for the hmc object to create
451 static enum i40iw_status_code i40iw_create_hmc_obj_type(struct i40iw_sc_dev *dev,
452 struct i40iw_hmc_create_obj_info *info)
454 return dev->hmc_ops->create_hmc_object(dev, info);
458 * i40iw_create_hmc_objs - create all hmc objects for the device
459 * @iwdev: iwarp device
460 * @is_pf: true if the function is PF otherwise false
462 * Create the device hmc objects and allocate hmc pages
463 * Return 0 if successful, otherwise clean up and return error
465 static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev,
468 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
469 struct i40iw_hmc_create_obj_info info;
470 enum i40iw_status_code status;
473 memset(&info, 0, sizeof(info));
474 info.hmc_info = dev->hmc_info;
476 info.entry_type = iwdev->sd_type;
477 for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
478 info.rsrc_type = iw_hmc_obj_types[i];
479 info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
480 status = i40iw_create_hmc_obj_type(dev, &info);
482 i40iw_pr_err("create obj type %d status = %d\n",
483 iw_hmc_obj_types[i], status);
488 return (dev->cqp_misc_ops->static_hmc_pages_allocated(dev->cqp, 0,
494 /* destroy the hmc objects of a given type */
495 i40iw_close_hmc_objects_type(dev,
505 * i40iw_obj_aligned_mem - get aligned memory from device allocated memory
506 * @iwdev: iwarp device
507 * @memptr: points to the memory addresses
508 * @size: size of memory needed
509 * @mask: mask for the aligned memory
511 * Get aligned memory of the requested size and
512 * update the memptr to point to the new aligned memory
513 * Return 0 if successful, otherwise return no memory error
515 enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,
516 struct i40iw_dma_mem *memptr,
520 unsigned long va, newva;
523 va = (unsigned long)iwdev->obj_next.va;
526 newva = ALIGN(va, (mask + 1));
528 memptr->va = (u8 *)va + extra;
529 memptr->pa = iwdev->obj_next.pa + extra;
531 if ((memptr->va + size) > (iwdev->obj_mem.va + iwdev->obj_mem.size))
532 return I40IW_ERR_NO_MEMORY;
534 iwdev->obj_next.va = memptr->va + size;
535 iwdev->obj_next.pa = memptr->pa + size;
540 * i40iw_create_cqp - create control qp
541 * @iwdev: iwarp device
543 * Return 0, if the cqp and all the resources associated with it
544 * are successfully created, otherwise return error
546 static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)
548 enum i40iw_status_code status;
549 u32 sqsize = I40IW_CQP_SW_SQSIZE_2048;
550 struct i40iw_dma_mem mem;
551 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
552 struct i40iw_cqp_init_info cqp_init_info;
553 struct i40iw_cqp *cqp = &iwdev->cqp;
554 u16 maj_err, min_err;
557 cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
558 if (!cqp->cqp_requests)
559 return I40IW_ERR_NO_MEMORY;
560 cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
561 if (!cqp->scratch_array) {
562 kfree(cqp->cqp_requests);
563 return I40IW_ERR_NO_MEMORY;
565 dev->cqp = &cqp->sc_cqp;
567 memset(&cqp_init_info, 0, sizeof(cqp_init_info));
568 status = i40iw_allocate_dma_mem(dev->hw, &cqp->sq,
569 (sizeof(struct i40iw_cqp_sq_wqe) * sqsize),
570 I40IW_CQP_ALIGNMENT);
573 status = i40iw_obj_aligned_mem(iwdev, &mem, sizeof(struct i40iw_cqp_ctx),
574 I40IW_HOST_CTX_ALIGNMENT_MASK);
577 dev->cqp->host_ctx_pa = mem.pa;
578 dev->cqp->host_ctx = mem.va;
579 /* populate the cqp init info */
580 cqp_init_info.dev = dev;
581 cqp_init_info.sq_size = sqsize;
582 cqp_init_info.sq = cqp->sq.va;
583 cqp_init_info.sq_pa = cqp->sq.pa;
584 cqp_init_info.host_ctx_pa = mem.pa;
585 cqp_init_info.host_ctx = mem.va;
586 cqp_init_info.hmc_profile = iwdev->resource_profile;
587 cqp_init_info.enabled_vf_count = iwdev->max_rdma_vfs;
588 cqp_init_info.scratch_array = cqp->scratch_array;
589 status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info);
591 i40iw_pr_err("cqp init status %d\n", status);
594 status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
596 i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",
597 status, maj_err, min_err);
600 spin_lock_init(&cqp->req_lock);
601 INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
602 INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
603 /* init the waitq of the cqp_requests and add them to the list */
604 for (i = 0; i < I40IW_CQP_SW_SQSIZE_2048; i++) {
605 init_waitqueue_head(&cqp->cqp_requests[i].waitq);
606 list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
610 /* clean up the created resources */
611 i40iw_destroy_cqp(iwdev, false);
616 * i40iw_create_ccq - create control cq
617 * @iwdev: iwarp device
619 * Return 0, if the ccq and the resources associated with it
620 * are successfully created, otherwise return error
622 static enum i40iw_status_code i40iw_create_ccq(struct i40iw_device *iwdev)
624 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
625 struct i40iw_dma_mem mem;
626 enum i40iw_status_code status;
627 struct i40iw_ccq_init_info info;
628 struct i40iw_ccq *ccq = &iwdev->ccq;
630 memset(&info, 0, sizeof(info));
631 dev->ccq = &ccq->sc_cq;
634 ccq->shadow_area.size = sizeof(struct i40iw_cq_shadow_area);
635 ccq->mem_cq.size = sizeof(struct i40iw_cqe) * IW_CCQ_SIZE;
636 status = i40iw_allocate_dma_mem(dev->hw, &ccq->mem_cq,
637 ccq->mem_cq.size, I40IW_CQ0_ALIGNMENT);
640 status = i40iw_obj_aligned_mem(iwdev, &mem, ccq->shadow_area.size,
641 I40IW_SHADOWAREA_MASK);
644 ccq->sc_cq.back_cq = (void *)ccq;
645 /* populate the ccq init info */
646 info.cq_base = ccq->mem_cq.va;
647 info.cq_pa = ccq->mem_cq.pa;
648 info.num_elem = IW_CCQ_SIZE;
649 info.shadow_area = mem.va;
650 info.shadow_area_pa = mem.pa;
651 info.ceqe_mask = false;
652 info.ceq_id_valid = true;
653 info.shadow_read_threshold = 16;
654 status = dev->ccq_ops->ccq_init(dev->ccq, &info);
656 status = dev->ccq_ops->ccq_create(dev->ccq, 0, true, true);
659 i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
664 * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq
665 * @iwdev: iwarp device
666 * @msix_vec: interrupt vector information
667 * @iwceq: ceq associated with the vector
668 * @ceq_id: the id number of the iwceq
670 * Allocate interrupt resources and enable irq handling
671 * Return 0 if successful, otherwise return error
673 static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iwdev,
674 struct i40iw_ceq *iwceq,
676 struct i40iw_msix_vector *msix_vec)
678 enum i40iw_status_code status;
680 if (iwdev->msix_shared && !ceq_id) {
681 tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
682 status = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "AEQCEQ", iwdev);
684 tasklet_init(&iwceq->dpc_tasklet, i40iw_ceq_dpc, (unsigned long)iwceq);
685 status = request_irq(msix_vec->irq, i40iw_ceq_handler, 0, "CEQ", iwceq);
688 cpumask_clear(&msix_vec->mask);
689 cpumask_set_cpu(msix_vec->cpu_affinity, &msix_vec->mask);
690 irq_set_affinity_hint(msix_vec->irq, &msix_vec->mask);
693 i40iw_pr_err("ceq irq config fail\n");
694 return I40IW_ERR_CONFIG;
696 msix_vec->ceq_id = ceq_id;
702 * i40iw_create_ceq - create completion event queue
703 * @iwdev: iwarp device
704 * @iwceq: pointer to the ceq resources to be created
705 * @ceq_id: the id number of the iwceq
707 * Return 0, if the ceq and the resources associated with it
708 * are successfully created, otherwise return error
710 static enum i40iw_status_code i40iw_create_ceq(struct i40iw_device *iwdev,
711 struct i40iw_ceq *iwceq,
714 enum i40iw_status_code status;
715 struct i40iw_ceq_init_info info;
716 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
719 memset(&info, 0, sizeof(info));
720 info.ceq_id = ceq_id;
721 iwceq->iwdev = iwdev;
722 iwceq->mem.size = sizeof(struct i40iw_ceqe) *
723 iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
724 status = i40iw_allocate_dma_mem(dev->hw, &iwceq->mem, iwceq->mem.size,
725 I40IW_CEQ_ALIGNMENT);
728 info.ceq_id = ceq_id;
729 info.ceqe_base = iwceq->mem.va;
730 info.ceqe_pa = iwceq->mem.pa;
732 info.elem_cnt = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
733 iwceq->sc_ceq.ceq_id = ceq_id;
735 scratch = (uintptr_t)&iwdev->cqp.sc_cqp;
736 status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info);
738 status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq, scratch);
742 i40iw_free_dma_mem(dev->hw, &iwceq->mem);
746 void i40iw_request_reset(struct i40iw_device *iwdev)
748 struct i40e_info *ldev = iwdev->ldev;
750 ldev->ops->request_reset(ldev, iwdev->client, 1);
754 * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources
755 * @iwdev: iwarp device
756 * @ldev: i40e lan device
758 * Allocate a list for all device completion event queues
759 * Create the ceq's and configure their msix interrupt vectors
760 * Return 0, if at least one ceq is successfully set up, otherwise return error
762 static enum i40iw_status_code i40iw_setup_ceqs(struct i40iw_device *iwdev,
763 struct i40e_info *ldev)
767 struct i40iw_ceq *iwceq;
768 struct i40iw_msix_vector *msix_vec;
769 enum i40iw_status_code status = 0;
772 if (ldev && ldev->ops && ldev->ops->setup_qvlist) {
773 status = ldev->ops->setup_qvlist(ldev, &i40iw_client,
778 status = I40IW_ERR_BAD_PTR;
782 num_ceqs = min(iwdev->msix_count, iwdev->sc_dev.hmc_fpm_misc.max_ceqs);
783 iwdev->ceqlist = kcalloc(num_ceqs, sizeof(*iwdev->ceqlist), GFP_KERNEL);
784 if (!iwdev->ceqlist) {
785 status = I40IW_ERR_NO_MEMORY;
788 i = (iwdev->msix_shared) ? 0 : 1;
789 for (ceq_id = 0; i < num_ceqs; i++, ceq_id++) {
790 iwceq = &iwdev->ceqlist[ceq_id];
791 status = i40iw_create_ceq(iwdev, iwceq, ceq_id);
793 i40iw_pr_err("create ceq status = %d\n", status);
797 msix_vec = &iwdev->iw_msixtbl[i];
798 iwceq->irq = msix_vec->irq;
799 iwceq->msix_idx = msix_vec->idx;
800 status = i40iw_configure_ceq_vector(iwdev, iwceq, ceq_id, msix_vec);
802 i40iw_destroy_ceq(iwdev, iwceq);
805 i40iw_enable_intr(&iwdev->sc_dev, msix_vec->idx);
811 if (!iwdev->ceqs_count) {
812 kfree(iwdev->ceqlist);
813 iwdev->ceqlist = NULL;
822 * i40iw_configure_aeq_vector - set up the msix vector for aeq
823 * @iwdev: iwarp device
825 * Allocate interrupt resources and enable irq handling
826 * Return 0 if successful, otherwise return error
828 static enum i40iw_status_code i40iw_configure_aeq_vector(struct i40iw_device *iwdev)
830 struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
833 if (!iwdev->msix_shared) {
834 tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
835 ret = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "i40iw", iwdev);
838 i40iw_pr_err("aeq irq config fail\n");
839 return I40IW_ERR_CONFIG;
846 * i40iw_create_aeq - create async event queue
847 * @iwdev: iwarp device
849 * Return 0, if the aeq and the resources associated with it
850 * are successfully created, otherwise return error
852 static enum i40iw_status_code i40iw_create_aeq(struct i40iw_device *iwdev)
854 enum i40iw_status_code status;
855 struct i40iw_aeq_init_info info;
856 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
857 struct i40iw_aeq *aeq = &iwdev->aeq;
861 aeq_size = 2 * iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt +
862 iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
863 memset(&info, 0, sizeof(info));
864 aeq->mem.size = sizeof(struct i40iw_sc_aeqe) * aeq_size;
865 status = i40iw_allocate_dma_mem(dev->hw, &aeq->mem, aeq->mem.size,
866 I40IW_AEQ_ALIGNMENT);
870 info.aeqe_base = aeq->mem.va;
871 info.aeq_elem_pa = aeq->mem.pa;
872 info.elem_cnt = aeq_size;
874 status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info);
877 status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1);
879 status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq);
882 i40iw_free_dma_mem(dev->hw, &aeq->mem);
887 * i40iw_setup_aeq - set up the device aeq
888 * @iwdev: iwarp device
890 * Create the aeq and configure its msix interrupt vector
891 * Return 0 if successful, otherwise return error
893 static enum i40iw_status_code i40iw_setup_aeq(struct i40iw_device *iwdev)
895 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
896 enum i40iw_status_code status;
898 status = i40iw_create_aeq(iwdev);
902 status = i40iw_configure_aeq_vector(iwdev);
904 i40iw_destroy_aeq(iwdev);
908 if (!iwdev->msix_shared)
909 i40iw_enable_intr(dev, iwdev->iw_msixtbl[0].idx);
914 * i40iw_initialize_ilq - create iwarp local queue for cm
915 * @iwdev: iwarp device
917 * Return 0 if successful, otherwise return error
919 static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev)
921 struct i40iw_puda_rsrc_info info;
922 enum i40iw_status_code status;
924 memset(&info, 0, sizeof(info));
925 info.type = I40IW_PUDA_RSRC_TYPE_ILQ;
932 info.buf_size = 1024;
933 info.tx_buf_cnt = 16384;
934 info.receive = i40iw_receive_ilq;
935 info.xmit_complete = i40iw_free_sqbuf;
936 status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
938 i40iw_pr_err("ilq create fail\n");
943 * i40iw_initialize_ieq - create iwarp exception queue
944 * @iwdev: iwarp device
946 * Return 0 if successful, otherwise return error
948 static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev)
950 struct i40iw_puda_rsrc_info info;
951 enum i40iw_status_code status;
953 memset(&info, 0, sizeof(info));
954 info.type = I40IW_PUDA_RSRC_TYPE_IEQ;
956 info.qp_id = iwdev->sc_dev.exception_lan_queue;
961 info.buf_size = 2048;
962 info.tx_buf_cnt = 16384;
963 status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
965 i40iw_pr_err("ieq create fail\n");
970 * i40iw_hmc_setup - create hmc objects for the device
971 * @iwdev: iwarp device
973 * Set up the device private memory space for the number and size of
974 * the hmc objects and create the objects
975 * Return 0 if successful, otherwise return error
977 static enum i40iw_status_code i40iw_hmc_setup(struct i40iw_device *iwdev)
979 enum i40iw_status_code status;
981 iwdev->sd_type = I40IW_SD_TYPE_DIRECT;
982 status = i40iw_config_fpm_values(&iwdev->sc_dev, IW_CFG_FPM_QP_COUNT);
985 status = i40iw_create_hmc_objs(iwdev, true);
988 iwdev->init_state = HMC_OBJS_CREATED;
994 * i40iw_del_init_mem - deallocate memory resources
995 * @iwdev: iwarp device
997 static void i40iw_del_init_mem(struct i40iw_device *iwdev)
999 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1001 i40iw_free_dma_mem(&iwdev->hw, &iwdev->obj_mem);
1002 kfree(dev->hmc_info->sd_table.sd_entry);
1003 dev->hmc_info->sd_table.sd_entry = NULL;
1004 kfree(iwdev->mem_resources);
1005 iwdev->mem_resources = NULL;
1006 kfree(iwdev->ceqlist);
1007 iwdev->ceqlist = NULL;
1008 kfree(iwdev->iw_msixtbl);
1009 iwdev->iw_msixtbl = NULL;
1010 kfree(iwdev->hmc_info_mem);
1011 iwdev->hmc_info_mem = NULL;
1015 * i40iw_del_macip_entry - remove a mac ip address entry from the hw table
1016 * @iwdev: iwarp device
1017 * @idx: the index of the mac ip address to delete
1019 static void i40iw_del_macip_entry(struct i40iw_device *iwdev, u8 idx)
1021 struct i40iw_cqp *iwcqp = &iwdev->cqp;
1022 struct i40iw_cqp_request *cqp_request;
1023 struct cqp_commands_info *cqp_info;
1024 enum i40iw_status_code status = 0;
1026 cqp_request = i40iw_get_cqp_request(iwcqp, true);
1028 i40iw_pr_err("cqp_request memory failed\n");
1031 cqp_info = &cqp_request->info;
1032 cqp_info->cqp_cmd = OP_DELETE_LOCAL_MAC_IPADDR_ENTRY;
1033 cqp_info->post_sq = 1;
1034 cqp_info->in.u.del_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1035 cqp_info->in.u.del_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1036 cqp_info->in.u.del_local_mac_ipaddr_entry.entry_idx = idx;
1037 cqp_info->in.u.del_local_mac_ipaddr_entry.ignore_ref_count = 0;
1038 status = i40iw_handle_cqp_op(iwdev, cqp_request);
1040 i40iw_pr_err("CQP-OP Del MAC Ip entry fail");
1044 * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table
1045 * @iwdev: iwarp device
1046 * @mac_addr: pointer to mac address
1047 * @idx: the index of the mac ip address to add
1049 static enum i40iw_status_code i40iw_add_mac_ipaddr_entry(struct i40iw_device *iwdev,
1053 struct i40iw_local_mac_ipaddr_entry_info *info;
1054 struct i40iw_cqp *iwcqp = &iwdev->cqp;
1055 struct i40iw_cqp_request *cqp_request;
1056 struct cqp_commands_info *cqp_info;
1057 enum i40iw_status_code status = 0;
1059 cqp_request = i40iw_get_cqp_request(iwcqp, true);
1061 i40iw_pr_err("cqp_request memory failed\n");
1062 return I40IW_ERR_NO_MEMORY;
1065 cqp_info = &cqp_request->info;
1067 cqp_info->post_sq = 1;
1068 info = &cqp_info->in.u.add_local_mac_ipaddr_entry.info;
1069 ether_addr_copy(info->mac_addr, mac_addr);
1070 info->entry_idx = idx;
1071 cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1072 cqp_info->cqp_cmd = OP_ADD_LOCAL_MAC_IPADDR_ENTRY;
1073 cqp_info->in.u.add_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1074 cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1075 status = i40iw_handle_cqp_op(iwdev, cqp_request);
1077 i40iw_pr_err("CQP-OP Add MAC Ip entry fail");
1082 * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry
1083 * @iwdev: iwarp device
1084 * @mac_ip_tbl_idx: the index of the new mac ip address
1086 * Allocate a mac ip address entry and update the mac_ip_tbl_idx
1087 * to hold the index of the newly created mac ip address
1088 * Return 0 if successful, otherwise return error
1090 static enum i40iw_status_code i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device *iwdev,
1091 u16 *mac_ip_tbl_idx)
1093 struct i40iw_cqp *iwcqp = &iwdev->cqp;
1094 struct i40iw_cqp_request *cqp_request;
1095 struct cqp_commands_info *cqp_info;
1096 enum i40iw_status_code status = 0;
1098 cqp_request = i40iw_get_cqp_request(iwcqp, true);
1100 i40iw_pr_err("cqp_request memory failed\n");
1101 return I40IW_ERR_NO_MEMORY;
1104 /* increment refcount, because we need the cqp request ret value */
1105 atomic_inc(&cqp_request->refcount);
1107 cqp_info = &cqp_request->info;
1108 cqp_info->cqp_cmd = OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY;
1109 cqp_info->post_sq = 1;
1110 cqp_info->in.u.alloc_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1111 cqp_info->in.u.alloc_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1112 status = i40iw_handle_cqp_op(iwdev, cqp_request);
1114 *mac_ip_tbl_idx = cqp_request->compl_info.op_ret_val;
1116 i40iw_pr_err("CQP-OP Alloc MAC Ip entry fail");
1117 /* decrement refcount and free the cqp request, if no longer used */
1118 i40iw_put_cqp_request(iwcqp, cqp_request);
1123 * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry
1124 * @iwdev: iwarp device
1125 * @macaddr: pointer to mac address
1127 * Allocate a mac ip address entry and add it to the hw table
1128 * Return 0 if successful, otherwise return error
1130 static enum i40iw_status_code i40iw_alloc_set_mac_ipaddr(struct i40iw_device *iwdev,
1133 enum i40iw_status_code status;
1135 status = i40iw_alloc_local_mac_ipaddr_entry(iwdev, &iwdev->mac_ip_table_idx);
1137 status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr,
1138 (u8)iwdev->mac_ip_table_idx);
1140 i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
1146 * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table
1147 * @iwdev: iwarp device
1149 static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev)
1151 struct net_device *ip_dev;
1152 struct inet6_dev *idev;
1153 struct inet6_ifaddr *ifp, *tmp;
1154 u32 local_ipaddr6[4];
1157 for_each_netdev_rcu(&init_net, ip_dev) {
1158 if ((((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF) &&
1159 (rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev)) ||
1160 (ip_dev == iwdev->netdev)) && (ip_dev->flags & IFF_UP)) {
1161 idev = __in6_dev_get(ip_dev);
1163 i40iw_pr_err("ipv6 inet device not found\n");
1166 list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) {
1167 i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &ifp->addr,
1168 rdma_vlan_dev_vlan_id(ip_dev), ip_dev->dev_addr);
1169 i40iw_copy_ip_ntohl(local_ipaddr6,
1170 ifp->addr.in6_u.u6_addr32);
1171 i40iw_manage_arp_cache(iwdev,
1183 * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table
1184 * @iwdev: iwarp device
1186 static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev)
1188 struct net_device *dev;
1189 struct in_device *idev;
1190 bool got_lock = true;
1193 if (!rtnl_trylock())
1196 for_each_netdev(&init_net, dev) {
1197 if ((((rdma_vlan_dev_vlan_id(dev) < 0xFFFF) &&
1198 (rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) ||
1199 (dev == iwdev->netdev)) && (dev->flags & IFF_UP)) {
1200 idev = in_dev_get(dev);
1202 i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
1203 "IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_address,
1204 rdma_vlan_dev_vlan_id(dev), dev->dev_addr);
1206 ip_addr = ntohl(ifa->ifa_address);
1207 i40iw_manage_arp_cache(iwdev,
1222 * i40iw_add_mac_ip - add mac and ip addresses
1223 * @iwdev: iwarp device
1225 * Create and add a mac ip address entry to the hw table and
1226 * ipv4/ipv6 addresses to the arp cache
1227 * Return 0 if successful, otherwise return error
1229 static enum i40iw_status_code i40iw_add_mac_ip(struct i40iw_device *iwdev)
1231 struct net_device *netdev = iwdev->netdev;
1232 enum i40iw_status_code status;
1234 status = i40iw_alloc_set_mac_ipaddr(iwdev, (u8 *)netdev->dev_addr);
1237 i40iw_add_ipv4_addr(iwdev);
1238 i40iw_add_ipv6_addr(iwdev);
1243 * i40iw_wait_pe_ready - Check if firmware is ready
1244 * @hw: provides access to registers
1246 static void i40iw_wait_pe_ready(struct i40iw_hw *hw)
1255 statusfw = i40iw_rd32(hw, I40E_GLPE_FWLDSTATUS);
1256 i40iw_pr_info("[%04d] fm load status[x%04X]\n", __LINE__, statusfw);
1257 statuscpu0 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS0);
1258 i40iw_pr_info("[%04d] CSR_CQP status[x%04X]\n", __LINE__, statuscpu0);
1259 statuscpu1 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS1);
1260 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS1 status[x%04X]\n",
1261 __LINE__, statuscpu1);
1262 statuscpu2 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS2);
1263 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS2 status[x%04X]\n",
1264 __LINE__, statuscpu2);
1265 if ((statuscpu0 == 0x80) && (statuscpu1 == 0x80) && (statuscpu2 == 0x80))
1266 break; /* SUCCESS */
1269 } while (retrycount < 14);
1270 i40iw_wr32(hw, 0xb4040, 0x4C104C5);
1274 * i40iw_initialize_dev - initialize device
1275 * @iwdev: iwarp device
1276 * @ldev: lan device information
1278 * Allocate memory for the hmc objects and initialize iwdev
1279 * Return 0 if successful, otherwise clean up the resources
1282 static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev,
1283 struct i40e_info *ldev)
1285 enum i40iw_status_code status;
1286 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1287 struct i40iw_device_init_info info;
1288 struct i40iw_vsi_init_info vsi_info;
1289 struct i40iw_dma_mem mem;
1290 struct i40iw_l2params l2params;
1292 struct i40iw_vsi_stats_info stats_info;
1293 u16 last_qset = I40IW_NO_QSET;
1297 memset(&l2params, 0, sizeof(l2params));
1298 memset(&info, 0, sizeof(info));
1299 size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) +
1300 (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX);
1301 iwdev->hmc_info_mem = kzalloc(size, GFP_KERNEL);
1302 if (!iwdev->hmc_info_mem)
1303 return I40IW_ERR_NO_MEMORY;
1305 iwdev->pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev->hmc_info_mem;
1306 dev->hmc_info = &iwdev->hw.hmc;
1307 dev->hmc_info->hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev->pble_rsrc + 1);
1308 status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_QUERY_FPM_BUF_SIZE,
1309 I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);
1312 info.fpm_query_buf_pa = mem.pa;
1313 info.fpm_query_buf = mem.va;
1314 status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_COMMIT_FPM_BUF_SIZE,
1315 I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK);
1318 info.fpm_commit_buf_pa = mem.pa;
1319 info.fpm_commit_buf = mem.va;
1320 info.hmc_fn_id = ldev->fid;
1321 info.is_pf = (ldev->ftype) ? false : true;
1322 info.bar0 = ldev->hw_addr;
1323 info.hw = &iwdev->hw;
1324 info.debug_mask = debug;
1326 (ldev->params.mtu) ? ldev->params.mtu - I40IW_MTU_TO_MSS : I40IW_DEFAULT_MSS;
1327 for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) {
1328 qset = ldev->params.qos.prio_qos[i].qs_handle;
1329 l2params.qs_handle_list[i] = qset;
1330 if (last_qset == I40IW_NO_QSET)
1332 else if ((qset != last_qset) && (qset != I40IW_NO_QSET))
1335 i40iw_pr_info("DCB is set/clear = %d\n", iwdev->dcb);
1336 info.exception_lan_queue = 1;
1337 info.vchnl_send = i40iw_virtchnl_send;
1338 status = i40iw_device_init(&iwdev->sc_dev, &info);
1342 memset(&vsi_info, 0, sizeof(vsi_info));
1343 vsi_info.dev = &iwdev->sc_dev;
1344 vsi_info.back_vsi = (void *)iwdev;
1345 vsi_info.params = &l2params;
1346 i40iw_sc_vsi_init(&iwdev->vsi, &vsi_info);
1349 memset(&stats_info, 0, sizeof(stats_info));
1350 stats_info.fcn_id = ldev->fid;
1351 stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
1352 if (!stats_info.pestat) {
1353 status = I40IW_ERR_NO_MEMORY;
1356 stats_info.stats_initialize = true;
1357 if (stats_info.pestat)
1358 i40iw_vsi_stats_init(&iwdev->vsi, &stats_info);
1362 kfree(iwdev->hmc_info_mem);
1363 iwdev->hmc_info_mem = NULL;
1368 * i40iw_register_notifiers - register tcp ip notifiers
1370 static void i40iw_register_notifiers(void)
1372 register_inetaddr_notifier(&i40iw_inetaddr_notifier);
1373 register_inet6addr_notifier(&i40iw_inetaddr6_notifier);
1374 register_netevent_notifier(&i40iw_net_notifier);
1378 * i40iw_unregister_notifiers - unregister tcp ip notifiers
1381 static void i40iw_unregister_notifiers(void)
1383 unregister_netevent_notifier(&i40iw_net_notifier);
1384 unregister_inetaddr_notifier(&i40iw_inetaddr_notifier);
1385 unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier);
1389 * i40iw_save_msix_info - copy msix vector information to iwarp device
1390 * @iwdev: iwarp device
1391 * @ldev: lan device information
1393 * Allocate iwdev msix table and copy the ldev msix info to the table
1394 * Return 0 if successful, otherwise return error
1396 static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev,
1397 struct i40e_info *ldev)
1399 struct i40e_qvlist_info *iw_qvlist;
1400 struct i40e_qv_info *iw_qvinfo;
1405 if (!ldev->msix_count) {
1406 i40iw_pr_err("No MSI-X vectors\n");
1407 return I40IW_ERR_CONFIG;
1410 iwdev->msix_count = ldev->msix_count;
1412 size = sizeof(struct i40iw_msix_vector) * iwdev->msix_count;
1413 size += sizeof(struct i40e_qvlist_info);
1414 size += sizeof(struct i40e_qv_info) * iwdev->msix_count - 1;
1415 iwdev->iw_msixtbl = kzalloc(size, GFP_KERNEL);
1417 if (!iwdev->iw_msixtbl)
1418 return I40IW_ERR_NO_MEMORY;
1419 iwdev->iw_qvlist = (struct i40e_qvlist_info *)(&iwdev->iw_msixtbl[iwdev->msix_count]);
1420 iw_qvlist = iwdev->iw_qvlist;
1421 iw_qvinfo = iw_qvlist->qv_info;
1422 iw_qvlist->num_vectors = iwdev->msix_count;
1423 if (iwdev->msix_count <= num_online_cpus())
1424 iwdev->msix_shared = true;
1425 for (i = 0, ceq_idx = 0; i < iwdev->msix_count; i++, iw_qvinfo++) {
1426 iwdev->iw_msixtbl[i].idx = ldev->msix_entries[i].entry;
1427 iwdev->iw_msixtbl[i].irq = ldev->msix_entries[i].vector;
1428 iwdev->iw_msixtbl[i].cpu_affinity = ceq_idx;
1430 iw_qvinfo->aeq_idx = 0;
1431 if (iwdev->msix_shared)
1432 iw_qvinfo->ceq_idx = ceq_idx++;
1434 iw_qvinfo->ceq_idx = I40E_QUEUE_INVALID_IDX;
1436 iw_qvinfo->aeq_idx = I40E_QUEUE_INVALID_IDX;
1437 iw_qvinfo->ceq_idx = ceq_idx++;
1439 iw_qvinfo->itr_idx = 3;
1440 iw_qvinfo->v_idx = iwdev->iw_msixtbl[i].idx;
1446 * i40iw_deinit_device - clean up the device resources
1447 * @iwdev: iwarp device
1449 * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses,
1450 * destroy the device queues and free the pble and the hmc objects
1452 static void i40iw_deinit_device(struct i40iw_device *iwdev)
1454 struct i40e_info *ldev = iwdev->ldev;
1456 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1458 i40iw_pr_info("state = %d\n", iwdev->init_state);
1459 if (iwdev->param_wq)
1460 destroy_workqueue(iwdev->param_wq);
1462 switch (iwdev->init_state) {
1463 case RDMA_DEV_REGISTERED:
1464 iwdev->iw_status = 0;
1465 i40iw_port_ibevent(iwdev);
1466 i40iw_destroy_rdma_device(iwdev->iwibdev);
1468 case IP_ADDR_REGISTERED:
1470 i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
1473 case PBLE_CHUNK_MEM:
1474 i40iw_destroy_pble_pool(dev, iwdev->pble_rsrc);
1477 i40iw_dele_ceqs(iwdev);
1480 i40iw_destroy_aeq(iwdev);
1483 i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, iwdev->reset);
1486 i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_ILQ, iwdev->reset);
1489 i40iw_destroy_ccq(iwdev);
1491 case HMC_OBJS_CREATED:
1492 i40iw_del_hmc_objects(dev, dev->hmc_info, true, iwdev->reset);
1495 i40iw_destroy_cqp(iwdev, true);
1498 i40iw_cleanup_cm_core(&iwdev->cm_core);
1499 if (iwdev->vsi.pestat) {
1500 i40iw_vsi_stats_free(&iwdev->vsi);
1501 kfree(iwdev->vsi.pestat);
1503 i40iw_del_init_mem(iwdev);
1508 i40iw_pr_err("bad init_state = %d\n", iwdev->init_state);
1512 i40iw_del_handler(i40iw_find_i40e_handler(ldev));
1517 * i40iw_setup_init_state - set up the initial device struct
1518 * @hdl: handler for iwarp device - one per instance
1519 * @ldev: lan device information
1520 * @client: iwarp client information, provided during registration
1522 * Initialize the iwarp device and its hdl information
1523 * using the ldev and client information
1524 * Return 0 if successful, otherwise return error
1526 static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl,
1527 struct i40e_info *ldev,
1528 struct i40e_client *client)
1530 struct i40iw_device *iwdev = &hdl->device;
1531 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1532 enum i40iw_status_code status;
1534 memcpy(&hdl->ldev, ldev, sizeof(*ldev));
1535 if (resource_profile == 1)
1536 resource_profile = 2;
1538 iwdev->mpa_version = mpa_version;
1539 iwdev->resource_profile = (resource_profile < I40IW_HMC_PROFILE_EQUAL) ?
1540 (u8)resource_profile + I40IW_HMC_PROFILE_DEFAULT :
1541 I40IW_HMC_PROFILE_DEFAULT;
1542 iwdev->max_rdma_vfs =
1543 (iwdev->resource_profile != I40IW_HMC_PROFILE_DEFAULT) ? max_rdma_vfs : 0;
1544 iwdev->max_enabled_vfs = iwdev->max_rdma_vfs;
1545 iwdev->netdev = ldev->netdev;
1546 hdl->client = client;
1548 iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_DB_ADDR_OFFSET;
1550 iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_VF_DB_ADDR_OFFSET;
1552 status = i40iw_save_msix_info(iwdev, ldev);
1555 iwdev->hw.dev_context = (void *)ldev->pcidev;
1556 iwdev->hw.hw_addr = ldev->hw_addr;
1557 status = i40iw_allocate_dma_mem(&iwdev->hw,
1558 &iwdev->obj_mem, 8192, 4096);
1561 iwdev->obj_next = iwdev->obj_mem;
1563 init_waitqueue_head(&iwdev->vchnl_waitq);
1564 init_waitqueue_head(&dev->vf_reqs);
1565 init_waitqueue_head(&iwdev->close_wq);
1567 status = i40iw_initialize_dev(iwdev, ldev);
1570 kfree(iwdev->iw_msixtbl);
1571 i40iw_free_dma_mem(dev->hw, &iwdev->obj_mem);
1572 iwdev->iw_msixtbl = NULL;
1578 * i40iw_get_used_rsrc - determine resources used internally
1579 * @iwdev: iwarp device
1581 * Called after internal allocations
1583 static void i40iw_get_used_rsrc(struct i40iw_device *iwdev)
1585 iwdev->used_pds = find_next_zero_bit(iwdev->allocated_pds, iwdev->max_pd, 0);
1586 iwdev->used_qps = find_next_zero_bit(iwdev->allocated_qps, iwdev->max_qp, 0);
1587 iwdev->used_cqs = find_next_zero_bit(iwdev->allocated_cqs, iwdev->max_cq, 0);
1588 iwdev->used_mrs = find_next_zero_bit(iwdev->allocated_mrs, iwdev->max_mr, 0);
1592 * i40iw_open - client interface operation open for iwarp/uda device
1593 * @ldev: lan device information
1594 * @client: iwarp client information, provided during registration
1596 * Called by the lan driver during the processing of client register
1597 * Create device resources, set up queues, pble and hmc objects and
1598 * register the device with the ib verbs interface
1599 * Return 0 if successful, otherwise return error
1601 static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
1603 struct i40iw_device *iwdev;
1604 struct i40iw_sc_dev *dev;
1605 enum i40iw_status_code status;
1606 struct i40iw_handler *hdl;
1608 hdl = i40iw_find_netdev(ldev->netdev);
1612 hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
1615 iwdev = &hdl->device;
1617 dev = &iwdev->sc_dev;
1618 i40iw_setup_cm_core(iwdev);
1620 dev->back_dev = (void *)iwdev;
1621 iwdev->ldev = &hdl->ldev;
1622 iwdev->client = client;
1623 mutex_init(&iwdev->pbl_mutex);
1624 i40iw_add_handler(hdl);
1627 status = i40iw_setup_init_state(hdl, ldev, client);
1630 iwdev->init_state = INITIAL_STATE;
1632 i40iw_wait_pe_ready(dev->hw);
1633 status = i40iw_create_cqp(iwdev);
1636 iwdev->init_state = CQP_CREATED;
1637 status = i40iw_hmc_setup(iwdev);
1640 status = i40iw_create_ccq(iwdev);
1643 iwdev->init_state = CCQ_CREATED;
1644 status = i40iw_initialize_ilq(iwdev);
1647 iwdev->init_state = ILQ_CREATED;
1648 status = i40iw_initialize_ieq(iwdev);
1651 iwdev->init_state = IEQ_CREATED;
1652 status = i40iw_setup_aeq(iwdev);
1655 iwdev->init_state = AEQ_CREATED;
1656 status = i40iw_setup_ceqs(iwdev, ldev);
1659 iwdev->init_state = CEQ_CREATED;
1660 status = i40iw_initialize_hw_resources(iwdev);
1663 i40iw_get_used_rsrc(iwdev);
1664 dev->ccq_ops->ccq_arm(dev->ccq);
1665 status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc);
1668 iwdev->init_state = PBLE_CHUNK_MEM;
1669 iwdev->virtchnl_wq = alloc_ordered_workqueue("iwvch", WQ_MEM_RECLAIM);
1670 status = i40iw_add_mac_ip(iwdev);
1673 iwdev->init_state = IP_ADDR_REGISTERED;
1674 if (i40iw_register_rdma_device(iwdev)) {
1675 i40iw_pr_err("register rdma device fail\n");
1679 iwdev->init_state = RDMA_DEV_REGISTERED;
1680 iwdev->iw_status = 1;
1681 i40iw_port_ibevent(iwdev);
1682 iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);
1683 if(iwdev->param_wq == NULL)
1685 i40iw_pr_info("i40iw_open completed\n");
1689 i40iw_pr_err("status = %d last completion = %d\n", status, iwdev->init_state);
1690 i40iw_deinit_device(iwdev);
1695 * i40iw_l2params_worker - worker for l2 params change
1696 * @work: work pointer for l2 params
1698 static void i40iw_l2params_worker(struct work_struct *work)
1700 struct l2params_work *dwork =
1701 container_of(work, struct l2params_work, work);
1702 struct i40iw_device *iwdev = dwork->iwdev;
1704 i40iw_change_l2params(&iwdev->vsi, &dwork->l2params);
1705 atomic_dec(&iwdev->params_busy);
1710 * i40iw_l2param_change - handle qs handles for qos and mss change
1711 * @ldev: lan device information
1712 * @client: client for paramater change
1713 * @params: new parameters from L2
1715 static void i40iw_l2param_change(struct i40e_info *ldev, struct i40e_client *client,
1716 struct i40e_params *params)
1718 struct i40iw_handler *hdl;
1719 struct i40iw_l2params *l2params;
1720 struct l2params_work *work;
1721 struct i40iw_device *iwdev;
1724 hdl = i40iw_find_i40e_handler(ldev);
1728 iwdev = &hdl->device;
1730 if (atomic_read(&iwdev->params_busy))
1734 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1738 atomic_inc(&iwdev->params_busy);
1740 work->iwdev = iwdev;
1741 l2params = &work->l2params;
1742 for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++)
1743 l2params->qs_handle_list[i] = params->qos.prio_qos[i].qs_handle;
1745 l2params->mss = (params->mtu) ? params->mtu - I40IW_MTU_TO_MSS : iwdev->vsi.mss;
1747 INIT_WORK(&work->work, i40iw_l2params_worker);
1748 queue_work(iwdev->param_wq, &work->work);
1752 * i40iw_close - client interface operation close for iwarp/uda device
1753 * @ldev: lan device information
1754 * @client: client to close
1756 * Called by the lan driver during the processing of client unregister
1757 * Destroy and clean up the driver resources
1759 static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool reset)
1761 struct i40iw_device *iwdev;
1762 struct i40iw_handler *hdl;
1764 hdl = i40iw_find_i40e_handler(ldev);
1768 iwdev = &hdl->device;
1769 iwdev->closing = true;
1772 iwdev->reset = true;
1774 i40iw_cm_disconnect_all(iwdev);
1775 destroy_workqueue(iwdev->virtchnl_wq);
1776 i40iw_deinit_device(iwdev);
1780 * i40iw_vf_reset - process VF reset
1781 * @ldev: lan device information
1782 * @client: client interface instance
1783 * @vf_id: virtual function id
1785 * Called when a VF is reset by the PF
1786 * Destroy and clean up the VF resources
1788 static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u32 vf_id)
1790 struct i40iw_handler *hdl;
1791 struct i40iw_sc_dev *dev;
1792 struct i40iw_hmc_fcn_info hmc_fcn_info;
1793 struct i40iw_virt_mem vf_dev_mem;
1794 struct i40iw_vfdev *tmp_vfdev;
1796 unsigned long flags;
1797 struct i40iw_device *iwdev;
1799 hdl = i40iw_find_i40e_handler(ldev);
1803 dev = &hdl->device.sc_dev;
1804 iwdev = (struct i40iw_device *)dev->back_dev;
1806 for (i = 0; i < I40IW_MAX_PE_ENABLED_VF_COUNT; i++) {
1807 if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id != vf_id))
1809 /* free all resources allocated on behalf of vf */
1810 tmp_vfdev = dev->vf_dev[i];
1811 spin_lock_irqsave(&iwdev->vsi.pestat->lock, flags);
1812 dev->vf_dev[i] = NULL;
1813 spin_unlock_irqrestore(&iwdev->vsi.pestat->lock, flags);
1814 i40iw_del_hmc_objects(dev, &tmp_vfdev->hmc_info, false, false);
1815 /* remove vf hmc function */
1816 memset(&hmc_fcn_info, 0, sizeof(hmc_fcn_info));
1817 hmc_fcn_info.vf_id = vf_id;
1818 hmc_fcn_info.iw_vf_idx = tmp_vfdev->iw_vf_idx;
1819 hmc_fcn_info.free_fcn = true;
1820 i40iw_cqp_manage_hmc_fcn_cmd(dev, &hmc_fcn_info);
1822 vf_dev_mem.va = tmp_vfdev;
1823 vf_dev_mem.size = sizeof(struct i40iw_vfdev) +
1824 sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX;
1825 i40iw_free_virt_mem(dev->hw, &vf_dev_mem);
1831 * i40iw_vf_enable - enable a number of VFs
1832 * @ldev: lan device information
1833 * @client: client interface instance
1834 * @num_vfs: number of VFs for the PF
1836 * Called when the number of VFs changes
1838 static void i40iw_vf_enable(struct i40e_info *ldev,
1839 struct i40e_client *client,
1842 struct i40iw_handler *hdl;
1844 hdl = i40iw_find_i40e_handler(ldev);
1848 if (num_vfs > I40IW_MAX_PE_ENABLED_VF_COUNT)
1849 hdl->device.max_enabled_vfs = I40IW_MAX_PE_ENABLED_VF_COUNT;
1851 hdl->device.max_enabled_vfs = num_vfs;
1855 * i40iw_vf_capable - check if VF capable
1856 * @ldev: lan device information
1857 * @client: client interface instance
1858 * @vf_id: virtual function id
1860 * Return 1 if a VF slot is available or if VF is already RDMA enabled
1861 * Return 0 otherwise
1863 static int i40iw_vf_capable(struct i40e_info *ldev,
1864 struct i40e_client *client,
1867 struct i40iw_handler *hdl;
1868 struct i40iw_sc_dev *dev;
1871 hdl = i40iw_find_i40e_handler(ldev);
1875 dev = &hdl->device.sc_dev;
1877 for (i = 0; i < hdl->device.max_enabled_vfs; i++) {
1878 if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id == vf_id))
1886 * i40iw_virtchnl_receive - receive a message through the virtual channel
1887 * @ldev: lan device information
1888 * @client: client interface instance
1889 * @vf_id: virtual function id associated with the message
1890 * @msg: message buffer pointer
1891 * @len: length of the message
1893 * Invoke virtual channel receive operation for the given msg
1894 * Return 0 if successful, otherwise return error
1896 static int i40iw_virtchnl_receive(struct i40e_info *ldev,
1897 struct i40e_client *client,
1902 struct i40iw_handler *hdl;
1903 struct i40iw_sc_dev *dev;
1904 struct i40iw_device *iwdev;
1905 int ret_code = I40IW_NOT_SUPPORTED;
1908 return I40IW_ERR_PARAM;
1910 hdl = i40iw_find_i40e_handler(ldev);
1912 return I40IW_ERR_PARAM;
1914 dev = &hdl->device.sc_dev;
1915 iwdev = dev->back_dev;
1917 if (dev->vchnl_if.vchnl_recv) {
1918 ret_code = dev->vchnl_if.vchnl_recv(dev, vf_id, msg, len);
1920 atomic_dec(&iwdev->vchnl_msgs);
1921 wake_up(&iwdev->vchnl_waitq);
1928 * i40iw_vf_clear_to_send - wait to send virtual channel message
1929 * @dev: iwarp device *
1930 * Wait for until virtual channel is clear
1931 * before sending the next message
1933 * Returns false if error
1934 * Returns true if clear to send
1936 bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev)
1938 struct i40iw_device *iwdev;
1939 wait_queue_entry_t wait;
1941 iwdev = dev->back_dev;
1943 if (!wq_has_sleeper(&dev->vf_reqs) &&
1944 (atomic_read(&iwdev->vchnl_msgs) == 0))
1945 return true; /* virtual channel is clear */
1948 add_wait_queue_exclusive(&dev->vf_reqs, &wait);
1950 if (!wait_event_timeout(dev->vf_reqs,
1951 (atomic_read(&iwdev->vchnl_msgs) == 0),
1952 I40IW_VCHNL_EVENT_TIMEOUT))
1953 dev->vchnl_up = false;
1955 remove_wait_queue(&dev->vf_reqs, &wait);
1957 return dev->vchnl_up;
1961 * i40iw_virtchnl_send - send a message through the virtual channel
1962 * @dev: iwarp device
1963 * @vf_id: virtual function id associated with the message
1964 * @msg: virtual channel message buffer pointer
1965 * @len: length of the message
1967 * Invoke virtual channel send operation for the given msg
1968 * Return 0 if successful, otherwise return error
1970 static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
1975 struct i40iw_device *iwdev;
1976 struct i40e_info *ldev;
1978 if (!dev || !dev->back_dev)
1979 return I40IW_ERR_BAD_PTR;
1981 iwdev = dev->back_dev;
1984 if (ldev && ldev->ops && ldev->ops->virtchnl_send)
1985 return ldev->ops->virtchnl_send(ldev, &i40iw_client, vf_id, msg, len);
1986 return I40IW_ERR_BAD_PTR;
1989 /* client interface functions */
1990 static const struct i40e_client_ops i40e_ops = {
1992 .close = i40iw_close,
1993 .l2_param_change = i40iw_l2param_change,
1994 .virtchnl_receive = i40iw_virtchnl_receive,
1995 .vf_reset = i40iw_vf_reset,
1996 .vf_enable = i40iw_vf_enable,
1997 .vf_capable = i40iw_vf_capable
2001 * i40iw_init_module - driver initialization function
2003 * First function to call when the driver is loaded
2004 * Register the driver as i40e client and port mapper client
2006 static int __init i40iw_init_module(void)
2010 memset(&i40iw_client, 0, sizeof(i40iw_client));
2011 i40iw_client.version.major = CLIENT_IW_INTERFACE_VERSION_MAJOR;
2012 i40iw_client.version.minor = CLIENT_IW_INTERFACE_VERSION_MINOR;
2013 i40iw_client.version.build = CLIENT_IW_INTERFACE_VERSION_BUILD;
2014 i40iw_client.ops = &i40e_ops;
2015 memcpy(i40iw_client.name, i40iw_client_name, I40E_CLIENT_STR_LENGTH);
2016 i40iw_client.type = I40E_CLIENT_IWARP;
2017 spin_lock_init(&i40iw_handler_lock);
2018 ret = i40e_register_client(&i40iw_client);
2019 i40iw_register_notifiers();
2025 * i40iw_exit_module - driver exit clean up function
2027 * The function is called just before the driver is unloaded
2028 * Unregister the driver as i40e client and port mapper client
2030 static void __exit i40iw_exit_module(void)
2032 i40iw_unregister_notifiers();
2033 i40e_unregister_client(&i40iw_client);
2036 module_init(i40iw_init_module);
2037 module_exit(i40iw_exit_module);