drivers/infiniband/hw/i40iw/i40iw_main.c

   1 /*******************************************************************************
   2 *
   3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenFabrics.org BSD license below:
  10 *
  11 *   Redistribution and use in source and binary forms, with or
  12 *   without modification, are permitted provided that the following
  13 *   conditions are met:
  14 *
  15 *    - Redistributions of source code must retain the above
  16 *       copyright notice, this list of conditions and the following
  17 *       disclaimer.
  18 *
  19 *    - Redistributions in binary form must reproduce the above
  20 *       copyright notice, this list of conditions and the following
  21 *       disclaimer in the documentation and/or other materials
  22 *       provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 *******************************************************************************/
  34
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/ip.h>
  40 #include <linux/tcp.h>
  41 #include <linux/if_vlan.h>
  42 #include <net/addrconf.h>
  43
  44 #include "i40iw.h"
  45 #include "i40iw_register.h"
  46 #include <net/netevent.h>
  47 #define CLIENT_IW_INTERFACE_VERSION_MAJOR 0
  48 #define CLIENT_IW_INTERFACE_VERSION_MINOR 01
  49 #define CLIENT_IW_INTERFACE_VERSION_BUILD 00
  50
  51 #define DRV_VERSION_MAJOR 0
  52 #define DRV_VERSION_MINOR 5
  53 #define DRV_VERSION_BUILD 123
  54 #define DRV_VERSION     __stringify(DRV_VERSION_MAJOR) "."              \
  55         __stringify(DRV_VERSION_MINOR) "." __stringify(DRV_VERSION_BUILD)
  56
  57 static int debug;
  58 module_param(debug, int, 0644);
  59 MODULE_PARM_DESC(debug, "debug flags: 0=disabled (default), 0x7fffffff=all");
  60
  61 static int resource_profile;
  62 module_param(resource_profile, int, 0644);
  63 MODULE_PARM_DESC(resource_profile,
  64                  "Resource Profile: 0=no VF RDMA support (default), 1=Weighted VF, 2=Even Distribution");
  65
  66 static int max_rdma_vfs = 32;
  67 module_param(max_rdma_vfs, int, 0644);
  68 MODULE_PARM_DESC(max_rdma_vfs, "Maximum VF count: 0-32 32=default");
  69 static int mpa_version = 2;
  70 module_param(mpa_version, int, 0644);
  71 MODULE_PARM_DESC(mpa_version, "MPA version to be used in MPA Req/Resp 1 or 2");
  72
  73 MODULE_AUTHOR("Intel Corporation, <e1000-rdma@lists.sourceforge.net>");
  74 MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver");
  75 MODULE_LICENSE("Dual BSD/GPL");
  76
  77 static struct i40e_client i40iw_client;
  78 static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw";
  79
  80 static LIST_HEAD(i40iw_handlers);
  81 static spinlock_t i40iw_handler_lock;
  82
  83 static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
  84                                                   u32 vf_id, u8 *msg, u16 len);
  85
  86 static struct notifier_block i40iw_inetaddr_notifier = {
  87         .notifier_call = i40iw_inetaddr_event
  88 };
  89
  90 static struct notifier_block i40iw_inetaddr6_notifier = {
  91         .notifier_call = i40iw_inet6addr_event
  92 };
  93
  94 static struct notifier_block i40iw_net_notifier = {
  95         .notifier_call = i40iw_net_event
  96 };
  97
  98 /**
  99  * i40iw_find_i40e_handler - find a handler given a client info
 100  * @ldev: pointer to a client info
 101  */
 102 static struct i40iw_handler *i40iw_find_i40e_handler(struct i40e_info *ldev)
 103 {
 104         struct i40iw_handler *hdl;
 105         unsigned long flags;
 106
 107         spin_lock_irqsave(&i40iw_handler_lock, flags);
 108         list_for_each_entry(hdl, &i40iw_handlers, list) {
 109                 if (hdl->ldev.netdev == ldev->netdev) {
 110                         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 111                         return hdl;
 112                 }
 113         }
 114         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 115         return NULL;
 116 }
 117
 118 /**
 119  * i40iw_find_netdev - find a handler given a netdev
 120  * @netdev: pointer to net_device
 121  */
 122 struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev)
 123 {
 124         struct i40iw_handler *hdl;
 125         unsigned long flags;
 126
 127         spin_lock_irqsave(&i40iw_handler_lock, flags);
 128         list_for_each_entry(hdl, &i40iw_handlers, list) {
 129                 if (hdl->ldev.netdev == netdev) {
 130                         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 131                         return hdl;
 132                 }
 133         }
 134         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 135         return NULL;
 136 }
 137
 138 /**
 139  * i40iw_add_handler - add a handler to the list
 140  * @hdl: handler to be added to the handler list
 141  */
 142 static void i40iw_add_handler(struct i40iw_handler *hdl)
 143 {
 144         unsigned long flags;
 145
 146         spin_lock_irqsave(&i40iw_handler_lock, flags);
 147         list_add(&hdl->list, &i40iw_handlers);
 148         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 149 }
 150
 151 /**
 152  * i40iw_del_handler - delete a handler from the list
 153  * @hdl: handler to be deleted from the handler list
 154  */
 155 static int i40iw_del_handler(struct i40iw_handler *hdl)
 156 {
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&i40iw_handler_lock, flags);
 160         list_del(&hdl->list);
 161         spin_unlock_irqrestore(&i40iw_handler_lock, flags);
 162         return 0;
 163 }
 164
 165 /**
 166  * i40iw_enable_intr - set up device interrupts
 167  * @dev: hardware control device structure
 168  * @msix_id: id of the interrupt to be enabled
 169  */
 170 static void i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id)
 171 {
 172         u32 val;
 173
 174         val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
 175                 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
 176                 (3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
 177         if (dev->is_pf)
 178                 i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_id - 1), val);
 179         else
 180                 i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_id - 1), val);
 181 }
 182
 183 /**
 184  * i40iw_dpc - tasklet for aeq and ceq 0
 185  * @data: iwarp device
 186  */
 187 static void i40iw_dpc(unsigned long data)
 188 {
 189         struct i40iw_device *iwdev = (struct i40iw_device *)data;
 190
 191         if (iwdev->msix_shared)
 192                 i40iw_process_ceq(iwdev, iwdev->ceqlist);
 193         i40iw_process_aeq(iwdev);
 194         i40iw_enable_intr(&iwdev->sc_dev, iwdev->iw_msixtbl[0].idx);
 195 }
 196
 197 /**
 198  * i40iw_ceq_dpc - dpc handler for CEQ
 199  * @data: data points to CEQ
 200  */
 201 static void i40iw_ceq_dpc(unsigned long data)
 202 {
 203         struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
 204         struct i40iw_device *iwdev = iwceq->iwdev;
 205
 206         i40iw_process_ceq(iwdev, iwceq);
 207         i40iw_enable_intr(&iwdev->sc_dev, iwceq->msix_idx);
 208 }
 209
 210 /**
 211  * i40iw_irq_handler - interrupt handler for aeq and ceq0
 212  * @irq: Interrupt request number
 213  * @data: iwarp device
 214  */
 215 static irqreturn_t i40iw_irq_handler(int irq, void *data)
 216 {
 217         struct i40iw_device *iwdev = (struct i40iw_device *)data;
 218
 219         tasklet_schedule(&iwdev->dpc_tasklet);
 220         return IRQ_HANDLED;
 221 }
 222
 223 /**
 224  * i40iw_destroy_cqp  - destroy control qp
 225  * @iwdev: iwarp device
 226  * @create_done: 1 if cqp create poll was success
 227  *
 228  * Issue destroy cqp request and
 229  * free the resources associated with the cqp
 230  */
 231 static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp)
 232 {
 233         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 234         struct i40iw_cqp *cqp = &iwdev->cqp;
 235
 236         if (free_hwcqp)
 237                 dev->cqp_ops->cqp_destroy(dev->cqp);
 238
 239         i40iw_cleanup_pending_cqp_op(iwdev);
 240
 241         i40iw_free_dma_mem(dev->hw, &cqp->sq);
 242         kfree(cqp->scratch_array);
 243         iwdev->cqp.scratch_array = NULL;
 244
 245         kfree(cqp->cqp_requests);
 246         cqp->cqp_requests = NULL;
 247 }
 248
 249 /**
 250  * i40iw_disable_irqs - disable device interrupts
 251  * @dev: hardware control device structure
 252  * @msic_vec: msix vector to disable irq
 253  * @dev_id: parameter to pass to free_irq (used during irq setup)
 254  *
 255  * The function is called when destroying aeq/ceq
 256  */
 257 static void i40iw_disable_irq(struct i40iw_sc_dev *dev,
 258                               struct i40iw_msix_vector *msix_vec,
 259                               void *dev_id)
 260 {
 261         if (dev->is_pf)
 262                 i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0);
 263         else
 264                 i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_vec->idx - 1), 0);
 265         irq_set_affinity_hint(msix_vec->irq, NULL);
 266         free_irq(msix_vec->irq, dev_id);
 267 }
 268
 269 /**
 270  * i40iw_destroy_aeq - destroy aeq
 271  * @iwdev: iwarp device
 272  *
 273  * Issue a destroy aeq request and
 274  * free the resources associated with the aeq
 275  * The function is called during driver unload
 276  */
 277 static void i40iw_destroy_aeq(struct i40iw_device *iwdev)
 278 {
 279         enum i40iw_status_code status = I40IW_ERR_NOT_READY;
 280         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 281         struct i40iw_aeq *aeq = &iwdev->aeq;
 282
 283         if (!iwdev->msix_shared)
 284                 i40iw_disable_irq(dev, iwdev->iw_msixtbl, (void *)iwdev);
 285         if (iwdev->reset)
 286                 goto exit;
 287
 288         if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1))
 289                 status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq);
 290         if (status)
 291                 i40iw_pr_err("destroy aeq failed %d\n", status);
 292
 293 exit:
 294         i40iw_free_dma_mem(dev->hw, &aeq->mem);
 295 }
 296
 297 /**
 298  * i40iw_destroy_ceq - destroy ceq
 299  * @iwdev: iwarp device
 300  * @iwceq: ceq to be destroyed
 301  *
 302  * Issue a destroy ceq request and
 303  * free the resources associated with the ceq
 304  */
 305 static void i40iw_destroy_ceq(struct i40iw_device *iwdev,
 306                               struct i40iw_ceq *iwceq)
 307 {
 308         enum i40iw_status_code status;
 309         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 310
 311         if (iwdev->reset)
 312                 goto exit;
 313
 314         status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1);
 315         if (status) {
 316                 i40iw_pr_err("ceq destroy command failed %d\n", status);
 317                 goto exit;
 318         }
 319
 320         status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq);
 321         if (status)
 322                 i40iw_pr_err("ceq destroy completion failed %d\n", status);
 323 exit:
 324         i40iw_free_dma_mem(dev->hw, &iwceq->mem);
 325 }
 326
 327 /**
 328  * i40iw_dele_ceqs - destroy all ceq's
 329  * @iwdev: iwarp device
 330  *
 331  * Go through all of the device ceq's and for each ceq
 332  * disable the ceq interrupt and destroy the ceq
 333  */
 334 static void i40iw_dele_ceqs(struct i40iw_device *iwdev)
 335 {
 336         u32 i = 0;
 337         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 338         struct i40iw_ceq *iwceq = iwdev->ceqlist;
 339         struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
 340
 341         if (iwdev->msix_shared) {
 342                 i40iw_disable_irq(dev, msix_vec, (void *)iwdev);
 343                 i40iw_destroy_ceq(iwdev, iwceq);
 344                 iwceq++;
 345                 i++;
 346         }
 347
 348         for (msix_vec++; i < iwdev->ceqs_count; i++, msix_vec++, iwceq++) {
 349                 i40iw_disable_irq(dev, msix_vec, (void *)iwceq);
 350                 i40iw_destroy_ceq(iwdev, iwceq);
 351         }
 352 }
 353
 354 /**
 355  * i40iw_destroy_ccq - destroy control cq
 356  * @iwdev: iwarp device
 357  *
 358  * Issue destroy ccq request and
 359  * free the resources associated with the ccq
 360  */
 361 static void i40iw_destroy_ccq(struct i40iw_device *iwdev)
 362 {
 363         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 364         struct i40iw_ccq *ccq = &iwdev->ccq;
 365         enum i40iw_status_code status = 0;
 366
 367         if (!iwdev->reset)
 368                 status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, true);
 369         if (status)
 370                 i40iw_pr_err("ccq destroy failed %d\n", status);
 371         i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
 372 }
 373
 374 /* types of hmc objects */
 375 static enum i40iw_hmc_rsrc_type iw_hmc_obj_types[] = {
 376         I40IW_HMC_IW_QP,
 377         I40IW_HMC_IW_CQ,
 378         I40IW_HMC_IW_HTE,
 379         I40IW_HMC_IW_ARP,
 380         I40IW_HMC_IW_APBVT_ENTRY,
 381         I40IW_HMC_IW_MR,
 382         I40IW_HMC_IW_XF,
 383         I40IW_HMC_IW_XFFL,
 384         I40IW_HMC_IW_Q1,
 385         I40IW_HMC_IW_Q1FL,
 386         I40IW_HMC_IW_TIMER,
 387 };
 388
 389 /**
 390  * i40iw_close_hmc_objects_type - delete hmc objects of a given type
 391  * @iwdev: iwarp device
 392  * @obj_type: the hmc object type to be deleted
 393  * @is_pf: true if the function is PF otherwise false
 394  * @reset: true if called before reset
 395  */
 396 static void i40iw_close_hmc_objects_type(struct i40iw_sc_dev *dev,
 397                                          enum i40iw_hmc_rsrc_type obj_type,
 398                                          struct i40iw_hmc_info *hmc_info,
 399                                          bool is_pf,
 400                                          bool reset)
 401 {
 402         struct i40iw_hmc_del_obj_info info;
 403
 404         memset(&info, 0, sizeof(info));
 405         info.hmc_info = hmc_info;
 406         info.rsrc_type = obj_type;
 407         info.count = hmc_info->hmc_obj[obj_type].cnt;
 408         info.is_pf = is_pf;
 409         if (dev->hmc_ops->del_hmc_object(dev, &info, reset))
 410                 i40iw_pr_err("del obj of type %d failed\n", obj_type);
 411 }
 412
 413 /**
 414  * i40iw_del_hmc_objects - remove all device hmc objects
 415  * @dev: iwarp device
 416  * @hmc_info: hmc_info to free
 417  * @is_pf: true if hmc_info belongs to PF, not vf nor allocated
 418  *         by PF on behalf of VF
 419  * @reset: true if called before reset
 420  */
 421 static void i40iw_del_hmc_objects(struct i40iw_sc_dev *dev,
 422                                   struct i40iw_hmc_info *hmc_info,
 423                                   bool is_pf,
 424                                   bool reset)
 425 {
 426         unsigned int i;
 427
 428         for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++)
 429                 i40iw_close_hmc_objects_type(dev, iw_hmc_obj_types[i], hmc_info, is_pf, reset);
 430 }
 431
 432 /**
 433  * i40iw_ceq_handler - interrupt handler for ceq
 434  * @data: ceq pointer
 435  */
 436 static irqreturn_t i40iw_ceq_handler(int irq, void *data)
 437 {
 438         struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
 439
 440         if (iwceq->irq != irq)
 441                 i40iw_pr_err("expected irq = %d received irq = %d\n", iwceq->irq, irq);
 442         tasklet_schedule(&iwceq->dpc_tasklet);
 443         return IRQ_HANDLED;
 444 }
 445
 446 /**
 447  * i40iw_create_hmc_obj_type - create hmc object of a given type
 448  * @dev: hardware control device structure
 449  * @info: information for the hmc object to create
 450  */
 451 static enum i40iw_status_code i40iw_create_hmc_obj_type(struct i40iw_sc_dev *dev,
 452                                                         struct i40iw_hmc_create_obj_info *info)
 453 {
 454         return dev->hmc_ops->create_hmc_object(dev, info);
 455 }
 456
 457 /**
 458  * i40iw_create_hmc_objs - create all hmc objects for the device
 459  * @iwdev: iwarp device
 460  * @is_pf: true if the function is PF otherwise false
 461  *
 462  * Create the device hmc objects and allocate hmc pages
 463  * Return 0 if successful, otherwise clean up and return error
 464  */
 465 static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev,
 466                                                     bool is_pf)
 467 {
 468         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 469         struct i40iw_hmc_create_obj_info info;
 470         enum i40iw_status_code status;
 471         int i;
 472
 473         memset(&info, 0, sizeof(info));
 474         info.hmc_info = dev->hmc_info;
 475         info.is_pf = is_pf;
 476         info.entry_type = iwdev->sd_type;
 477         for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
 478                 info.rsrc_type = iw_hmc_obj_types[i];
 479                 info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
 480                 status = i40iw_create_hmc_obj_type(dev, &info);
 481                 if (status) {
 482                         i40iw_pr_err("create obj type %d status = %d\n",
 483                                      iw_hmc_obj_types[i], status);
 484                         break;
 485                 }
 486         }
 487         if (!status)
 488                 return (dev->cqp_misc_ops->static_hmc_pages_allocated(dev->cqp, 0,
 489                                                                       dev->hmc_fn_id,
 490                                                                       true, true));
 491
 492         while (i) {
 493                 i--;
 494                 /* destroy the hmc objects of a given type */
 495                 i40iw_close_hmc_objects_type(dev,
 496                                              iw_hmc_obj_types[i],
 497                                              dev->hmc_info,
 498                                              is_pf,
 499                                              false);
 500         }
 501         return status;
 502 }
 503
 504 /**
 505  * i40iw_obj_aligned_mem - get aligned memory from device allocated memory
 506  * @iwdev: iwarp device
 507  * @memptr: points to the memory addresses
 508  * @size: size of memory needed
 509  * @mask: mask for the aligned memory
 510  *
 511  * Get aligned memory of the requested size and
 512  * update the memptr to point to the new aligned memory
 513  * Return 0 if successful, otherwise return no memory error
 514  */
 515 enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,
 516                                              struct i40iw_dma_mem *memptr,
 517                                              u32 size,
 518                                              u32 mask)
 519 {
 520         unsigned long va, newva;
 521         unsigned long extra;
 522
 523         va = (unsigned long)iwdev->obj_next.va;
 524         newva = va;
 525         if (mask)
 526                 newva = ALIGN(va, (mask + 1));
 527         extra = newva - va;
 528         memptr->va = (u8 *)va + extra;
 529         memptr->pa = iwdev->obj_next.pa + extra;
 530         memptr->size = size;
 531         if ((memptr->va + size) > (iwdev->obj_mem.va + iwdev->obj_mem.size))
 532                 return I40IW_ERR_NO_MEMORY;
 533
 534         iwdev->obj_next.va = memptr->va + size;
 535         iwdev->obj_next.pa = memptr->pa + size;
 536         return 0;
 537 }
 538
 539 /**
 540  * i40iw_create_cqp - create control qp
 541  * @iwdev: iwarp device
 542  *
 543  * Return 0, if the cqp and all the resources associated with it
 544  * are successfully created, otherwise return error
 545  */
 546 static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)
 547 {
 548         enum i40iw_status_code status;
 549         u32 sqsize = I40IW_CQP_SW_SQSIZE_2048;
 550         struct i40iw_dma_mem mem;
 551         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 552         struct i40iw_cqp_init_info cqp_init_info;
 553         struct i40iw_cqp *cqp = &iwdev->cqp;
 554         u16 maj_err, min_err;
 555         int i;
 556
 557         cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
 558         if (!cqp->cqp_requests)
 559                 return I40IW_ERR_NO_MEMORY;
 560         cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
 561         if (!cqp->scratch_array) {
 562                 kfree(cqp->cqp_requests);
 563                 return I40IW_ERR_NO_MEMORY;
 564         }
 565         dev->cqp = &cqp->sc_cqp;
 566         dev->cqp->dev = dev;
 567         memset(&cqp_init_info, 0, sizeof(cqp_init_info));
 568         status = i40iw_allocate_dma_mem(dev->hw, &cqp->sq,
 569                                         (sizeof(struct i40iw_cqp_sq_wqe) * sqsize),
 570                                         I40IW_CQP_ALIGNMENT);
 571         if (status)
 572                 goto exit;
 573         status = i40iw_obj_aligned_mem(iwdev, &mem, sizeof(struct i40iw_cqp_ctx),
 574                                        I40IW_HOST_CTX_ALIGNMENT_MASK);
 575         if (status)
 576                 goto exit;
 577         dev->cqp->host_ctx_pa = mem.pa;
 578         dev->cqp->host_ctx = mem.va;
 579         /* populate the cqp init info */
 580         cqp_init_info.dev = dev;
 581         cqp_init_info.sq_size = sqsize;
 582         cqp_init_info.sq = cqp->sq.va;
 583         cqp_init_info.sq_pa = cqp->sq.pa;
 584         cqp_init_info.host_ctx_pa = mem.pa;
 585         cqp_init_info.host_ctx = mem.va;
 586         cqp_init_info.hmc_profile = iwdev->resource_profile;
 587         cqp_init_info.enabled_vf_count = iwdev->max_rdma_vfs;
 588         cqp_init_info.scratch_array = cqp->scratch_array;
 589         status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info);
 590         if (status) {
 591                 i40iw_pr_err("cqp init status %d\n", status);
 592                 goto exit;
 593         }
 594         status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
 595         if (status) {
 596                 i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",
 597                              status, maj_err, min_err);
 598                 goto exit;
 599         }
 600         spin_lock_init(&cqp->req_lock);
 601         INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
 602         INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
 603         /* init the waitq of the cqp_requests and add them to the list */
 604         for (i = 0; i < I40IW_CQP_SW_SQSIZE_2048; i++) {
 605                 init_waitqueue_head(&cqp->cqp_requests[i].waitq);
 606                 list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
 607         }
 608         return 0;
 609 exit:
 610         /* clean up the created resources */
 611         i40iw_destroy_cqp(iwdev, false);
 612         return status;
 613 }
 614
 615 /**
 616  * i40iw_create_ccq - create control cq
 617  * @iwdev: iwarp device
 618  *
 619  * Return 0, if the ccq and the resources associated with it
 620  * are successfully created, otherwise return error
 621  */
 622 static enum i40iw_status_code i40iw_create_ccq(struct i40iw_device *iwdev)
 623 {
 624         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 625         struct i40iw_dma_mem mem;
 626         enum i40iw_status_code status;
 627         struct i40iw_ccq_init_info info;
 628         struct i40iw_ccq *ccq = &iwdev->ccq;
 629
 630         memset(&info, 0, sizeof(info));
 631         dev->ccq = &ccq->sc_cq;
 632         dev->ccq->dev = dev;
 633         info.dev = dev;
 634         ccq->shadow_area.size = sizeof(struct i40iw_cq_shadow_area);
 635         ccq->mem_cq.size = sizeof(struct i40iw_cqe) * IW_CCQ_SIZE;
 636         status = i40iw_allocate_dma_mem(dev->hw, &ccq->mem_cq,
 637                                         ccq->mem_cq.size, I40IW_CQ0_ALIGNMENT);
 638         if (status)
 639                 goto exit;
 640         status = i40iw_obj_aligned_mem(iwdev, &mem, ccq->shadow_area.size,
 641                                        I40IW_SHADOWAREA_MASK);
 642         if (status)
 643                 goto exit;
 644         ccq->sc_cq.back_cq = (void *)ccq;
 645         /* populate the ccq init info */
 646         info.cq_base = ccq->mem_cq.va;
 647         info.cq_pa = ccq->mem_cq.pa;
 648         info.num_elem = IW_CCQ_SIZE;
 649         info.shadow_area = mem.va;
 650         info.shadow_area_pa = mem.pa;
 651         info.ceqe_mask = false;
 652         info.ceq_id_valid = true;
 653         info.shadow_read_threshold = 16;
 654         status = dev->ccq_ops->ccq_init(dev->ccq, &info);
 655         if (!status)
 656                 status = dev->ccq_ops->ccq_create(dev->ccq, 0, true, true);
 657 exit:
 658         if (status)
 659                 i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
 660         return status;
 661 }
 662
 663 /**
 664  * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq
 665  * @iwdev: iwarp device
 666  * @msix_vec: interrupt vector information
 667  * @iwceq: ceq associated with the vector
 668  * @ceq_id: the id number of the iwceq
 669  *
 670  * Allocate interrupt resources and enable irq handling
 671  * Return 0 if successful, otherwise return error
 672  */
 673 static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iwdev,
 674                                                          struct i40iw_ceq *iwceq,
 675                                                          u32 ceq_id,
 676                                                          struct i40iw_msix_vector *msix_vec)
 677 {
 678         enum i40iw_status_code status;
 679
 680         if (iwdev->msix_shared && !ceq_id) {
 681                 tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
 682                 status = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "AEQCEQ", iwdev);
 683         } else {
 684                 tasklet_init(&iwceq->dpc_tasklet, i40iw_ceq_dpc, (unsigned long)iwceq);
 685                 status = request_irq(msix_vec->irq, i40iw_ceq_handler, 0, "CEQ", iwceq);
 686         }
 687
 688         cpumask_clear(&msix_vec->mask);
 689         cpumask_set_cpu(msix_vec->cpu_affinity, &msix_vec->mask);
 690         irq_set_affinity_hint(msix_vec->irq, &msix_vec->mask);
 691
 692         if (status) {
 693                 i40iw_pr_err("ceq irq config fail\n");
 694                 return I40IW_ERR_CONFIG;
 695         }
 696         msix_vec->ceq_id = ceq_id;
 697
 698         return 0;
 699 }
 700
 701 /**
 702  * i40iw_create_ceq - create completion event queue
 703  * @iwdev: iwarp device
 704  * @iwceq: pointer to the ceq resources to be created
 705  * @ceq_id: the id number of the iwceq
 706  *
 707  * Return 0, if the ceq and the resources associated with it
 708  * are successfully created, otherwise return error
 709  */
 710 static enum i40iw_status_code i40iw_create_ceq(struct i40iw_device *iwdev,
 711                                                struct i40iw_ceq *iwceq,
 712                                                u32 ceq_id)
 713 {
 714         enum i40iw_status_code status;
 715         struct i40iw_ceq_init_info info;
 716         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 717         u64 scratch;
 718
 719         memset(&info, 0, sizeof(info));
 720         info.ceq_id = ceq_id;
 721         iwceq->iwdev = iwdev;
 722         iwceq->mem.size = sizeof(struct i40iw_ceqe) *
 723                 iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
 724         status = i40iw_allocate_dma_mem(dev->hw, &iwceq->mem, iwceq->mem.size,
 725                                         I40IW_CEQ_ALIGNMENT);
 726         if (status)
 727                 goto exit;
 728         info.ceq_id = ceq_id;
 729         info.ceqe_base = iwceq->mem.va;
 730         info.ceqe_pa = iwceq->mem.pa;
 731
 732         info.elem_cnt = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
 733         iwceq->sc_ceq.ceq_id = ceq_id;
 734         info.dev = dev;
 735         scratch = (uintptr_t)&iwdev->cqp.sc_cqp;
 736         status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info);
 737         if (!status)
 738                 status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq, scratch);
 739
 740 exit:
 741         if (status)
 742                 i40iw_free_dma_mem(dev->hw, &iwceq->mem);
 743         return status;
 744 }
 745
 746 void i40iw_request_reset(struct i40iw_device *iwdev)
 747 {
 748         struct i40e_info *ldev = iwdev->ldev;
 749
 750         ldev->ops->request_reset(ldev, iwdev->client, 1);
 751 }
 752
 753 /**
 754  * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources
 755  * @iwdev: iwarp device
 756  * @ldev: i40e lan device
 757  *
 758  * Allocate a list for all device completion event queues
 759  * Create the ceq's and configure their msix interrupt vectors
 760  * Return 0, if at least one ceq is successfully set up, otherwise return error
 761  */
 762 static enum i40iw_status_code i40iw_setup_ceqs(struct i40iw_device *iwdev,
 763                                                struct i40e_info *ldev)
 764 {
 765         u32 i;
 766         u32 ceq_id;
 767         struct i40iw_ceq *iwceq;
 768         struct i40iw_msix_vector *msix_vec;
 769         enum i40iw_status_code status = 0;
 770         u32 num_ceqs;
 771
 772         if (ldev && ldev->ops && ldev->ops->setup_qvlist) {
 773                 status = ldev->ops->setup_qvlist(ldev, &i40iw_client,
 774                                                  iwdev->iw_qvlist);
 775                 if (status)
 776                         goto exit;
 777         } else {
 778                 status = I40IW_ERR_BAD_PTR;
 779                 goto exit;
 780         }
 781
 782         num_ceqs = min(iwdev->msix_count, iwdev->sc_dev.hmc_fpm_misc.max_ceqs);
 783         iwdev->ceqlist = kcalloc(num_ceqs, sizeof(*iwdev->ceqlist), GFP_KERNEL);
 784         if (!iwdev->ceqlist) {
 785                 status = I40IW_ERR_NO_MEMORY;
 786                 goto exit;
 787         }
 788         i = (iwdev->msix_shared) ? 0 : 1;
 789         for (ceq_id = 0; i < num_ceqs; i++, ceq_id++) {
 790                 iwceq = &iwdev->ceqlist[ceq_id];
 791                 status = i40iw_create_ceq(iwdev, iwceq, ceq_id);
 792                 if (status) {
 793                         i40iw_pr_err("create ceq status = %d\n", status);
 794                         break;
 795                 }
 796
 797                 msix_vec = &iwdev->iw_msixtbl[i];
 798                 iwceq->irq = msix_vec->irq;
 799                 iwceq->msix_idx = msix_vec->idx;
 800                 status = i40iw_configure_ceq_vector(iwdev, iwceq, ceq_id, msix_vec);
 801                 if (status) {
 802                         i40iw_destroy_ceq(iwdev, iwceq);
 803                         break;
 804                 }
 805                 i40iw_enable_intr(&iwdev->sc_dev, msix_vec->idx);
 806                 iwdev->ceqs_count++;
 807         }
 808
 809 exit:
 810         if (status) {
 811                 if (!iwdev->ceqs_count) {
 812                         kfree(iwdev->ceqlist);
 813                         iwdev->ceqlist = NULL;
 814                 } else {
 815                         status = 0;
 816                 }
 817         }
 818         return status;
 819 }
 820
 821 /**
 822  * i40iw_configure_aeq_vector - set up the msix vector for aeq
 823  * @iwdev: iwarp device
 824  *
 825  * Allocate interrupt resources and enable irq handling
 826  * Return 0 if successful, otherwise return error
 827  */
 828 static enum i40iw_status_code i40iw_configure_aeq_vector(struct i40iw_device *iwdev)
 829 {
 830         struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
 831         u32 ret = 0;
 832
 833         if (!iwdev->msix_shared) {
 834                 tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
 835                 ret = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "i40iw", iwdev);
 836         }
 837         if (ret) {
 838                 i40iw_pr_err("aeq irq config fail\n");
 839                 return I40IW_ERR_CONFIG;
 840         }
 841
 842         return 0;
 843 }
 844
 845 /**
 846  * i40iw_create_aeq - create async event queue
 847  * @iwdev: iwarp device
 848  *
 849  * Return 0, if the aeq and the resources associated with it
 850  * are successfully created, otherwise return error
 851  */
 852 static enum i40iw_status_code i40iw_create_aeq(struct i40iw_device *iwdev)
 853 {
 854         enum i40iw_status_code status;
 855         struct i40iw_aeq_init_info info;
 856         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 857         struct i40iw_aeq *aeq = &iwdev->aeq;
 858         u64 scratch = 0;
 859         u32 aeq_size;
 860
 861         aeq_size = 2 * iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt +
 862                 iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
 863         memset(&info, 0, sizeof(info));
 864         aeq->mem.size = sizeof(struct i40iw_sc_aeqe) * aeq_size;
 865         status = i40iw_allocate_dma_mem(dev->hw, &aeq->mem, aeq->mem.size,
 866                                         I40IW_AEQ_ALIGNMENT);
 867         if (status)
 868                 goto exit;
 869
 870         info.aeqe_base = aeq->mem.va;
 871         info.aeq_elem_pa = aeq->mem.pa;
 872         info.elem_cnt = aeq_size;
 873         info.dev = dev;
 874         status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info);
 875         if (status)
 876                 goto exit;
 877         status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1);
 878         if (!status)
 879                 status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq);
 880 exit:
 881         if (status)
 882                 i40iw_free_dma_mem(dev->hw, &aeq->mem);
 883         return status;
 884 }
 885
 886 /**
 887  * i40iw_setup_aeq - set up the device aeq
 888  * @iwdev: iwarp device
 889  *
 890  * Create the aeq and configure its msix interrupt vector
 891  * Return 0 if successful, otherwise return error
 892  */
 893 static enum i40iw_status_code i40iw_setup_aeq(struct i40iw_device *iwdev)
 894 {
 895         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 896         enum i40iw_status_code status;
 897
 898         status = i40iw_create_aeq(iwdev);
 899         if (status)
 900                 return status;
 901
 902         status = i40iw_configure_aeq_vector(iwdev);
 903         if (status) {
 904                 i40iw_destroy_aeq(iwdev);
 905                 return status;
 906         }
 907
 908         if (!iwdev->msix_shared)
 909                 i40iw_enable_intr(dev, iwdev->iw_msixtbl[0].idx);
 910         return 0;
 911 }
 912
 913 /**
 914  * i40iw_initialize_ilq - create iwarp local queue for cm
 915  * @iwdev: iwarp device
 916  *
 917  * Return 0 if successful, otherwise return error
 918  */
 919 static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev)
 920 {
 921         struct i40iw_puda_rsrc_info info;
 922         enum i40iw_status_code status;
 923
 924         memset(&info, 0, sizeof(info));
 925         info.type = I40IW_PUDA_RSRC_TYPE_ILQ;
 926         info.cq_id = 1;
 927         info.qp_id = 0;
 928         info.count = 1;
 929         info.pd_id = 1;
 930         info.sq_size = 8192;
 931         info.rq_size = 8192;
 932         info.buf_size = 1024;
 933         info.tx_buf_cnt = 16384;
 934         info.receive = i40iw_receive_ilq;
 935         info.xmit_complete = i40iw_free_sqbuf;
 936         status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
 937         if (status)
 938                 i40iw_pr_err("ilq create fail\n");
 939         return status;
 940 }
 941
 942 /**
 943  * i40iw_initialize_ieq - create iwarp exception queue
 944  * @iwdev: iwarp device
 945  *
 946  * Return 0 if successful, otherwise return error
 947  */
 948 static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev)
 949 {
 950         struct i40iw_puda_rsrc_info info;
 951         enum i40iw_status_code status;
 952
 953         memset(&info, 0, sizeof(info));
 954         info.type = I40IW_PUDA_RSRC_TYPE_IEQ;
 955         info.cq_id = 2;
 956         info.qp_id = iwdev->sc_dev.exception_lan_queue;
 957         info.count = 1;
 958         info.pd_id = 2;
 959         info.sq_size = 8192;
 960         info.rq_size = 8192;
 961         info.buf_size = 2048;
 962         info.tx_buf_cnt = 16384;
 963         status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
 964         if (status)
 965                 i40iw_pr_err("ieq create fail\n");
 966         return status;
 967 }
 968
 969 /**
 970  * i40iw_hmc_setup - create hmc objects for the device
 971  * @iwdev: iwarp device
 972  *
 973  * Set up the device private memory space for the number and size of
 974  * the hmc objects and create the objects
 975  * Return 0 if successful, otherwise return error
 976  */
 977 static enum i40iw_status_code i40iw_hmc_setup(struct i40iw_device *iwdev)
 978 {
 979         enum i40iw_status_code status;
 980
 981         iwdev->sd_type = I40IW_SD_TYPE_DIRECT;
 982         status = i40iw_config_fpm_values(&iwdev->sc_dev, IW_CFG_FPM_QP_COUNT);
 983         if (status)
 984                 goto exit;
 985         status = i40iw_create_hmc_objs(iwdev, true);
 986         if (status)
 987                 goto exit;
 988         iwdev->init_state = HMC_OBJS_CREATED;
 989 exit:
 990         return status;
 991 }
 992
 993 /**
 994  * i40iw_del_init_mem - deallocate memory resources
 995  * @iwdev: iwarp device
 996  */
 997 static void i40iw_del_init_mem(struct i40iw_device *iwdev)
 998 {
 999         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1000
1001         i40iw_free_dma_mem(&iwdev->hw, &iwdev->obj_mem);
1002         kfree(dev->hmc_info->sd_table.sd_entry);
1003         dev->hmc_info->sd_table.sd_entry = NULL;
1004         kfree(iwdev->mem_resources);
1005         iwdev->mem_resources = NULL;
1006         kfree(iwdev->ceqlist);
1007         iwdev->ceqlist = NULL;
1008         kfree(iwdev->iw_msixtbl);
1009         iwdev->iw_msixtbl = NULL;
1010         kfree(iwdev->hmc_info_mem);
1011         iwdev->hmc_info_mem = NULL;
1012 }
1013
1014 /**
1015  * i40iw_del_macip_entry - remove a mac ip address entry from the hw table
1016  * @iwdev: iwarp device
1017  * @idx: the index of the mac ip address to delete
1018  */
1019 static void i40iw_del_macip_entry(struct i40iw_device *iwdev, u8 idx)
1020 {
1021         struct i40iw_cqp *iwcqp = &iwdev->cqp;
1022         struct i40iw_cqp_request *cqp_request;
1023         struct cqp_commands_info *cqp_info;
1024         enum i40iw_status_code status = 0;
1025
1026         cqp_request = i40iw_get_cqp_request(iwcqp, true);
1027         if (!cqp_request) {
1028                 i40iw_pr_err("cqp_request memory failed\n");
1029                 return;
1030         }
1031         cqp_info = &cqp_request->info;
1032         cqp_info->cqp_cmd = OP_DELETE_LOCAL_MAC_IPADDR_ENTRY;
1033         cqp_info->post_sq = 1;
1034         cqp_info->in.u.del_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1035         cqp_info->in.u.del_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1036         cqp_info->in.u.del_local_mac_ipaddr_entry.entry_idx = idx;
1037         cqp_info->in.u.del_local_mac_ipaddr_entry.ignore_ref_count = 0;
1038         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1039         if (status)
1040                 i40iw_pr_err("CQP-OP Del MAC Ip entry fail");
1041 }
1042
1043 /**
1044  * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table
1045  * @iwdev: iwarp device
1046  * @mac_addr: pointer to mac address
1047  * @idx: the index of the mac ip address to add
1048  */
1049 static enum i40iw_status_code i40iw_add_mac_ipaddr_entry(struct i40iw_device *iwdev,
1050                                                          u8 *mac_addr,
1051                                                          u8 idx)
1052 {
1053         struct i40iw_local_mac_ipaddr_entry_info *info;
1054         struct i40iw_cqp *iwcqp = &iwdev->cqp;
1055         struct i40iw_cqp_request *cqp_request;
1056         struct cqp_commands_info *cqp_info;
1057         enum i40iw_status_code status = 0;
1058
1059         cqp_request = i40iw_get_cqp_request(iwcqp, true);
1060         if (!cqp_request) {
1061                 i40iw_pr_err("cqp_request memory failed\n");
1062                 return I40IW_ERR_NO_MEMORY;
1063         }
1064
1065         cqp_info = &cqp_request->info;
1066
1067         cqp_info->post_sq = 1;
1068         info = &cqp_info->in.u.add_local_mac_ipaddr_entry.info;
1069         ether_addr_copy(info->mac_addr, mac_addr);
1070         info->entry_idx = idx;
1071         cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1072         cqp_info->cqp_cmd = OP_ADD_LOCAL_MAC_IPADDR_ENTRY;
1073         cqp_info->in.u.add_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1074         cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1075         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1076         if (status)
1077                 i40iw_pr_err("CQP-OP Add MAC Ip entry fail");
1078         return status;
1079 }
1080
1081 /**
1082  * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry
1083  * @iwdev: iwarp device
1084  * @mac_ip_tbl_idx: the index of the new mac ip address
1085  *
1086  * Allocate a mac ip address entry and update the mac_ip_tbl_idx
1087  * to hold the index of the newly created mac ip address
1088  * Return 0 if successful, otherwise return error
1089  */
1090 static enum i40iw_status_code i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device *iwdev,
1091                                                                  u16 *mac_ip_tbl_idx)
1092 {
1093         struct i40iw_cqp *iwcqp = &iwdev->cqp;
1094         struct i40iw_cqp_request *cqp_request;
1095         struct cqp_commands_info *cqp_info;
1096         enum i40iw_status_code status = 0;
1097
1098         cqp_request = i40iw_get_cqp_request(iwcqp, true);
1099         if (!cqp_request) {
1100                 i40iw_pr_err("cqp_request memory failed\n");
1101                 return I40IW_ERR_NO_MEMORY;
1102         }
1103
1104         /* increment refcount, because we need the cqp request ret value */
1105         atomic_inc(&cqp_request->refcount);
1106
1107         cqp_info = &cqp_request->info;
1108         cqp_info->cqp_cmd = OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY;
1109         cqp_info->post_sq = 1;
1110         cqp_info->in.u.alloc_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
1111         cqp_info->in.u.alloc_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
1112         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1113         if (!status)
1114                 *mac_ip_tbl_idx = cqp_request->compl_info.op_ret_val;
1115         else
1116                 i40iw_pr_err("CQP-OP Alloc MAC Ip entry fail");
1117         /* decrement refcount and free the cqp request, if no longer used */
1118         i40iw_put_cqp_request(iwcqp, cqp_request);
1119         return status;
1120 }
1121
1122 /**
1123  * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry
1124  * @iwdev: iwarp device
1125  * @macaddr: pointer to mac address
1126  *
1127  * Allocate a mac ip address entry and add it to the hw table
1128  * Return 0 if successful, otherwise return error
1129  */
1130 static enum i40iw_status_code i40iw_alloc_set_mac_ipaddr(struct i40iw_device *iwdev,
1131                                                          u8 *macaddr)
1132 {
1133         enum i40iw_status_code status;
1134
1135         status = i40iw_alloc_local_mac_ipaddr_entry(iwdev, &iwdev->mac_ip_table_idx);
1136         if (!status) {
1137                 status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr,
1138                                                     (u8)iwdev->mac_ip_table_idx);
1139                 if (status)
1140                         i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
1141         }
1142         return status;
1143 }
1144
1145 /**
1146  * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table
1147  * @iwdev: iwarp device
1148  */
1149 static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev)
1150 {
1151         struct net_device *ip_dev;
1152         struct inet6_dev *idev;
1153         struct inet6_ifaddr *ifp, *tmp;
1154         u32 local_ipaddr6[4];
1155
1156         rcu_read_lock();
1157         for_each_netdev_rcu(&init_net, ip_dev) {
1158                 if ((((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF) &&
1159                       (rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev)) ||
1160                      (ip_dev == iwdev->netdev)) && (ip_dev->flags & IFF_UP)) {
1161                         idev = __in6_dev_get(ip_dev);
1162                         if (!idev) {
1163                                 i40iw_pr_err("ipv6 inet device not found\n");
1164                                 break;
1165                         }
1166                         list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) {
1167                                 i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &ifp->addr,
1168                                               rdma_vlan_dev_vlan_id(ip_dev), ip_dev->dev_addr);
1169                                 i40iw_copy_ip_ntohl(local_ipaddr6,
1170                                                     ifp->addr.in6_u.u6_addr32);
1171                                 i40iw_manage_arp_cache(iwdev,
1172                                                        ip_dev->dev_addr,
1173                                                        local_ipaddr6,
1174                                                        false,
1175                                                        I40IW_ARP_ADD);
1176                         }
1177                 }
1178         }
1179         rcu_read_unlock();
1180 }
1181
1182 /**
1183  * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table
1184  * @iwdev: iwarp device
1185  */
1186 static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev)
1187 {
1188         struct net_device *dev;
1189         struct in_device *idev;
1190         bool got_lock = true;
1191         u32 ip_addr;
1192
1193         if (!rtnl_trylock())
1194                 got_lock = false;
1195
1196         for_each_netdev(&init_net, dev) {
1197                 if ((((rdma_vlan_dev_vlan_id(dev) < 0xFFFF) &&
1198                       (rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) ||
1199                     (dev == iwdev->netdev)) && (dev->flags & IFF_UP)) {
1200                         idev = in_dev_get(dev);
1201                         for_ifa(idev) {
1202                                 i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
1203                                             "IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_address,
1204                                              rdma_vlan_dev_vlan_id(dev), dev->dev_addr);
1205
1206                                 ip_addr = ntohl(ifa->ifa_address);
1207                                 i40iw_manage_arp_cache(iwdev,
1208                                                        dev->dev_addr,
1209                                                        &ip_addr,
1210                                                        true,
1211                                                        I40IW_ARP_ADD);
1212                         }
1213                         endfor_ifa(idev);
1214                         in_dev_put(idev);
1215                 }
1216         }
1217         if (got_lock)
1218                 rtnl_unlock();
1219 }
1220
1221 /**
1222  * i40iw_add_mac_ip - add mac and ip addresses
1223  * @iwdev: iwarp device
1224  *
1225  * Create and add a mac ip address entry to the hw table and
1226  * ipv4/ipv6 addresses to the arp cache
1227  * Return 0 if successful, otherwise return error
1228  */
1229 static enum i40iw_status_code i40iw_add_mac_ip(struct i40iw_device *iwdev)
1230 {
1231         struct net_device *netdev = iwdev->netdev;
1232         enum i40iw_status_code status;
1233
1234         status = i40iw_alloc_set_mac_ipaddr(iwdev, (u8 *)netdev->dev_addr);
1235         if (status)
1236                 return status;
1237         i40iw_add_ipv4_addr(iwdev);
1238         i40iw_add_ipv6_addr(iwdev);
1239         return 0;
1240 }
1241
1242 /**
1243  * i40iw_wait_pe_ready - Check if firmware is ready
1244  * @hw: provides access to registers
1245  */
1246 static void i40iw_wait_pe_ready(struct i40iw_hw *hw)
1247 {
1248         u32 statusfw;
1249         u32 statuscpu0;
1250         u32 statuscpu1;
1251         u32 statuscpu2;
1252         u32 retrycount = 0;
1253
1254         do {
1255                 statusfw = i40iw_rd32(hw, I40E_GLPE_FWLDSTATUS);
1256                 i40iw_pr_info("[%04d] fm load status[x%04X]\n", __LINE__, statusfw);
1257                 statuscpu0 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS0);
1258                 i40iw_pr_info("[%04d] CSR_CQP status[x%04X]\n", __LINE__, statuscpu0);
1259                 statuscpu1 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS1);
1260                 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS1 status[x%04X]\n",
1261                               __LINE__, statuscpu1);
1262                 statuscpu2 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS2);
1263                 i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS2 status[x%04X]\n",
1264                               __LINE__, statuscpu2);
1265                 if ((statuscpu0 == 0x80) && (statuscpu1 == 0x80) && (statuscpu2 == 0x80))
1266                         break;  /* SUCCESS */
1267                 mdelay(1000);
1268                 retrycount++;
1269         } while (retrycount < 14);
1270         i40iw_wr32(hw, 0xb4040, 0x4C104C5);
1271 }
1272
1273 /**
1274  * i40iw_initialize_dev - initialize device
1275  * @iwdev: iwarp device
1276  * @ldev: lan device information
1277  *
1278  * Allocate memory for the hmc objects and initialize iwdev
1279  * Return 0 if successful, otherwise clean up the resources
1280  * and return error
1281  */
1282 static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev,
1283                                                    struct i40e_info *ldev)
1284 {
1285         enum i40iw_status_code status;
1286         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1287         struct i40iw_device_init_info info;
1288         struct i40iw_vsi_init_info vsi_info;
1289         struct i40iw_dma_mem mem;
1290         struct i40iw_l2params l2params;
1291         u32 size;
1292         struct i40iw_vsi_stats_info stats_info;
1293         u16 last_qset = I40IW_NO_QSET;
1294         u16 qset;
1295         u32 i;
1296
1297         memset(&l2params, 0, sizeof(l2params));
1298         memset(&info, 0, sizeof(info));
1299         size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) +
1300                                 (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX);
1301         iwdev->hmc_info_mem = kzalloc(size, GFP_KERNEL);
1302         if (!iwdev->hmc_info_mem)
1303                 return I40IW_ERR_NO_MEMORY;
1304
1305         iwdev->pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev->hmc_info_mem;
1306         dev->hmc_info = &iwdev->hw.hmc;
1307         dev->hmc_info->hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev->pble_rsrc + 1);
1308         status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_QUERY_FPM_BUF_SIZE,
1309                                        I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);
1310         if (status)
1311                 goto error;
1312         info.fpm_query_buf_pa = mem.pa;
1313         info.fpm_query_buf = mem.va;
1314         status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_COMMIT_FPM_BUF_SIZE,
1315                                        I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK);
1316         if (status)
1317                 goto error;
1318         info.fpm_commit_buf_pa = mem.pa;
1319         info.fpm_commit_buf = mem.va;
1320         info.hmc_fn_id = ldev->fid;
1321         info.is_pf = (ldev->ftype) ? false : true;
1322         info.bar0 = ldev->hw_addr;
1323         info.hw = &iwdev->hw;
1324         info.debug_mask = debug;
1325         l2params.mss =
1326                 (ldev->params.mtu) ? ldev->params.mtu - I40IW_MTU_TO_MSS : I40IW_DEFAULT_MSS;
1327         for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) {
1328                 qset = ldev->params.qos.prio_qos[i].qs_handle;
1329                 l2params.qs_handle_list[i] = qset;
1330                 if (last_qset == I40IW_NO_QSET)
1331                         last_qset = qset;
1332                 else if ((qset != last_qset) && (qset != I40IW_NO_QSET))
1333                         iwdev->dcb = true;
1334         }
1335         i40iw_pr_info("DCB is set/clear = %d\n", iwdev->dcb);
1336         info.exception_lan_queue = 1;
1337         info.vchnl_send = i40iw_virtchnl_send;
1338         status = i40iw_device_init(&iwdev->sc_dev, &info);
1339
1340         if (status)
1341                 goto error;
1342         memset(&vsi_info, 0, sizeof(vsi_info));
1343         vsi_info.dev = &iwdev->sc_dev;
1344         vsi_info.back_vsi = (void *)iwdev;
1345         vsi_info.params = &l2params;
1346         i40iw_sc_vsi_init(&iwdev->vsi, &vsi_info);
1347
1348         if (dev->is_pf) {
1349                 memset(&stats_info, 0, sizeof(stats_info));
1350                 stats_info.fcn_id = ldev->fid;
1351                 stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
1352                 if (!stats_info.pestat) {
1353                         status = I40IW_ERR_NO_MEMORY;
1354                         goto error;
1355                 }
1356                 stats_info.stats_initialize = true;
1357                 if (stats_info.pestat)
1358                         i40iw_vsi_stats_init(&iwdev->vsi, &stats_info);
1359         }
1360         return status;
1361 error:
1362         kfree(iwdev->hmc_info_mem);
1363         iwdev->hmc_info_mem = NULL;
1364         return status;
1365 }
1366
1367 /**
1368  * i40iw_register_notifiers - register tcp ip notifiers
1369  */
1370 static void i40iw_register_notifiers(void)
1371 {
1372         register_inetaddr_notifier(&i40iw_inetaddr_notifier);
1373         register_inet6addr_notifier(&i40iw_inetaddr6_notifier);
1374         register_netevent_notifier(&i40iw_net_notifier);
1375 }
1376
1377 /**
1378  * i40iw_unregister_notifiers - unregister tcp ip notifiers
1379  */
1380
1381 static void i40iw_unregister_notifiers(void)
1382 {
1383         unregister_netevent_notifier(&i40iw_net_notifier);
1384         unregister_inetaddr_notifier(&i40iw_inetaddr_notifier);
1385         unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier);
1386 }
1387
1388 /**
1389  * i40iw_save_msix_info - copy msix vector information to iwarp device
1390  * @iwdev: iwarp device
1391  * @ldev: lan device information
1392  *
1393  * Allocate iwdev msix table and copy the ldev msix info to the table
1394  * Return 0 if successful, otherwise return error
1395  */
1396 static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev,
1397                                                    struct i40e_info *ldev)
1398 {
1399         struct i40e_qvlist_info *iw_qvlist;
1400         struct i40e_qv_info *iw_qvinfo;
1401         u32 ceq_idx;
1402         u32 i;
1403         u32 size;
1404
1405         if (!ldev->msix_count) {
1406                 i40iw_pr_err("No MSI-X vectors\n");
1407                 return I40IW_ERR_CONFIG;
1408         }
1409
1410         iwdev->msix_count = ldev->msix_count;
1411
1412         size = sizeof(struct i40iw_msix_vector) * iwdev->msix_count;
1413         size += sizeof(struct i40e_qvlist_info);
1414         size +=  sizeof(struct i40e_qv_info) * iwdev->msix_count - 1;
1415         iwdev->iw_msixtbl = kzalloc(size, GFP_KERNEL);
1416
1417         if (!iwdev->iw_msixtbl)
1418                 return I40IW_ERR_NO_MEMORY;
1419         iwdev->iw_qvlist = (struct i40e_qvlist_info *)(&iwdev->iw_msixtbl[iwdev->msix_count]);
1420         iw_qvlist = iwdev->iw_qvlist;
1421         iw_qvinfo = iw_qvlist->qv_info;
1422         iw_qvlist->num_vectors = iwdev->msix_count;
1423         if (iwdev->msix_count <= num_online_cpus())
1424                 iwdev->msix_shared = true;
1425         for (i = 0, ceq_idx = 0; i < iwdev->msix_count; i++, iw_qvinfo++) {
1426                 iwdev->iw_msixtbl[i].idx = ldev->msix_entries[i].entry;
1427                 iwdev->iw_msixtbl[i].irq = ldev->msix_entries[i].vector;
1428                 iwdev->iw_msixtbl[i].cpu_affinity = ceq_idx;
1429                 if (i == 0) {
1430                         iw_qvinfo->aeq_idx = 0;
1431                         if (iwdev->msix_shared)
1432                                 iw_qvinfo->ceq_idx = ceq_idx++;
1433                         else
1434                                 iw_qvinfo->ceq_idx = I40E_QUEUE_INVALID_IDX;
1435                 } else {
1436                         iw_qvinfo->aeq_idx = I40E_QUEUE_INVALID_IDX;
1437                         iw_qvinfo->ceq_idx = ceq_idx++;
1438                 }
1439                 iw_qvinfo->itr_idx = 3;
1440                 iw_qvinfo->v_idx = iwdev->iw_msixtbl[i].idx;
1441         }
1442         return 0;
1443 }
1444
1445 /**
1446  * i40iw_deinit_device - clean up the device resources
1447  * @iwdev: iwarp device
1448  *
1449  * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses,
1450  * destroy the device queues and free the pble and the hmc objects
1451  */
1452 static void i40iw_deinit_device(struct i40iw_device *iwdev)
1453 {
1454         struct i40e_info *ldev = iwdev->ldev;
1455
1456         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1457
1458         i40iw_pr_info("state = %d\n", iwdev->init_state);
1459         if (iwdev->param_wq)
1460                 destroy_workqueue(iwdev->param_wq);
1461
1462         switch (iwdev->init_state) {
1463         case RDMA_DEV_REGISTERED:
1464                 iwdev->iw_status = 0;
1465                 i40iw_port_ibevent(iwdev);
1466                 i40iw_destroy_rdma_device(iwdev->iwibdev);
1467                 /* fallthrough */
1468         case IP_ADDR_REGISTERED:
1469                 if (!iwdev->reset)
1470                         i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
1471                 /* fallthrough */
1472                 /* fallthrough */
1473         case PBLE_CHUNK_MEM:
1474                 i40iw_destroy_pble_pool(dev, iwdev->pble_rsrc);
1475                 /* fallthrough */
1476         case CEQ_CREATED:
1477                 i40iw_dele_ceqs(iwdev);
1478                 /* fallthrough */
1479         case AEQ_CREATED:
1480                 i40iw_destroy_aeq(iwdev);
1481                 /* fallthrough */
1482         case IEQ_CREATED:
1483                 i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, iwdev->reset);
1484                 /* fallthrough */
1485         case ILQ_CREATED:
1486                 i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_ILQ, iwdev->reset);
1487                 /* fallthrough */
1488         case CCQ_CREATED:
1489                 i40iw_destroy_ccq(iwdev);
1490                 /* fallthrough */
1491         case HMC_OBJS_CREATED:
1492                 i40iw_del_hmc_objects(dev, dev->hmc_info, true, iwdev->reset);
1493                 /* fallthrough */
1494         case CQP_CREATED:
1495                 i40iw_destroy_cqp(iwdev, true);
1496                 /* fallthrough */
1497         case INITIAL_STATE:
1498                 i40iw_cleanup_cm_core(&iwdev->cm_core);
1499                 if (iwdev->vsi.pestat) {
1500                         i40iw_vsi_stats_free(&iwdev->vsi);
1501                         kfree(iwdev->vsi.pestat);
1502                 }
1503                 i40iw_del_init_mem(iwdev);
1504                 break;
1505         case INVALID_STATE:
1506                 /* fallthrough */
1507         default:
1508                 i40iw_pr_err("bad init_state = %d\n", iwdev->init_state);
1509                 break;
1510         }
1511
1512         i40iw_del_handler(i40iw_find_i40e_handler(ldev));
1513         kfree(iwdev->hdl);
1514 }
1515
1516 /**
1517  * i40iw_setup_init_state - set up the initial device struct
1518  * @hdl: handler for iwarp device - one per instance
1519  * @ldev: lan device information
1520  * @client: iwarp client information, provided during registration
1521  *
1522  * Initialize the iwarp device and its hdl information
1523  * using the ldev and client information
1524  * Return 0 if successful, otherwise return error
1525  */
1526 static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl,
1527                                                      struct i40e_info *ldev,
1528                                                      struct i40e_client *client)
1529 {
1530         struct i40iw_device *iwdev = &hdl->device;
1531         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1532         enum i40iw_status_code status;
1533
1534         memcpy(&hdl->ldev, ldev, sizeof(*ldev));
1535         if (resource_profile == 1)
1536                 resource_profile = 2;
1537
1538         iwdev->mpa_version = mpa_version;
1539         iwdev->resource_profile = (resource_profile < I40IW_HMC_PROFILE_EQUAL) ?
1540             (u8)resource_profile + I40IW_HMC_PROFILE_DEFAULT :
1541             I40IW_HMC_PROFILE_DEFAULT;
1542         iwdev->max_rdma_vfs =
1543                 (iwdev->resource_profile != I40IW_HMC_PROFILE_DEFAULT) ?  max_rdma_vfs : 0;
1544         iwdev->max_enabled_vfs = iwdev->max_rdma_vfs;
1545         iwdev->netdev = ldev->netdev;
1546         hdl->client = client;
1547         if (!ldev->ftype)
1548                 iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_DB_ADDR_OFFSET;
1549         else
1550                 iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_VF_DB_ADDR_OFFSET;
1551
1552         status = i40iw_save_msix_info(iwdev, ldev);
1553         if (status)
1554                 return status;
1555         iwdev->hw.dev_context = (void *)ldev->pcidev;
1556         iwdev->hw.hw_addr = ldev->hw_addr;
1557         status = i40iw_allocate_dma_mem(&iwdev->hw,
1558                                         &iwdev->obj_mem, 8192, 4096);
1559         if (status)
1560                 goto exit;
1561         iwdev->obj_next = iwdev->obj_mem;
1562
1563         init_waitqueue_head(&iwdev->vchnl_waitq);
1564         init_waitqueue_head(&dev->vf_reqs);
1565         init_waitqueue_head(&iwdev->close_wq);
1566
1567         status = i40iw_initialize_dev(iwdev, ldev);
1568 exit:
1569         if (status) {
1570                 kfree(iwdev->iw_msixtbl);
1571                 i40iw_free_dma_mem(dev->hw, &iwdev->obj_mem);
1572                 iwdev->iw_msixtbl = NULL;
1573         }
1574         return status;
1575 }
1576
1577 /**
1578  * i40iw_get_used_rsrc - determine resources used internally
1579  * @iwdev: iwarp device
1580  *
1581  * Called after internal allocations
1582  */
1583 static void i40iw_get_used_rsrc(struct i40iw_device *iwdev)
1584 {
1585         iwdev->used_pds = find_next_zero_bit(iwdev->allocated_pds, iwdev->max_pd, 0);
1586         iwdev->used_qps = find_next_zero_bit(iwdev->allocated_qps, iwdev->max_qp, 0);
1587         iwdev->used_cqs = find_next_zero_bit(iwdev->allocated_cqs, iwdev->max_cq, 0);
1588         iwdev->used_mrs = find_next_zero_bit(iwdev->allocated_mrs, iwdev->max_mr, 0);
1589 }
1590
1591 /**
1592  * i40iw_open - client interface operation open for iwarp/uda device
1593  * @ldev: lan device information
1594  * @client: iwarp client information, provided during registration
1595  *
1596  * Called by the lan driver during the processing of client register
1597  * Create device resources, set up queues, pble and hmc objects and
1598  * register the device with the ib verbs interface
1599  * Return 0 if successful, otherwise return error
1600  */
1601 static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
1602 {
1603         struct i40iw_device *iwdev;
1604         struct i40iw_sc_dev *dev;
1605         enum i40iw_status_code status;
1606         struct i40iw_handler *hdl;
1607
1608         hdl = i40iw_find_netdev(ldev->netdev);
1609         if (hdl)
1610                 return 0;
1611
1612         hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
1613         if (!hdl)
1614                 return -ENOMEM;
1615         iwdev = &hdl->device;
1616         iwdev->hdl = hdl;
1617         dev = &iwdev->sc_dev;
1618         i40iw_setup_cm_core(iwdev);
1619
1620         dev->back_dev = (void *)iwdev;
1621         iwdev->ldev = &hdl->ldev;
1622         iwdev->client = client;
1623         mutex_init(&iwdev->pbl_mutex);
1624         i40iw_add_handler(hdl);
1625
1626         do {
1627                 status = i40iw_setup_init_state(hdl, ldev, client);
1628                 if (status)
1629                         break;
1630                 iwdev->init_state = INITIAL_STATE;
1631                 if (dev->is_pf)
1632                         i40iw_wait_pe_ready(dev->hw);
1633                 status = i40iw_create_cqp(iwdev);
1634                 if (status)
1635                         break;
1636                 iwdev->init_state = CQP_CREATED;
1637                 status = i40iw_hmc_setup(iwdev);
1638                 if (status)
1639                         break;
1640                 status = i40iw_create_ccq(iwdev);
1641                 if (status)
1642                         break;
1643                 iwdev->init_state = CCQ_CREATED;
1644                 status = i40iw_initialize_ilq(iwdev);
1645                 if (status)
1646                         break;
1647                 iwdev->init_state = ILQ_CREATED;
1648                 status = i40iw_initialize_ieq(iwdev);
1649                 if (status)
1650                         break;
1651                 iwdev->init_state = IEQ_CREATED;
1652                 status = i40iw_setup_aeq(iwdev);
1653                 if (status)
1654                         break;
1655                 iwdev->init_state = AEQ_CREATED;
1656                 status = i40iw_setup_ceqs(iwdev, ldev);
1657                 if (status)
1658                         break;
1659                 iwdev->init_state = CEQ_CREATED;
1660                 status = i40iw_initialize_hw_resources(iwdev);
1661                 if (status)
1662                         break;
1663                 i40iw_get_used_rsrc(iwdev);
1664                 dev->ccq_ops->ccq_arm(dev->ccq);
1665                 status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc);
1666                 if (status)
1667                         break;
1668                 iwdev->init_state = PBLE_CHUNK_MEM;
1669                 iwdev->virtchnl_wq = alloc_ordered_workqueue("iwvch", WQ_MEM_RECLAIM);
1670                 status = i40iw_add_mac_ip(iwdev);
1671                 if (status)
1672                         break;
1673                 iwdev->init_state = IP_ADDR_REGISTERED;
1674                 if (i40iw_register_rdma_device(iwdev)) {
1675                         i40iw_pr_err("register rdma device fail\n");
1676                         break;
1677                 };
1678
1679                 iwdev->init_state = RDMA_DEV_REGISTERED;
1680                 iwdev->iw_status = 1;
1681                 i40iw_port_ibevent(iwdev);
1682                 iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);
1683                 if(iwdev->param_wq == NULL)
1684                         break;
1685                 i40iw_pr_info("i40iw_open completed\n");
1686                 return 0;
1687         } while (0);
1688
1689         i40iw_pr_err("status = %d last completion = %d\n", status, iwdev->init_state);
1690         i40iw_deinit_device(iwdev);
1691         return -ERESTART;
1692 }
1693
1694 /**
1695  * i40iw_l2params_worker - worker for l2 params change
1696  * @work: work pointer for l2 params
1697  */
1698 static void i40iw_l2params_worker(struct work_struct *work)
1699 {
1700         struct l2params_work *dwork =
1701             container_of(work, struct l2params_work, work);
1702         struct i40iw_device *iwdev = dwork->iwdev;
1703
1704         i40iw_change_l2params(&iwdev->vsi, &dwork->l2params);
1705         atomic_dec(&iwdev->params_busy);
1706         kfree(work);
1707 }
1708
1709 /**
1710  * i40iw_l2param_change - handle qs handles for qos and mss change
1711  * @ldev: lan device information
1712  * @client: client for paramater change
1713  * @params: new parameters from L2
1714  */
1715 static void i40iw_l2param_change(struct i40e_info *ldev, struct i40e_client *client,
1716                                  struct i40e_params *params)
1717 {
1718         struct i40iw_handler *hdl;
1719         struct i40iw_l2params *l2params;
1720         struct l2params_work *work;
1721         struct i40iw_device *iwdev;
1722         int i;
1723
1724         hdl = i40iw_find_i40e_handler(ldev);
1725         if (!hdl)
1726                 return;
1727
1728         iwdev = &hdl->device;
1729
1730         if (atomic_read(&iwdev->params_busy))
1731                 return;
1732
1733
1734         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1735         if (!work)
1736                 return;
1737
1738         atomic_inc(&iwdev->params_busy);
1739
1740         work->iwdev = iwdev;
1741         l2params = &work->l2params;
1742         for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++)
1743                 l2params->qs_handle_list[i] = params->qos.prio_qos[i].qs_handle;
1744
1745         l2params->mss = (params->mtu) ? params->mtu - I40IW_MTU_TO_MSS : iwdev->vsi.mss;
1746
1747         INIT_WORK(&work->work, i40iw_l2params_worker);
1748         queue_work(iwdev->param_wq, &work->work);
1749 }
1750
1751 /**
1752  * i40iw_close - client interface operation close for iwarp/uda device
1753  * @ldev: lan device information
1754  * @client: client to close
1755  *
1756  * Called by the lan driver during the processing of client unregister
1757  * Destroy and clean up the driver resources
1758  */
1759 static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool reset)
1760 {
1761         struct i40iw_device *iwdev;
1762         struct i40iw_handler *hdl;
1763
1764         hdl = i40iw_find_i40e_handler(ldev);
1765         if (!hdl)
1766                 return;
1767
1768         iwdev = &hdl->device;
1769         iwdev->closing = true;
1770
1771         if (reset)
1772                 iwdev->reset = true;
1773
1774         i40iw_cm_disconnect_all(iwdev);
1775         destroy_workqueue(iwdev->virtchnl_wq);
1776         i40iw_deinit_device(iwdev);
1777 }
1778
1779 /**
1780  * i40iw_vf_reset - process VF reset
1781  * @ldev: lan device information
1782  * @client: client interface instance
1783  * @vf_id: virtual function id
1784  *
1785  * Called when a VF is reset by the PF
1786  * Destroy and clean up the VF resources
1787  */
1788 static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u32 vf_id)
1789 {
1790         struct i40iw_handler *hdl;
1791         struct i40iw_sc_dev *dev;
1792         struct i40iw_hmc_fcn_info hmc_fcn_info;
1793         struct i40iw_virt_mem vf_dev_mem;
1794         struct i40iw_vfdev *tmp_vfdev;
1795         unsigned int i;
1796         unsigned long flags;
1797         struct i40iw_device *iwdev;
1798
1799         hdl = i40iw_find_i40e_handler(ldev);
1800         if (!hdl)
1801                 return;
1802
1803         dev = &hdl->device.sc_dev;
1804         iwdev = (struct i40iw_device *)dev->back_dev;
1805
1806         for (i = 0; i < I40IW_MAX_PE_ENABLED_VF_COUNT; i++) {
1807                 if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id != vf_id))
1808                         continue;
1809                 /* free all resources allocated on behalf of vf */
1810                 tmp_vfdev = dev->vf_dev[i];
1811                 spin_lock_irqsave(&iwdev->vsi.pestat->lock, flags);
1812                 dev->vf_dev[i] = NULL;
1813                 spin_unlock_irqrestore(&iwdev->vsi.pestat->lock, flags);
1814                 i40iw_del_hmc_objects(dev, &tmp_vfdev->hmc_info, false, false);
1815                 /* remove vf hmc function */
1816                 memset(&hmc_fcn_info, 0, sizeof(hmc_fcn_info));
1817                 hmc_fcn_info.vf_id = vf_id;
1818                 hmc_fcn_info.iw_vf_idx = tmp_vfdev->iw_vf_idx;
1819                 hmc_fcn_info.free_fcn = true;
1820                 i40iw_cqp_manage_hmc_fcn_cmd(dev, &hmc_fcn_info);
1821                 /* free vf_dev */
1822                 vf_dev_mem.va = tmp_vfdev;
1823                 vf_dev_mem.size = sizeof(struct i40iw_vfdev) +
1824                                         sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX;
1825                 i40iw_free_virt_mem(dev->hw, &vf_dev_mem);
1826                 break;
1827         }
1828 }
1829
1830 /**
1831  * i40iw_vf_enable - enable a number of VFs
1832  * @ldev: lan device information
1833  * @client: client interface instance
1834  * @num_vfs: number of VFs for the PF
1835  *
1836  * Called when the number of VFs changes
1837  */
1838 static void i40iw_vf_enable(struct i40e_info *ldev,
1839                             struct i40e_client *client,
1840                             u32 num_vfs)
1841 {
1842         struct i40iw_handler *hdl;
1843
1844         hdl = i40iw_find_i40e_handler(ldev);
1845         if (!hdl)
1846                 return;
1847
1848         if (num_vfs > I40IW_MAX_PE_ENABLED_VF_COUNT)
1849                 hdl->device.max_enabled_vfs = I40IW_MAX_PE_ENABLED_VF_COUNT;
1850         else
1851                 hdl->device.max_enabled_vfs = num_vfs;
1852 }
1853
1854 /**
1855  * i40iw_vf_capable - check if VF capable
1856  * @ldev: lan device information
1857  * @client: client interface instance
1858  * @vf_id: virtual function id
1859  *
1860  * Return 1 if a VF slot is available or if VF is already RDMA enabled
1861  * Return 0 otherwise
1862  */
1863 static int i40iw_vf_capable(struct i40e_info *ldev,
1864                             struct i40e_client *client,
1865                             u32 vf_id)
1866 {
1867         struct i40iw_handler *hdl;
1868         struct i40iw_sc_dev *dev;
1869         unsigned int i;
1870
1871         hdl = i40iw_find_i40e_handler(ldev);
1872         if (!hdl)
1873                 return 0;
1874
1875         dev = &hdl->device.sc_dev;
1876
1877         for (i = 0; i < hdl->device.max_enabled_vfs; i++) {
1878                 if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id == vf_id))
1879                         return 1;
1880         }
1881
1882         return 0;
1883 }
1884
1885 /**
1886  * i40iw_virtchnl_receive - receive a message through the virtual channel
1887  * @ldev: lan device information
1888  * @client: client interface instance
1889  * @vf_id: virtual function id associated with the message
1890  * @msg: message buffer pointer
1891  * @len: length of the message
1892  *
1893  * Invoke virtual channel receive operation for the given msg
1894  * Return 0 if successful, otherwise return error
1895  */
1896 static int i40iw_virtchnl_receive(struct i40e_info *ldev,
1897                                   struct i40e_client *client,
1898                                   u32 vf_id,
1899                                   u8 *msg,
1900                                   u16 len)
1901 {
1902         struct i40iw_handler *hdl;
1903         struct i40iw_sc_dev *dev;
1904         struct i40iw_device *iwdev;
1905         int ret_code = I40IW_NOT_SUPPORTED;
1906
1907         if (!len || !msg)
1908                 return I40IW_ERR_PARAM;
1909
1910         hdl = i40iw_find_i40e_handler(ldev);
1911         if (!hdl)
1912                 return I40IW_ERR_PARAM;
1913
1914         dev = &hdl->device.sc_dev;
1915         iwdev = dev->back_dev;
1916
1917         if (dev->vchnl_if.vchnl_recv) {
1918                 ret_code = dev->vchnl_if.vchnl_recv(dev, vf_id, msg, len);
1919                 if (!dev->is_pf) {
1920                         atomic_dec(&iwdev->vchnl_msgs);
1921                         wake_up(&iwdev->vchnl_waitq);
1922                 }
1923         }
1924         return ret_code;
1925 }
1926
1927 /**
1928  * i40iw_vf_clear_to_send - wait to send virtual channel message
1929  * @dev: iwarp device *
1930  * Wait for until virtual channel is clear
1931  * before sending the next message
1932  *
1933  * Returns false if error
1934  * Returns true if clear to send
1935  */
1936 bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev)
1937 {
1938         struct i40iw_device *iwdev;
1939         wait_queue_entry_t wait;
1940
1941         iwdev = dev->back_dev;
1942
1943         if (!wq_has_sleeper(&dev->vf_reqs) &&
1944             (atomic_read(&iwdev->vchnl_msgs) == 0))
1945                 return true; /* virtual channel is clear */
1946
1947         init_wait(&wait);
1948         add_wait_queue_exclusive(&dev->vf_reqs, &wait);
1949
1950         if (!wait_event_timeout(dev->vf_reqs,
1951                                 (atomic_read(&iwdev->vchnl_msgs) == 0),
1952                                 I40IW_VCHNL_EVENT_TIMEOUT))
1953                 dev->vchnl_up = false;
1954
1955         remove_wait_queue(&dev->vf_reqs, &wait);
1956
1957         return dev->vchnl_up;
1958 }
1959
1960 /**
1961  * i40iw_virtchnl_send - send a message through the virtual channel
1962  * @dev: iwarp device
1963  * @vf_id: virtual function id associated with the message
1964  * @msg: virtual channel message buffer pointer
1965  * @len: length of the message
1966  *
1967  * Invoke virtual channel send operation for the given msg
1968  * Return 0 if successful, otherwise return error
1969  */
1970 static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
1971                                                   u32 vf_id,
1972                                                   u8 *msg,
1973                                                   u16 len)
1974 {
1975         struct i40iw_device *iwdev;
1976         struct i40e_info *ldev;
1977
1978         if (!dev || !dev->back_dev)
1979                 return I40IW_ERR_BAD_PTR;
1980
1981         iwdev = dev->back_dev;
1982         ldev = iwdev->ldev;
1983
1984         if (ldev && ldev->ops && ldev->ops->virtchnl_send)
1985                 return ldev->ops->virtchnl_send(ldev, &i40iw_client, vf_id, msg, len);
1986         return I40IW_ERR_BAD_PTR;
1987 }
1988
1989 /* client interface functions */
1990 static const struct i40e_client_ops i40e_ops = {
1991         .open = i40iw_open,
1992         .close = i40iw_close,
1993         .l2_param_change = i40iw_l2param_change,
1994         .virtchnl_receive = i40iw_virtchnl_receive,
1995         .vf_reset = i40iw_vf_reset,
1996         .vf_enable = i40iw_vf_enable,
1997         .vf_capable = i40iw_vf_capable
1998 };
1999
2000 /**
2001  * i40iw_init_module - driver initialization function
2002  *
2003  * First function to call when the driver is loaded
2004  * Register the driver as i40e client and port mapper client
2005  */
2006 static int __init i40iw_init_module(void)
2007 {
2008         int ret;
2009
2010         memset(&i40iw_client, 0, sizeof(i40iw_client));
2011         i40iw_client.version.major = CLIENT_IW_INTERFACE_VERSION_MAJOR;
2012         i40iw_client.version.minor = CLIENT_IW_INTERFACE_VERSION_MINOR;
2013         i40iw_client.version.build = CLIENT_IW_INTERFACE_VERSION_BUILD;
2014         i40iw_client.ops = &i40e_ops;
2015         memcpy(i40iw_client.name, i40iw_client_name, I40E_CLIENT_STR_LENGTH);
2016         i40iw_client.type = I40E_CLIENT_IWARP;
2017         spin_lock_init(&i40iw_handler_lock);
2018         ret = i40e_register_client(&i40iw_client);
2019         i40iw_register_notifiers();
2020
2021         return ret;
2022 }
2023
2024 /**
2025  * i40iw_exit_module - driver exit clean up function
2026  *
2027  * The function is called just before the driver is unloaded
2028  * Unregister the driver as i40e client and port mapper client
2029  */
2030 static void __exit i40iw_exit_module(void)
2031 {
2032         i40iw_unregister_notifiers();
2033         i40e_unregister_client(&i40iw_client);
2034 }
2035
2036 module_init(i40iw_init_module);
2037 module_exit(i40iw_exit_module);