drivers/net/ethernet/neterion/vxge/vxge-traffic.c

   1 /******************************************************************************
   2  * This software may be used and distributed according to the terms of
   3  * the GNU General Public License (GPL), incorporated herein by reference.
   4  * Drivers based on or derived from this code fall under the GPL and must
   5  * retain the authorship, copyright and license notice.  This file is not
   6  * a complete program and may only be used when the entire operating
   7  * system is licensed under the GPL.
   8  * See the file COPYING in this distribution for more information.
   9  *
  10  * vxge-traffic.c: Driver for Exar Corp's X3100 Series 10GbE PCIe I/O
  11  *                 Virtualized Server Adapter.
  12  * Copyright(c) 2002-2010 Exar Corp.
  13  ******************************************************************************/
  14 #include <linux/etherdevice.h>
  15 #include <linux/prefetch.h>
  16
  17 #include "vxge-traffic.h"
  18 #include "vxge-config.h"
  19 #include "vxge-main.h"
  20
  21 /*
  22  * vxge_hw_vpath_intr_enable - Enable vpath interrupts.
  23  * @vp: Virtual Path handle.
  24  *
  25  * Enable vpath interrupts. The function is to be executed the last in
  26  * vpath initialization sequence.
  27  *
  28  * See also: vxge_hw_vpath_intr_disable()
  29  */
  30 enum vxge_hw_status vxge_hw_vpath_intr_enable(struct __vxge_hw_vpath_handle *vp)
  31 {
  32         struct __vxge_hw_virtualpath *vpath;
  33         struct vxge_hw_vpath_reg __iomem *vp_reg;
  34         enum vxge_hw_status status = VXGE_HW_OK;
  35         if (vp == NULL) {
  36                 status = VXGE_HW_ERR_INVALID_HANDLE;
  37                 goto exit;
  38         }
  39
  40         vpath = vp->vpath;
  41
  42         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
  43                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
  44                 goto exit;
  45         }
  46
  47         vp_reg = vpath->vp_reg;
  48
  49         writeq(VXGE_HW_INTR_MASK_ALL, &vp_reg->kdfcctl_errors_reg);
  50
  51         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  52                         &vp_reg->general_errors_reg);
  53
  54         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  55                         &vp_reg->pci_config_errors_reg);
  56
  57         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  58                         &vp_reg->mrpcim_to_vpath_alarm_reg);
  59
  60         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  61                         &vp_reg->srpcim_to_vpath_alarm_reg);
  62
  63         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  64                         &vp_reg->vpath_ppif_int_status);
  65
  66         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  67                         &vp_reg->srpcim_msg_to_vpath_reg);
  68
  69         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  70                         &vp_reg->vpath_pcipif_int_status);
  71
  72         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  73                         &vp_reg->prc_alarm_reg);
  74
  75         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  76                         &vp_reg->wrdma_alarm_status);
  77
  78         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  79                         &vp_reg->asic_ntwk_vp_err_reg);
  80
  81         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  82                         &vp_reg->xgmac_vp_int_status);
  83
  84         readq(&vp_reg->vpath_general_int_status);
  85
  86         /* Mask unwanted interrupts */
  87
  88         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  89                         &vp_reg->vpath_pcipif_int_mask);
  90
  91         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  92                         &vp_reg->srpcim_msg_to_vpath_mask);
  93
  94         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  95                         &vp_reg->srpcim_to_vpath_alarm_mask);
  96
  97         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
  98                         &vp_reg->mrpcim_to_vpath_alarm_mask);
  99
 100         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 101                         &vp_reg->pci_config_errors_mask);
 102
 103         /* Unmask the individual interrupts */
 104
 105         writeq((u32)vxge_bVALn((VXGE_HW_GENERAL_ERRORS_REG_DBLGEN_FIFO1_OVRFLOW|
 106                 VXGE_HW_GENERAL_ERRORS_REG_DBLGEN_FIFO2_OVRFLOW|
 107                 VXGE_HW_GENERAL_ERRORS_REG_STATSB_DROP_TIMEOUT_REQ|
 108                 VXGE_HW_GENERAL_ERRORS_REG_STATSB_PIF_CHAIN_ERR), 0, 32),
 109                 &vp_reg->general_errors_mask);
 110
 111         __vxge_hw_pio_mem_write32_upper(
 112                 (u32)vxge_bVALn((VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO1_OVRWR|
 113                 VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO2_OVRWR|
 114                 VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO1_POISON|
 115                 VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO2_POISON|
 116                 VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO1_DMA_ERR|
 117                 VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO2_DMA_ERR), 0, 32),
 118                 &vp_reg->kdfcctl_errors_mask);
 119
 120         __vxge_hw_pio_mem_write32_upper(0, &vp_reg->vpath_ppif_int_mask);
 121
 122         __vxge_hw_pio_mem_write32_upper(
 123                 (u32)vxge_bVALn(VXGE_HW_PRC_ALARM_REG_PRC_RING_BUMP, 0, 32),
 124                 &vp_reg->prc_alarm_mask);
 125
 126         __vxge_hw_pio_mem_write32_upper(0, &vp_reg->wrdma_alarm_mask);
 127         __vxge_hw_pio_mem_write32_upper(0, &vp_reg->xgmac_vp_int_mask);
 128
 129         if (vpath->hldev->first_vp_id != vpath->vp_id)
 130                 __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 131                         &vp_reg->asic_ntwk_vp_err_mask);
 132         else
 133                 __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn((
 134                 VXGE_HW_ASIC_NTWK_VP_ERR_REG_XMACJ_NTWK_REAFFIRMED_FAULT |
 135                 VXGE_HW_ASIC_NTWK_VP_ERR_REG_XMACJ_NTWK_REAFFIRMED_OK), 0, 32),
 136                 &vp_reg->asic_ntwk_vp_err_mask);
 137
 138         __vxge_hw_pio_mem_write32_upper(0,
 139                 &vp_reg->vpath_general_int_mask);
 140 exit:
 141         return status;
 142
 143 }
 144
 145 /*
 146  * vxge_hw_vpath_intr_disable - Disable vpath interrupts.
 147  * @vp: Virtual Path handle.
 148  *
 149  * Disable vpath interrupts. The function is to be executed the last in
 150  * vpath initialization sequence.
 151  *
 152  * See also: vxge_hw_vpath_intr_enable()
 153  */
 154 enum vxge_hw_status vxge_hw_vpath_intr_disable(
 155                         struct __vxge_hw_vpath_handle *vp)
 156 {
 157         struct __vxge_hw_virtualpath *vpath;
 158         enum vxge_hw_status status = VXGE_HW_OK;
 159         struct vxge_hw_vpath_reg __iomem *vp_reg;
 160         if (vp == NULL) {
 161                 status = VXGE_HW_ERR_INVALID_HANDLE;
 162                 goto exit;
 163         }
 164
 165         vpath = vp->vpath;
 166
 167         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
 168                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
 169                 goto exit;
 170         }
 171         vp_reg = vpath->vp_reg;
 172
 173         __vxge_hw_pio_mem_write32_upper(
 174                 (u32)VXGE_HW_INTR_MASK_ALL,
 175                 &vp_reg->vpath_general_int_mask);
 176
 177         writeq(VXGE_HW_INTR_MASK_ALL, &vp_reg->kdfcctl_errors_mask);
 178
 179         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 180                         &vp_reg->general_errors_mask);
 181
 182         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 183                         &vp_reg->pci_config_errors_mask);
 184
 185         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 186                         &vp_reg->mrpcim_to_vpath_alarm_mask);
 187
 188         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 189                         &vp_reg->srpcim_to_vpath_alarm_mask);
 190
 191         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 192                         &vp_reg->vpath_ppif_int_mask);
 193
 194         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 195                         &vp_reg->srpcim_msg_to_vpath_mask);
 196
 197         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 198                         &vp_reg->vpath_pcipif_int_mask);
 199
 200         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 201                         &vp_reg->wrdma_alarm_mask);
 202
 203         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 204                         &vp_reg->prc_alarm_mask);
 205
 206         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 207                         &vp_reg->xgmac_vp_int_mask);
 208
 209         __vxge_hw_pio_mem_write32_upper((u32)VXGE_HW_INTR_MASK_ALL,
 210                         &vp_reg->asic_ntwk_vp_err_mask);
 211
 212 exit:
 213         return status;
 214 }
 215
 216 void vxge_hw_vpath_tti_ci_set(struct __vxge_hw_fifo *fifo)
 217 {
 218         struct vxge_hw_vpath_reg __iomem *vp_reg;
 219         struct vxge_hw_vp_config *config;
 220         u64 val64;
 221
 222         if (fifo->config->enable != VXGE_HW_FIFO_ENABLE)
 223                 return;
 224
 225         vp_reg = fifo->vp_reg;
 226         config = container_of(fifo->config, struct vxge_hw_vp_config, fifo);
 227
 228         if (config->tti.timer_ci_en != VXGE_HW_TIM_TIMER_CI_ENABLE) {
 229                 config->tti.timer_ci_en = VXGE_HW_TIM_TIMER_CI_ENABLE;
 230                 val64 = readq(&vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_TX]);
 231                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
 232                 fifo->tim_tti_cfg1_saved = val64;
 233                 writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_TX]);
 234         }
 235 }
 236
 237 void vxge_hw_vpath_dynamic_rti_ci_set(struct __vxge_hw_ring *ring)
 238 {
 239         u64 val64 = ring->tim_rti_cfg1_saved;
 240
 241         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
 242         ring->tim_rti_cfg1_saved = val64;
 243         writeq(val64, &ring->vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_RX]);
 244 }
 245
 246 void vxge_hw_vpath_dynamic_tti_rtimer_set(struct __vxge_hw_fifo *fifo)
 247 {
 248         u64 val64 = fifo->tim_tti_cfg3_saved;
 249         u64 timer = (fifo->rtimer * 1000) / 272;
 250
 251         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(0x3ffffff);
 252         if (timer)
 253                 val64 |= VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(timer) |
 254                         VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_EVENT_SF(5);
 255
 256         writeq(val64, &fifo->vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_TX]);
 257         /* tti_cfg3_saved is not updated again because it is
 258          * initialized at one place only - init time.
 259          */
 260 }
 261
 262 void vxge_hw_vpath_dynamic_rti_rtimer_set(struct __vxge_hw_ring *ring)
 263 {
 264         u64 val64 = ring->tim_rti_cfg3_saved;
 265         u64 timer = (ring->rtimer * 1000) / 272;
 266
 267         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(0x3ffffff);
 268         if (timer)
 269                 val64 |= VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(timer) |
 270                         VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_EVENT_SF(4);
 271
 272         writeq(val64, &ring->vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_RX]);
 273         /* rti_cfg3_saved is not updated again because it is
 274          * initialized at one place only - init time.
 275          */
 276 }
 277
 278 /**
 279  * vxge_hw_channel_msix_mask - Mask MSIX Vector.
 280  * @channeh: Channel for rx or tx handle
 281  * @msix_id:  MSIX ID
 282  *
 283  * The function masks the msix interrupt for the given msix_id
 284  *
 285  * Returns: 0
 286  */
 287 void vxge_hw_channel_msix_mask(struct __vxge_hw_channel *channel, int msix_id)
 288 {
 289
 290         __vxge_hw_pio_mem_write32_upper(
 291                 (u32)vxge_bVALn(vxge_mBIT(msix_id >> 2), 0, 32),
 292                 &channel->common_reg->set_msix_mask_vect[msix_id%4]);
 293 }
 294
 295 /**
 296  * vxge_hw_channel_msix_unmask - Unmask the MSIX Vector.
 297  * @channeh: Channel for rx or tx handle
 298  * @msix_id:  MSI ID
 299  *
 300  * The function unmasks the msix interrupt for the given msix_id
 301  *
 302  * Returns: 0
 303  */
 304 void
 305 vxge_hw_channel_msix_unmask(struct __vxge_hw_channel *channel, int msix_id)
 306 {
 307
 308         __vxge_hw_pio_mem_write32_upper(
 309                 (u32)vxge_bVALn(vxge_mBIT(msix_id >> 2), 0, 32),
 310                 &channel->common_reg->clear_msix_mask_vect[msix_id%4]);
 311 }
 312
 313 /**
 314  * vxge_hw_channel_msix_clear - Unmask the MSIX Vector.
 315  * @channel: Channel for rx or tx handle
 316  * @msix_id:  MSI ID
 317  *
 318  * The function unmasks the msix interrupt for the given msix_id
 319  * if configured in MSIX oneshot mode
 320  *
 321  * Returns: 0
 322  */
 323 void vxge_hw_channel_msix_clear(struct __vxge_hw_channel *channel, int msix_id)
 324 {
 325         __vxge_hw_pio_mem_write32_upper(
 326                 (u32) vxge_bVALn(vxge_mBIT(msix_id >> 2), 0, 32),
 327                 &channel->common_reg->clr_msix_one_shot_vec[msix_id % 4]);
 328 }
 329
 330 /**
 331  * vxge_hw_device_set_intr_type - Updates the configuration
 332  *              with new interrupt type.
 333  * @hldev: HW device handle.
 334  * @intr_mode: New interrupt type
 335  */
 336 u32 vxge_hw_device_set_intr_type(struct __vxge_hw_device *hldev, u32 intr_mode)
 337 {
 338
 339         if ((intr_mode != VXGE_HW_INTR_MODE_IRQLINE) &&
 340            (intr_mode != VXGE_HW_INTR_MODE_MSIX) &&
 341            (intr_mode != VXGE_HW_INTR_MODE_MSIX_ONE_SHOT) &&
 342            (intr_mode != VXGE_HW_INTR_MODE_DEF))
 343                 intr_mode = VXGE_HW_INTR_MODE_IRQLINE;
 344
 345         hldev->config.intr_mode = intr_mode;
 346         return intr_mode;
 347 }
 348
 349 /**
 350  * vxge_hw_device_intr_enable - Enable interrupts.
 351  * @hldev: HW device handle.
 352  * @op: One of the enum vxge_hw_device_intr enumerated values specifying
 353  *      the type(s) of interrupts to enable.
 354  *
 355  * Enable Titan interrupts. The function is to be executed the last in
 356  * Titan initialization sequence.
 357  *
 358  * See also: vxge_hw_device_intr_disable()
 359  */
 360 void vxge_hw_device_intr_enable(struct __vxge_hw_device *hldev)
 361 {
 362         u32 i;
 363         u64 val64;
 364         u32 val32;
 365
 366         vxge_hw_device_mask_all(hldev);
 367
 368         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 369
 370                 if (!(hldev->vpaths_deployed & vxge_mBIT(i)))
 371                         continue;
 372
 373                 vxge_hw_vpath_intr_enable(
 374                         VXGE_HW_VIRTUAL_PATH_HANDLE(&hldev->virtual_paths[i]));
 375         }
 376
 377         if (hldev->config.intr_mode == VXGE_HW_INTR_MODE_IRQLINE) {
 378                 val64 = hldev->tim_int_mask0[VXGE_HW_VPATH_INTR_TX] |
 379                         hldev->tim_int_mask0[VXGE_HW_VPATH_INTR_RX];
 380
 381                 if (val64 != 0) {
 382                         writeq(val64, &hldev->common_reg->tim_int_status0);
 383
 384                         writeq(~val64, &hldev->common_reg->tim_int_mask0);
 385                 }
 386
 387                 val32 = hldev->tim_int_mask1[VXGE_HW_VPATH_INTR_TX] |
 388                         hldev->tim_int_mask1[VXGE_HW_VPATH_INTR_RX];
 389
 390                 if (val32 != 0) {
 391                         __vxge_hw_pio_mem_write32_upper(val32,
 392                                         &hldev->common_reg->tim_int_status1);
 393
 394                         __vxge_hw_pio_mem_write32_upper(~val32,
 395                                         &hldev->common_reg->tim_int_mask1);
 396                 }
 397         }
 398
 399         val64 = readq(&hldev->common_reg->titan_general_int_status);
 400
 401         vxge_hw_device_unmask_all(hldev);
 402 }
 403
 404 /**
 405  * vxge_hw_device_intr_disable - Disable Titan interrupts.
 406  * @hldev: HW device handle.
 407  * @op: One of the enum vxge_hw_device_intr enumerated values specifying
 408  *      the type(s) of interrupts to disable.
 409  *
 410  * Disable Titan interrupts.
 411  *
 412  * See also: vxge_hw_device_intr_enable()
 413  */
 414 void vxge_hw_device_intr_disable(struct __vxge_hw_device *hldev)
 415 {
 416         u32 i;
 417
 418         vxge_hw_device_mask_all(hldev);
 419
 420         /* mask all the tim interrupts */
 421         writeq(VXGE_HW_INTR_MASK_ALL, &hldev->common_reg->tim_int_mask0);
 422         __vxge_hw_pio_mem_write32_upper(VXGE_HW_DEFAULT_32,
 423                 &hldev->common_reg->tim_int_mask1);
 424
 425         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 426
 427                 if (!(hldev->vpaths_deployed & vxge_mBIT(i)))
 428                         continue;
 429
 430                 vxge_hw_vpath_intr_disable(
 431                         VXGE_HW_VIRTUAL_PATH_HANDLE(&hldev->virtual_paths[i]));
 432         }
 433 }
 434
 435 /**
 436  * vxge_hw_device_mask_all - Mask all device interrupts.
 437  * @hldev: HW device handle.
 438  *
 439  * Mask all device interrupts.
 440  *
 441  * See also: vxge_hw_device_unmask_all()
 442  */
 443 void vxge_hw_device_mask_all(struct __vxge_hw_device *hldev)
 444 {
 445         u64 val64;
 446
 447         val64 = VXGE_HW_TITAN_MASK_ALL_INT_ALARM |
 448                 VXGE_HW_TITAN_MASK_ALL_INT_TRAFFIC;
 449
 450         __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(val64, 0, 32),
 451                                 &hldev->common_reg->titan_mask_all_int);
 452 }
 453
 454 /**
 455  * vxge_hw_device_unmask_all - Unmask all device interrupts.
 456  * @hldev: HW device handle.
 457  *
 458  * Unmask all device interrupts.
 459  *
 460  * See also: vxge_hw_device_mask_all()
 461  */
 462 void vxge_hw_device_unmask_all(struct __vxge_hw_device *hldev)
 463 {
 464         u64 val64 = 0;
 465
 466         if (hldev->config.intr_mode == VXGE_HW_INTR_MODE_IRQLINE)
 467                 val64 =  VXGE_HW_TITAN_MASK_ALL_INT_TRAFFIC;
 468
 469         __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(val64, 0, 32),
 470                         &hldev->common_reg->titan_mask_all_int);
 471 }
 472
 473 /**
 474  * vxge_hw_device_flush_io - Flush io writes.
 475  * @hldev: HW device handle.
 476  *
 477  * The function performs a read operation to flush io writes.
 478  *
 479  * Returns: void
 480  */
 481 void vxge_hw_device_flush_io(struct __vxge_hw_device *hldev)
 482 {
 483         readl(&hldev->common_reg->titan_general_int_status);
 484 }
 485
 486 /**
 487  * __vxge_hw_device_handle_error - Handle error
 488  * @hldev: HW device
 489  * @vp_id: Vpath Id
 490  * @type: Error type. Please see enum vxge_hw_event{}
 491  *
 492  * Handle error.
 493  */
 494 static enum vxge_hw_status
 495 __vxge_hw_device_handle_error(struct __vxge_hw_device *hldev, u32 vp_id,
 496                               enum vxge_hw_event type)
 497 {
 498         switch (type) {
 499         case VXGE_HW_EVENT_UNKNOWN:
 500                 break;
 501         case VXGE_HW_EVENT_RESET_START:
 502         case VXGE_HW_EVENT_RESET_COMPLETE:
 503         case VXGE_HW_EVENT_LINK_DOWN:
 504         case VXGE_HW_EVENT_LINK_UP:
 505                 goto out;
 506         case VXGE_HW_EVENT_ALARM_CLEARED:
 507                 goto out;
 508         case VXGE_HW_EVENT_ECCERR:
 509         case VXGE_HW_EVENT_MRPCIM_ECCERR:
 510                 goto out;
 511         case VXGE_HW_EVENT_FIFO_ERR:
 512         case VXGE_HW_EVENT_VPATH_ERR:
 513         case VXGE_HW_EVENT_CRITICAL_ERR:
 514         case VXGE_HW_EVENT_SERR:
 515                 break;
 516         case VXGE_HW_EVENT_SRPCIM_SERR:
 517         case VXGE_HW_EVENT_MRPCIM_SERR:
 518                 goto out;
 519         case VXGE_HW_EVENT_SLOT_FREEZE:
 520                 break;
 521         default:
 522                 vxge_assert(0);
 523                 goto out;
 524         }
 525
 526         /* notify driver */
 527         if (hldev->uld_callbacks->crit_err)
 528                 hldev->uld_callbacks->crit_err(hldev,
 529                         type, vp_id);
 530 out:
 531
 532         return VXGE_HW_OK;
 533 }
 534
 535 /*
 536  * __vxge_hw_device_handle_link_down_ind
 537  * @hldev: HW device handle.
 538  *
 539  * Link down indication handler. The function is invoked by HW when
 540  * Titan indicates that the link is down.
 541  */
 542 static enum vxge_hw_status
 543 __vxge_hw_device_handle_link_down_ind(struct __vxge_hw_device *hldev)
 544 {
 545         /*
 546          * If the previous link state is not down, return.
 547          */
 548         if (hldev->link_state == VXGE_HW_LINK_DOWN)
 549                 goto exit;
 550
 551         hldev->link_state = VXGE_HW_LINK_DOWN;
 552
 553         /* notify driver */
 554         if (hldev->uld_callbacks->link_down)
 555                 hldev->uld_callbacks->link_down(hldev);
 556 exit:
 557         return VXGE_HW_OK;
 558 }
 559
 560 /*
 561  * __vxge_hw_device_handle_link_up_ind
 562  * @hldev: HW device handle.
 563  *
 564  * Link up indication handler. The function is invoked by HW when
 565  * Titan indicates that the link is up for programmable amount of time.
 566  */
 567 static enum vxge_hw_status
 568 __vxge_hw_device_handle_link_up_ind(struct __vxge_hw_device *hldev)
 569 {
 570         /*
 571          * If the previous link state is not down, return.
 572          */
 573         if (hldev->link_state == VXGE_HW_LINK_UP)
 574                 goto exit;
 575
 576         hldev->link_state = VXGE_HW_LINK_UP;
 577
 578         /* notify driver */
 579         if (hldev->uld_callbacks->link_up)
 580                 hldev->uld_callbacks->link_up(hldev);
 581 exit:
 582         return VXGE_HW_OK;
 583 }
 584
 585 /*
 586  * __vxge_hw_vpath_alarm_process - Process Alarms.
 587  * @vpath: Virtual Path.
 588  * @skip_alarms: Do not clear the alarms
 589  *
 590  * Process vpath alarms.
 591  *
 592  */
 593 static enum vxge_hw_status
 594 __vxge_hw_vpath_alarm_process(struct __vxge_hw_virtualpath *vpath,
 595                               u32 skip_alarms)
 596 {
 597         u64 val64;
 598         u64 alarm_status;
 599         u64 pic_status;
 600         struct __vxge_hw_device *hldev = NULL;
 601         enum vxge_hw_event alarm_event = VXGE_HW_EVENT_UNKNOWN;
 602         u64 mask64;
 603         struct vxge_hw_vpath_stats_sw_info *sw_stats;
 604         struct vxge_hw_vpath_reg __iomem *vp_reg;
 605
 606         if (vpath == NULL) {
 607                 alarm_event = VXGE_HW_SET_LEVEL(VXGE_HW_EVENT_UNKNOWN,
 608                         alarm_event);
 609                 goto out2;
 610         }
 611
 612         hldev = vpath->hldev;
 613         vp_reg = vpath->vp_reg;
 614         alarm_status = readq(&vp_reg->vpath_general_int_status);
 615
 616         if (alarm_status == VXGE_HW_ALL_FOXES) {
 617                 alarm_event = VXGE_HW_SET_LEVEL(VXGE_HW_EVENT_SLOT_FREEZE,
 618                         alarm_event);
 619                 goto out;
 620         }
 621
 622         sw_stats = vpath->sw_stats;
 623
 624         if (alarm_status & ~(
 625                 VXGE_HW_VPATH_GENERAL_INT_STATUS_PIC_INT |
 626                 VXGE_HW_VPATH_GENERAL_INT_STATUS_PCI_INT |
 627                 VXGE_HW_VPATH_GENERAL_INT_STATUS_WRDMA_INT |
 628                 VXGE_HW_VPATH_GENERAL_INT_STATUS_XMAC_INT)) {
 629                 sw_stats->error_stats.unknown_alarms++;
 630
 631                 alarm_event = VXGE_HW_SET_LEVEL(VXGE_HW_EVENT_UNKNOWN,
 632                         alarm_event);
 633                 goto out;
 634         }
 635
 636         if (alarm_status & VXGE_HW_VPATH_GENERAL_INT_STATUS_XMAC_INT) {
 637
 638                 val64 = readq(&vp_reg->xgmac_vp_int_status);
 639
 640                 if (val64 &
 641                 VXGE_HW_XGMAC_VP_INT_STATUS_ASIC_NTWK_VP_ERR_ASIC_NTWK_VP_INT) {
 642
 643                         val64 = readq(&vp_reg->asic_ntwk_vp_err_reg);
 644
 645                         if (((val64 &
 646                               VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_FLT) &&
 647                              (!(val64 &
 648                                 VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_OK))) ||
 649                             ((val64 &
 650                              VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_FLT_OCCURR) &&
 651                              (!(val64 &
 652                                 VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_OK_OCCURR)
 653                                      ))) {
 654                                 sw_stats->error_stats.network_sustained_fault++;
 655
 656                                 writeq(
 657                                 VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_FLT,
 658                                         &vp_reg->asic_ntwk_vp_err_mask);
 659
 660                                 __vxge_hw_device_handle_link_down_ind(hldev);
 661                                 alarm_event = VXGE_HW_SET_LEVEL(
 662                                         VXGE_HW_EVENT_LINK_DOWN, alarm_event);
 663                         }
 664
 665                         if (((val64 &
 666                               VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_OK) &&
 667                              (!(val64 &
 668                                 VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_FLT))) ||
 669                             ((val64 &
 670                               VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_OK_OCCURR) &&
 671                              (!(val64 &
 672                                 VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_FLT_OCCURR)
 673                                      ))) {
 674
 675                                 sw_stats->error_stats.network_sustained_ok++;
 676
 677                                 writeq(
 678                                 VXGE_HW_ASIC_NW_VP_ERR_REG_XMACJ_STN_OK,
 679                                         &vp_reg->asic_ntwk_vp_err_mask);
 680
 681                                 __vxge_hw_device_handle_link_up_ind(hldev);
 682                                 alarm_event = VXGE_HW_SET_LEVEL(
 683                                         VXGE_HW_EVENT_LINK_UP, alarm_event);
 684                         }
 685
 686                         writeq(VXGE_HW_INTR_MASK_ALL,
 687                                 &vp_reg->asic_ntwk_vp_err_reg);
 688
 689                         alarm_event = VXGE_HW_SET_LEVEL(
 690                                 VXGE_HW_EVENT_ALARM_CLEARED, alarm_event);
 691
 692                         if (skip_alarms)
 693                                 return VXGE_HW_OK;
 694                 }
 695         }
 696
 697         if (alarm_status & VXGE_HW_VPATH_GENERAL_INT_STATUS_PIC_INT) {
 698
 699                 pic_status = readq(&vp_reg->vpath_ppif_int_status);
 700
 701                 if (pic_status &
 702                     VXGE_HW_VPATH_PPIF_INT_STATUS_GENERAL_ERRORS_GENERAL_INT) {
 703
 704                         val64 = readq(&vp_reg->general_errors_reg);
 705                         mask64 = readq(&vp_reg->general_errors_mask);
 706
 707                         if ((val64 &
 708                                 VXGE_HW_GENERAL_ERRORS_REG_INI_SERR_DET) &
 709                                 ~mask64) {
 710                                 sw_stats->error_stats.ini_serr_det++;
 711
 712                                 alarm_event = VXGE_HW_SET_LEVEL(
 713                                         VXGE_HW_EVENT_SERR, alarm_event);
 714                         }
 715
 716                         if ((val64 &
 717                             VXGE_HW_GENERAL_ERRORS_REG_DBLGEN_FIFO0_OVRFLOW) &
 718                                 ~mask64) {
 719                                 sw_stats->error_stats.dblgen_fifo0_overflow++;
 720
 721                                 alarm_event = VXGE_HW_SET_LEVEL(
 722                                         VXGE_HW_EVENT_FIFO_ERR, alarm_event);
 723                         }
 724
 725                         if ((val64 &
 726                             VXGE_HW_GENERAL_ERRORS_REG_STATSB_PIF_CHAIN_ERR) &
 727                                 ~mask64)
 728                                 sw_stats->error_stats.statsb_pif_chain_error++;
 729
 730                         if ((val64 &
 731                            VXGE_HW_GENERAL_ERRORS_REG_STATSB_DROP_TIMEOUT_REQ) &
 732                                 ~mask64)
 733                                 sw_stats->error_stats.statsb_drop_timeout++;
 734
 735                         if ((val64 &
 736                                 VXGE_HW_GENERAL_ERRORS_REG_TGT_ILLEGAL_ACCESS) &
 737                                 ~mask64)
 738                                 sw_stats->error_stats.target_illegal_access++;
 739
 740                         if (!skip_alarms) {
 741                                 writeq(VXGE_HW_INTR_MASK_ALL,
 742                                         &vp_reg->general_errors_reg);
 743                                 alarm_event = VXGE_HW_SET_LEVEL(
 744                                         VXGE_HW_EVENT_ALARM_CLEARED,
 745                                         alarm_event);
 746                         }
 747                 }
 748
 749                 if (pic_status &
 750                     VXGE_HW_VPATH_PPIF_INT_STATUS_KDFCCTL_ERRORS_KDFCCTL_INT) {
 751
 752                         val64 = readq(&vp_reg->kdfcctl_errors_reg);
 753                         mask64 = readq(&vp_reg->kdfcctl_errors_mask);
 754
 755                         if ((val64 &
 756                             VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO0_OVRWR) &
 757                                 ~mask64) {
 758                                 sw_stats->error_stats.kdfcctl_fifo0_overwrite++;
 759
 760                                 alarm_event = VXGE_HW_SET_LEVEL(
 761                                         VXGE_HW_EVENT_FIFO_ERR,
 762                                         alarm_event);
 763                         }
 764
 765                         if ((val64 &
 766                             VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO0_POISON) &
 767                                 ~mask64) {
 768                                 sw_stats->error_stats.kdfcctl_fifo0_poison++;
 769
 770                                 alarm_event = VXGE_HW_SET_LEVEL(
 771                                         VXGE_HW_EVENT_FIFO_ERR,
 772                                         alarm_event);
 773                         }
 774
 775                         if ((val64 &
 776                             VXGE_HW_KDFCCTL_ERRORS_REG_KDFCCTL_FIFO0_DMA_ERR) &
 777                                 ~mask64) {
 778                                 sw_stats->error_stats.kdfcctl_fifo0_dma_error++;
 779
 780                                 alarm_event = VXGE_HW_SET_LEVEL(
 781                                         VXGE_HW_EVENT_FIFO_ERR,
 782                                         alarm_event);
 783                         }
 784
 785                         if (!skip_alarms) {
 786                                 writeq(VXGE_HW_INTR_MASK_ALL,
 787                                         &vp_reg->kdfcctl_errors_reg);
 788                                 alarm_event = VXGE_HW_SET_LEVEL(
 789                                         VXGE_HW_EVENT_ALARM_CLEARED,
 790                                         alarm_event);
 791                         }
 792                 }
 793
 794         }
 795
 796         if (alarm_status & VXGE_HW_VPATH_GENERAL_INT_STATUS_WRDMA_INT) {
 797
 798                 val64 = readq(&vp_reg->wrdma_alarm_status);
 799
 800                 if (val64 & VXGE_HW_WRDMA_ALARM_STATUS_PRC_ALARM_PRC_INT) {
 801
 802                         val64 = readq(&vp_reg->prc_alarm_reg);
 803                         mask64 = readq(&vp_reg->prc_alarm_mask);
 804
 805                         if ((val64 & VXGE_HW_PRC_ALARM_REG_PRC_RING_BUMP)&
 806                                 ~mask64)
 807                                 sw_stats->error_stats.prc_ring_bumps++;
 808
 809                         if ((val64 & VXGE_HW_PRC_ALARM_REG_PRC_RXDCM_SC_ERR) &
 810                                 ~mask64) {
 811                                 sw_stats->error_stats.prc_rxdcm_sc_err++;
 812
 813                                 alarm_event = VXGE_HW_SET_LEVEL(
 814                                         VXGE_HW_EVENT_VPATH_ERR,
 815                                         alarm_event);
 816                         }
 817
 818                         if ((val64 & VXGE_HW_PRC_ALARM_REG_PRC_RXDCM_SC_ABORT)
 819                                 & ~mask64) {
 820                                 sw_stats->error_stats.prc_rxdcm_sc_abort++;
 821
 822                                 alarm_event = VXGE_HW_SET_LEVEL(
 823                                                 VXGE_HW_EVENT_VPATH_ERR,
 824                                                 alarm_event);
 825                         }
 826
 827                         if ((val64 & VXGE_HW_PRC_ALARM_REG_PRC_QUANTA_SIZE_ERR)
 828                                  & ~mask64) {
 829                                 sw_stats->error_stats.prc_quanta_size_err++;
 830
 831                                 alarm_event = VXGE_HW_SET_LEVEL(
 832                                         VXGE_HW_EVENT_VPATH_ERR,
 833                                         alarm_event);
 834                         }
 835
 836                         if (!skip_alarms) {
 837                                 writeq(VXGE_HW_INTR_MASK_ALL,
 838                                         &vp_reg->prc_alarm_reg);
 839                                 alarm_event = VXGE_HW_SET_LEVEL(
 840                                                 VXGE_HW_EVENT_ALARM_CLEARED,
 841                                                 alarm_event);
 842                         }
 843                 }
 844         }
 845 out:
 846         hldev->stats.sw_dev_err_stats.vpath_alarms++;
 847 out2:
 848         if ((alarm_event == VXGE_HW_EVENT_ALARM_CLEARED) ||
 849                 (alarm_event == VXGE_HW_EVENT_UNKNOWN))
 850                 return VXGE_HW_OK;
 851
 852         __vxge_hw_device_handle_error(hldev, vpath->vp_id, alarm_event);
 853
 854         if (alarm_event == VXGE_HW_EVENT_SERR)
 855                 return VXGE_HW_ERR_CRITICAL;
 856
 857         return (alarm_event == VXGE_HW_EVENT_SLOT_FREEZE) ?
 858                 VXGE_HW_ERR_SLOT_FREEZE :
 859                 (alarm_event == VXGE_HW_EVENT_FIFO_ERR) ? VXGE_HW_ERR_FIFO :
 860                 VXGE_HW_ERR_VPATH;
 861 }
 862
 863 /**
 864  * vxge_hw_device_begin_irq - Begin IRQ processing.
 865  * @hldev: HW device handle.
 866  * @skip_alarms: Do not clear the alarms
 867  * @reason: "Reason" for the interrupt, the value of Titan's
 868  *      general_int_status register.
 869  *
 870  * The function performs two actions, It first checks whether (shared IRQ) the
 871  * interrupt was raised by the device. Next, it masks the device interrupts.
 872  *
 873  * Note:
 874  * vxge_hw_device_begin_irq() does not flush MMIO writes through the
 875  * bridge. Therefore, two back-to-back interrupts are potentially possible.
 876  *
 877  * Returns: 0, if the interrupt is not "ours" (note that in this case the
 878  * device remain enabled).
 879  * Otherwise, vxge_hw_device_begin_irq() returns 64bit general adapter
 880  * status.
 881  */
 882 enum vxge_hw_status vxge_hw_device_begin_irq(struct __vxge_hw_device *hldev,
 883                                              u32 skip_alarms, u64 *reason)
 884 {
 885         u32 i;
 886         u64 val64;
 887         u64 adapter_status;
 888         u64 vpath_mask;
 889         enum vxge_hw_status ret = VXGE_HW_OK;
 890
 891         val64 = readq(&hldev->common_reg->titan_general_int_status);
 892
 893         if (unlikely(!val64)) {
 894                 /* not Titan interrupt  */
 895                 *reason = 0;
 896                 ret = VXGE_HW_ERR_WRONG_IRQ;
 897                 goto exit;
 898         }
 899
 900         if (unlikely(val64 == VXGE_HW_ALL_FOXES)) {
 901
 902                 adapter_status = readq(&hldev->common_reg->adapter_status);
 903
 904                 if (adapter_status == VXGE_HW_ALL_FOXES) {
 905
 906                         __vxge_hw_device_handle_error(hldev,
 907                                 NULL_VPID, VXGE_HW_EVENT_SLOT_FREEZE);
 908                         *reason = 0;
 909                         ret = VXGE_HW_ERR_SLOT_FREEZE;
 910                         goto exit;
 911                 }
 912         }
 913
 914         hldev->stats.sw_dev_info_stats.total_intr_cnt++;
 915
 916         *reason = val64;
 917
 918         vpath_mask = hldev->vpaths_deployed >>
 919                                 (64 - VXGE_HW_MAX_VIRTUAL_PATHS);
 920
 921         if (val64 &
 922             VXGE_HW_TITAN_GENERAL_INT_STATUS_VPATH_TRAFFIC_INT(vpath_mask)) {
 923                 hldev->stats.sw_dev_info_stats.traffic_intr_cnt++;
 924
 925                 return VXGE_HW_OK;
 926         }
 927
 928         hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
 929
 930         if (unlikely(val64 &
 931                         VXGE_HW_TITAN_GENERAL_INT_STATUS_VPATH_ALARM_INT)) {
 932
 933                 enum vxge_hw_status error_level = VXGE_HW_OK;
 934
 935                 hldev->stats.sw_dev_err_stats.vpath_alarms++;
 936
 937                 for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 938
 939                         if (!(hldev->vpaths_deployed & vxge_mBIT(i)))
 940                                 continue;
 941
 942                         ret = __vxge_hw_vpath_alarm_process(
 943                                 &hldev->virtual_paths[i], skip_alarms);
 944
 945                         error_level = VXGE_HW_SET_LEVEL(ret, error_level);
 946
 947                         if (unlikely((ret == VXGE_HW_ERR_CRITICAL) ||
 948                                 (ret == VXGE_HW_ERR_SLOT_FREEZE)))
 949                                 break;
 950                 }
 951
 952                 ret = error_level;
 953         }
 954 exit:
 955         return ret;
 956 }
 957
 958 /**
 959  * vxge_hw_device_clear_tx_rx - Acknowledge (that is, clear) the
 960  * condition that has caused the Tx and RX interrupt.
 961  * @hldev: HW device.
 962  *
 963  * Acknowledge (that is, clear) the condition that has caused
 964  * the Tx and Rx interrupt.
 965  * See also: vxge_hw_device_begin_irq(),
 966  * vxge_hw_device_mask_tx_rx(), vxge_hw_device_unmask_tx_rx().
 967  */
 968 void vxge_hw_device_clear_tx_rx(struct __vxge_hw_device *hldev)
 969 {
 970
 971         if ((hldev->tim_int_mask0[VXGE_HW_VPATH_INTR_TX] != 0) ||
 972            (hldev->tim_int_mask0[VXGE_HW_VPATH_INTR_RX] != 0)) {
 973                 writeq((hldev->tim_int_mask0[VXGE_HW_VPATH_INTR_TX] |
 974                                  hldev->tim_int_mask0[VXGE_HW_VPATH_INTR_RX]),
 975                                 &hldev->common_reg->tim_int_status0);
 976         }
 977
 978         if ((hldev->tim_int_mask1[VXGE_HW_VPATH_INTR_TX] != 0) ||
 979            (hldev->tim_int_mask1[VXGE_HW_VPATH_INTR_RX] != 0)) {
 980                 __vxge_hw_pio_mem_write32_upper(
 981                                 (hldev->tim_int_mask1[VXGE_HW_VPATH_INTR_TX] |
 982                                  hldev->tim_int_mask1[VXGE_HW_VPATH_INTR_RX]),
 983                                 &hldev->common_reg->tim_int_status1);
 984         }
 985 }
 986
 987 /*
 988  * vxge_hw_channel_dtr_alloc - Allocate a dtr from the channel
 989  * @channel: Channel
 990  * @dtrh: Buffer to return the DTR pointer
 991  *
 992  * Allocates a dtr from the reserve array. If the reserve array is empty,
 993  * it swaps the reserve and free arrays.
 994  *
 995  */
 996 static enum vxge_hw_status
 997 vxge_hw_channel_dtr_alloc(struct __vxge_hw_channel *channel, void **dtrh)
 998 {
 999         if (channel->reserve_ptr - channel->reserve_top > 0) {
1000 _alloc_after_swap:
1001                 *dtrh = channel->reserve_arr[--channel->reserve_ptr];
1002
1003                 return VXGE_HW_OK;
1004         }
1005
1006         /* switch between empty and full arrays */
1007
1008         /* the idea behind such a design is that by having free and reserved
1009          * arrays separated we basically separated irq and non-irq parts.
1010          * i.e. no additional lock need to be done when we free a resource */
1011
1012         if (channel->length - channel->free_ptr > 0) {
1013                 swap(channel->reserve_arr, channel->free_arr);
1014                 channel->reserve_ptr = channel->length;
1015                 channel->reserve_top = channel->free_ptr;
1016                 channel->free_ptr = channel->length;
1017
1018                 channel->stats->reserve_free_swaps_cnt++;
1019
1020                 goto _alloc_after_swap;
1021         }
1022
1023         channel->stats->full_cnt++;
1024
1025         *dtrh = NULL;
1026         return VXGE_HW_INF_OUT_OF_DESCRIPTORS;
1027 }
1028
1029 /*
1030  * vxge_hw_channel_dtr_post - Post a dtr to the channel
1031  * @channelh: Channel
1032  * @dtrh: DTR pointer
1033  *
1034  * Posts a dtr to work array.
1035  *
1036  */
1037 static void
1038 vxge_hw_channel_dtr_post(struct __vxge_hw_channel *channel, void *dtrh)
1039 {
1040         vxge_assert(channel->work_arr[channel->post_index] == NULL);
1041
1042         channel->work_arr[channel->post_index++] = dtrh;
1043
1044         /* wrap-around */
1045         if (channel->post_index == channel->length)
1046                 channel->post_index = 0;
1047 }
1048
1049 /*
1050  * vxge_hw_channel_dtr_try_complete - Returns next completed dtr
1051  * @channel: Channel
1052  * @dtr: Buffer to return the next completed DTR pointer
1053  *
1054  * Returns the next completed dtr with out removing it from work array
1055  *
1056  */
1057 void
1058 vxge_hw_channel_dtr_try_complete(struct __vxge_hw_channel *channel, void **dtrh)
1059 {
1060         vxge_assert(channel->compl_index < channel->length);
1061
1062         *dtrh = channel->work_arr[channel->compl_index];
1063         prefetch(*dtrh);
1064 }
1065
1066 /*
1067  * vxge_hw_channel_dtr_complete - Removes next completed dtr from the work array
1068  * @channel: Channel handle
1069  *
1070  * Removes the next completed dtr from work array
1071  *
1072  */
1073 void vxge_hw_channel_dtr_complete(struct __vxge_hw_channel *channel)
1074 {
1075         channel->work_arr[channel->compl_index] = NULL;
1076
1077         /* wrap-around */
1078         if (++channel->compl_index == channel->length)
1079                 channel->compl_index = 0;
1080
1081         channel->stats->total_compl_cnt++;
1082 }
1083
1084 /*
1085  * vxge_hw_channel_dtr_free - Frees a dtr
1086  * @channel: Channel handle
1087  * @dtr:  DTR pointer
1088  *
1089  * Returns the dtr to free array
1090  *
1091  */
1092 void vxge_hw_channel_dtr_free(struct __vxge_hw_channel *channel, void *dtrh)
1093 {
1094         channel->free_arr[--channel->free_ptr] = dtrh;
1095 }
1096
1097 /*
1098  * vxge_hw_channel_dtr_count
1099  * @channel: Channel handle. Obtained via vxge_hw_channel_open().
1100  *
1101  * Retrieve number of DTRs available. This function can not be called
1102  * from data path. ring_initial_replenishi() is the only user.
1103  */
1104 int vxge_hw_channel_dtr_count(struct __vxge_hw_channel *channel)
1105 {
1106         return (channel->reserve_ptr - channel->reserve_top) +
1107                 (channel->length - channel->free_ptr);
1108 }
1109
1110 /**
1111  * vxge_hw_ring_rxd_reserve     - Reserve ring descriptor.
1112  * @ring: Handle to the ring object used for receive
1113  * @rxdh: Reserved descriptor. On success HW fills this "out" parameter
1114  * with a valid handle.
1115  *
1116  * Reserve Rx descriptor for the subsequent filling-in driver
1117  * and posting on the corresponding channel (@channelh)
1118  * via vxge_hw_ring_rxd_post().
1119  *
1120  * Returns: VXGE_HW_OK - success.
1121  * VXGE_HW_INF_OUT_OF_DESCRIPTORS - Currently no descriptors available.
1122  *
1123  */
1124 enum vxge_hw_status vxge_hw_ring_rxd_reserve(struct __vxge_hw_ring *ring,
1125         void **rxdh)
1126 {
1127         enum vxge_hw_status status;
1128         struct __vxge_hw_channel *channel;
1129
1130         channel = &ring->channel;
1131
1132         status = vxge_hw_channel_dtr_alloc(channel, rxdh);
1133
1134         if (status == VXGE_HW_OK) {
1135                 struct vxge_hw_ring_rxd_1 *rxdp =
1136                         (struct vxge_hw_ring_rxd_1 *)*rxdh;
1137
1138                 rxdp->control_0 = rxdp->control_1 = 0;
1139         }
1140
1141         return status;
1142 }
1143
1144 /**
1145  * vxge_hw_ring_rxd_free - Free descriptor.
1146  * @ring: Handle to the ring object used for receive
1147  * @rxdh: Descriptor handle.
1148  *
1149  * Free the reserved descriptor. This operation is "symmetrical" to
1150  * vxge_hw_ring_rxd_reserve. The "free-ing" completes the descriptor's
1151  * lifecycle.
1152  *
1153  * After free-ing (see vxge_hw_ring_rxd_free()) the descriptor again can
1154  * be:
1155  *
1156  * - reserved (vxge_hw_ring_rxd_reserve);
1157  *
1158  * - posted     (vxge_hw_ring_rxd_post);
1159  *
1160  * - completed (vxge_hw_ring_rxd_next_completed);
1161  *
1162  * - and recycled again (vxge_hw_ring_rxd_free).
1163  *
1164  * For alternative state transitions and more details please refer to
1165  * the design doc.
1166  *
1167  */
1168 void vxge_hw_ring_rxd_free(struct __vxge_hw_ring *ring, void *rxdh)
1169 {
1170         struct __vxge_hw_channel *channel;
1171
1172         channel = &ring->channel;
1173
1174         vxge_hw_channel_dtr_free(channel, rxdh);
1175
1176 }
1177
1178 /**
1179  * vxge_hw_ring_rxd_pre_post - Prepare rxd and post
1180  * @ring: Handle to the ring object used for receive
1181  * @rxdh: Descriptor handle.
1182  *
1183  * This routine prepares a rxd and posts
1184  */
1185 void vxge_hw_ring_rxd_pre_post(struct __vxge_hw_ring *ring, void *rxdh)
1186 {
1187         struct __vxge_hw_channel *channel;
1188
1189         channel = &ring->channel;
1190
1191         vxge_hw_channel_dtr_post(channel, rxdh);
1192 }
1193
1194 /**
1195  * vxge_hw_ring_rxd_post_post - Process rxd after post.
1196  * @ring: Handle to the ring object used for receive
1197  * @rxdh: Descriptor handle.
1198  *
1199  * Processes rxd after post
1200  */
1201 void vxge_hw_ring_rxd_post_post(struct __vxge_hw_ring *ring, void *rxdh)
1202 {
1203         struct vxge_hw_ring_rxd_1 *rxdp = (struct vxge_hw_ring_rxd_1 *)rxdh;
1204         struct __vxge_hw_channel *channel;
1205
1206         channel = &ring->channel;
1207
1208         rxdp->control_0 = VXGE_HW_RING_RXD_LIST_OWN_ADAPTER;
1209
1210         if (ring->stats->common_stats.usage_cnt > 0)
1211                 ring->stats->common_stats.usage_cnt--;
1212 }
1213
1214 /**
1215  * vxge_hw_ring_rxd_post - Post descriptor on the ring.
1216  * @ring: Handle to the ring object used for receive
1217  * @rxdh: Descriptor obtained via vxge_hw_ring_rxd_reserve().
1218  *
1219  * Post descriptor on the ring.
1220  * Prior to posting the descriptor should be filled in accordance with
1221  * Host/Titan interface specification for a given service (LL, etc.).
1222  *
1223  */
1224 void vxge_hw_ring_rxd_post(struct __vxge_hw_ring *ring, void *rxdh)
1225 {
1226         struct vxge_hw_ring_rxd_1 *rxdp = (struct vxge_hw_ring_rxd_1 *)rxdh;
1227         struct __vxge_hw_channel *channel;
1228
1229         channel = &ring->channel;
1230
1231         wmb();
1232         rxdp->control_0 = VXGE_HW_RING_RXD_LIST_OWN_ADAPTER;
1233
1234         vxge_hw_channel_dtr_post(channel, rxdh);
1235
1236         if (ring->stats->common_stats.usage_cnt > 0)
1237                 ring->stats->common_stats.usage_cnt--;
1238 }
1239
1240 /**
1241  * vxge_hw_ring_rxd_post_post_wmb - Process rxd after post with memory barrier.
1242  * @ring: Handle to the ring object used for receive
1243  * @rxdh: Descriptor handle.
1244  *
1245  * Processes rxd after post with memory barrier.
1246  */
1247 void vxge_hw_ring_rxd_post_post_wmb(struct __vxge_hw_ring *ring, void *rxdh)
1248 {
1249         wmb();
1250         vxge_hw_ring_rxd_post_post(ring, rxdh);
1251 }
1252
1253 /**
1254  * vxge_hw_ring_rxd_next_completed - Get the _next_ completed descriptor.
1255  * @ring: Handle to the ring object used for receive
1256  * @rxdh: Descriptor handle. Returned by HW.
1257  * @t_code:     Transfer code, as per Titan User Guide,
1258  *       Receive Descriptor Format. Returned by HW.
1259  *
1260  * Retrieve the _next_ completed descriptor.
1261  * HW uses ring callback (*vxge_hw_ring_callback_f) to notifiy
1262  * driver of new completed descriptors. After that
1263  * the driver can use vxge_hw_ring_rxd_next_completed to retrieve the rest
1264  * completions (the very first completion is passed by HW via
1265  * vxge_hw_ring_callback_f).
1266  *
1267  * Implementation-wise, the driver is free to call
1268  * vxge_hw_ring_rxd_next_completed either immediately from inside the
1269  * ring callback, or in a deferred fashion and separate (from HW)
1270  * context.
1271  *
1272  * Non-zero @t_code means failure to fill-in receive buffer(s)
1273  * of the descriptor.
1274  * For instance, parity error detected during the data transfer.
1275  * In this case Titan will complete the descriptor and indicate
1276  * for the host that the received data is not to be used.
1277  * For details please refer to Titan User Guide.
1278  *
1279  * Returns: VXGE_HW_OK - success.
1280  * VXGE_HW_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors
1281  * are currently available for processing.
1282  *
1283  * See also: vxge_hw_ring_callback_f{},
1284  * vxge_hw_fifo_rxd_next_completed(), enum vxge_hw_status{}.
1285  */
1286 enum vxge_hw_status vxge_hw_ring_rxd_next_completed(
1287         struct __vxge_hw_ring *ring, void **rxdh, u8 *t_code)
1288 {
1289         struct __vxge_hw_channel *channel;
1290         struct vxge_hw_ring_rxd_1 *rxdp;
1291         enum vxge_hw_status status = VXGE_HW_OK;
1292         u64 control_0, own;
1293
1294         channel = &ring->channel;
1295
1296         vxge_hw_channel_dtr_try_complete(channel, rxdh);
1297
1298         rxdp = *rxdh;
1299         if (rxdp == NULL) {
1300                 status = VXGE_HW_INF_NO_MORE_COMPLETED_DESCRIPTORS;
1301                 goto exit;
1302         }
1303
1304         control_0 = rxdp->control_0;
1305         own = control_0 & VXGE_HW_RING_RXD_LIST_OWN_ADAPTER;
1306         *t_code = (u8)VXGE_HW_RING_RXD_T_CODE_GET(control_0);
1307
1308         /* check whether it is not the end */
1309         if (!own || *t_code == VXGE_HW_RING_T_CODE_FRM_DROP) {
1310
1311                 vxge_assert((rxdp)->host_control !=
1312                                 0);
1313
1314                 ++ring->cmpl_cnt;
1315                 vxge_hw_channel_dtr_complete(channel);
1316
1317                 vxge_assert(*t_code != VXGE_HW_RING_RXD_T_CODE_UNUSED);
1318
1319                 ring->stats->common_stats.usage_cnt++;
1320                 if (ring->stats->common_stats.usage_max <
1321                                 ring->stats->common_stats.usage_cnt)
1322                         ring->stats->common_stats.usage_max =
1323                                 ring->stats->common_stats.usage_cnt;
1324
1325                 status = VXGE_HW_OK;
1326                 goto exit;
1327         }
1328
1329         /* reset it. since we don't want to return
1330          * garbage to the driver */
1331         *rxdh = NULL;
1332         status = VXGE_HW_INF_NO_MORE_COMPLETED_DESCRIPTORS;
1333 exit:
1334         return status;
1335 }
1336
1337 /**
1338  * vxge_hw_ring_handle_tcode - Handle transfer code.
1339  * @ring: Handle to the ring object used for receive
1340  * @rxdh: Descriptor handle.
1341  * @t_code: One of the enumerated (and documented in the Titan user guide)
1342  * "transfer codes".
1343  *
1344  * Handle descriptor's transfer code. The latter comes with each completed
1345  * descriptor.
1346  *
1347  * Returns: one of the enum vxge_hw_status{} enumerated types.
1348  * VXGE_HW_OK                   - for success.
1349  * VXGE_HW_ERR_CRITICAL         - when encounters critical error.
1350  */
1351 enum vxge_hw_status vxge_hw_ring_handle_tcode(
1352         struct __vxge_hw_ring *ring, void *rxdh, u8 t_code)
1353 {
1354         struct __vxge_hw_channel *channel;
1355         enum vxge_hw_status status = VXGE_HW_OK;
1356
1357         channel = &ring->channel;
1358
1359         /* If the t_code is not supported and if the
1360          * t_code is other than 0x5 (unparseable packet
1361          * such as unknown UPV6 header), Drop it !!!
1362          */
1363
1364         if (t_code ==  VXGE_HW_RING_T_CODE_OK ||
1365                 t_code == VXGE_HW_RING_T_CODE_L3_PKT_ERR) {
1366                 status = VXGE_HW_OK;
1367                 goto exit;
1368         }
1369
1370         if (t_code > VXGE_HW_RING_T_CODE_MULTI_ERR) {
1371                 status = VXGE_HW_ERR_INVALID_TCODE;
1372                 goto exit;
1373         }
1374
1375         ring->stats->rxd_t_code_err_cnt[t_code]++;
1376 exit:
1377         return status;
1378 }
1379
1380 /**
1381  * __vxge_hw_non_offload_db_post - Post non offload doorbell
1382  *
1383  * @fifo: fifohandle
1384  * @txdl_ptr: The starting location of the TxDL in host memory
1385  * @num_txds: The highest TxD in this TxDL (0 to 255 means 1 to 256)
1386  * @no_snoop: No snoop flags
1387  *
1388  * This function posts a non-offload doorbell to doorbell FIFO
1389  *
1390  */
1391 static void __vxge_hw_non_offload_db_post(struct __vxge_hw_fifo *fifo,
1392         u64 txdl_ptr, u32 num_txds, u32 no_snoop)
1393 {
1394         struct __vxge_hw_channel *channel;
1395
1396         channel = &fifo->channel;
1397
1398         writeq(VXGE_HW_NODBW_TYPE(VXGE_HW_NODBW_TYPE_NODBW) |
1399                 VXGE_HW_NODBW_LAST_TXD_NUMBER(num_txds) |
1400                 VXGE_HW_NODBW_GET_NO_SNOOP(no_snoop),
1401                 &fifo->nofl_db->control_0);
1402
1403         mmiowb();
1404
1405         writeq(txdl_ptr, &fifo->nofl_db->txdl_ptr);
1406
1407         mmiowb();
1408 }
1409
1410 /**
1411  * vxge_hw_fifo_free_txdl_count_get - returns the number of txdls available in
1412  * the fifo
1413  * @fifoh: Handle to the fifo object used for non offload send
1414  */
1415 u32 vxge_hw_fifo_free_txdl_count_get(struct __vxge_hw_fifo *fifoh)
1416 {
1417         return vxge_hw_channel_dtr_count(&fifoh->channel);
1418 }
1419
1420 /**
1421  * vxge_hw_fifo_txdl_reserve - Reserve fifo descriptor.
1422  * @fifoh: Handle to the fifo object used for non offload send
1423  * @txdlh: Reserved descriptor. On success HW fills this "out" parameter
1424  *        with a valid handle.
1425  * @txdl_priv: Buffer to return the pointer to per txdl space
1426  *
1427  * Reserve a single TxDL (that is, fifo descriptor)
1428  * for the subsequent filling-in by driver)
1429  * and posting on the corresponding channel (@channelh)
1430  * via vxge_hw_fifo_txdl_post().
1431  *
1432  * Note: it is the responsibility of driver to reserve multiple descriptors
1433  * for lengthy (e.g., LSO) transmit operation. A single fifo descriptor
1434  * carries up to configured number (fifo.max_frags) of contiguous buffers.
1435  *
1436  * Returns: VXGE_HW_OK - success;
1437  * VXGE_HW_INF_OUT_OF_DESCRIPTORS - Currently no descriptors available
1438  *
1439  */
1440 enum vxge_hw_status vxge_hw_fifo_txdl_reserve(
1441         struct __vxge_hw_fifo *fifo,
1442         void **txdlh, void **txdl_priv)
1443 {
1444         struct __vxge_hw_channel *channel;
1445         enum vxge_hw_status status;
1446         int i;
1447
1448         channel = &fifo->channel;
1449
1450         status = vxge_hw_channel_dtr_alloc(channel, txdlh);
1451
1452         if (status == VXGE_HW_OK) {
1453                 struct vxge_hw_fifo_txd *txdp =
1454                         (struct vxge_hw_fifo_txd *)*txdlh;
1455                 struct __vxge_hw_fifo_txdl_priv *priv;
1456
1457                 priv = __vxge_hw_fifo_txdl_priv(fifo, txdp);
1458
1459                 /* reset the TxDL's private */
1460                 priv->align_dma_offset = 0;
1461                 priv->align_vaddr_start = priv->align_vaddr;
1462                 priv->align_used_frags = 0;
1463                 priv->frags = 0;
1464                 priv->alloc_frags = fifo->config->max_frags;
1465                 priv->next_txdl_priv = NULL;
1466
1467                 *txdl_priv = (void *)(size_t)txdp->host_control;
1468
1469                 for (i = 0; i < fifo->config->max_frags; i++) {
1470                         txdp = ((struct vxge_hw_fifo_txd *)*txdlh) + i;
1471                         txdp->control_0 = txdp->control_1 = 0;
1472                 }
1473         }
1474
1475         return status;
1476 }
1477
1478 /**
1479  * vxge_hw_fifo_txdl_buffer_set - Set transmit buffer pointer in the
1480  * descriptor.
1481  * @fifo: Handle to the fifo object used for non offload send
1482  * @txdlh: Descriptor handle.
1483  * @frag_idx: Index of the data buffer in the caller's scatter-gather list
1484  *            (of buffers).
1485  * @dma_pointer: DMA address of the data buffer referenced by @frag_idx.
1486  * @size: Size of the data buffer (in bytes).
1487  *
1488  * This API is part of the preparation of the transmit descriptor for posting
1489  * (via vxge_hw_fifo_txdl_post()). The related "preparation" APIs include
1490  * vxge_hw_fifo_txdl_mss_set() and vxge_hw_fifo_txdl_cksum_set_bits().
1491  * All three APIs fill in the fields of the fifo descriptor,
1492  * in accordance with the Titan specification.
1493  *
1494  */
1495 void vxge_hw_fifo_txdl_buffer_set(struct __vxge_hw_fifo *fifo,
1496                                   void *txdlh, u32 frag_idx,
1497                                   dma_addr_t dma_pointer, u32 size)
1498 {
1499         struct __vxge_hw_fifo_txdl_priv *txdl_priv;
1500         struct vxge_hw_fifo_txd *txdp, *txdp_last;
1501         struct __vxge_hw_channel *channel;
1502
1503         channel = &fifo->channel;
1504
1505         txdl_priv = __vxge_hw_fifo_txdl_priv(fifo, txdlh);
1506         txdp = (struct vxge_hw_fifo_txd *)txdlh  +  txdl_priv->frags;
1507
1508         if (frag_idx != 0)
1509                 txdp->control_0 = txdp->control_1 = 0;
1510         else {
1511                 txdp->control_0 |= VXGE_HW_FIFO_TXD_GATHER_CODE(
1512                         VXGE_HW_FIFO_TXD_GATHER_CODE_FIRST);
1513                 txdp->control_1 |= fifo->interrupt_type;
1514                 txdp->control_1 |= VXGE_HW_FIFO_TXD_INT_NUMBER(
1515                         fifo->tx_intr_num);
1516                 if (txdl_priv->frags) {
1517                         txdp_last = (struct vxge_hw_fifo_txd *)txdlh  +
1518                         (txdl_priv->frags - 1);
1519                         txdp_last->control_0 |= VXGE_HW_FIFO_TXD_GATHER_CODE(
1520                                 VXGE_HW_FIFO_TXD_GATHER_CODE_LAST);
1521                 }
1522         }
1523
1524         vxge_assert(frag_idx < txdl_priv->alloc_frags);
1525
1526         txdp->buffer_pointer = (u64)dma_pointer;
1527         txdp->control_0 |= VXGE_HW_FIFO_TXD_BUFFER_SIZE(size);
1528         fifo->stats->total_buffers++;
1529         txdl_priv->frags++;
1530 }
1531
1532 /**
1533  * vxge_hw_fifo_txdl_post - Post descriptor on the fifo channel.
1534  * @fifo: Handle to the fifo object used for non offload send
1535  * @txdlh: Descriptor obtained via vxge_hw_fifo_txdl_reserve()
1536  * @frags: Number of contiguous buffers that are part of a single
1537  *         transmit operation.
1538  *
1539  * Post descriptor on the 'fifo' type channel for transmission.
1540  * Prior to posting the descriptor should be filled in accordance with
1541  * Host/Titan interface specification for a given service (LL, etc.).
1542  *
1543  */
1544 void vxge_hw_fifo_txdl_post(struct __vxge_hw_fifo *fifo, void *txdlh)
1545 {
1546         struct __vxge_hw_fifo_txdl_priv *txdl_priv;
1547         struct vxge_hw_fifo_txd *txdp_last;
1548         struct vxge_hw_fifo_txd *txdp_first;
1549         struct __vxge_hw_channel *channel;
1550
1551         channel = &fifo->channel;
1552
1553         txdl_priv = __vxge_hw_fifo_txdl_priv(fifo, txdlh);
1554         txdp_first = txdlh;
1555
1556         txdp_last = (struct vxge_hw_fifo_txd *)txdlh  +  (txdl_priv->frags - 1);
1557         txdp_last->control_0 |=
1558               VXGE_HW_FIFO_TXD_GATHER_CODE(VXGE_HW_FIFO_TXD_GATHER_CODE_LAST);
1559         txdp_first->control_0 |= VXGE_HW_FIFO_TXD_LIST_OWN_ADAPTER;
1560
1561         vxge_hw_channel_dtr_post(&fifo->channel, txdlh);
1562
1563         __vxge_hw_non_offload_db_post(fifo,
1564                 (u64)txdl_priv->dma_addr,
1565                 txdl_priv->frags - 1,
1566                 fifo->no_snoop_bits);
1567
1568         fifo->stats->total_posts++;
1569         fifo->stats->common_stats.usage_cnt++;
1570         if (fifo->stats->common_stats.usage_max <
1571                 fifo->stats->common_stats.usage_cnt)
1572                 fifo->stats->common_stats.usage_max =
1573                         fifo->stats->common_stats.usage_cnt;
1574 }
1575
1576 /**
1577  * vxge_hw_fifo_txdl_next_completed - Retrieve next completed descriptor.
1578  * @fifo: Handle to the fifo object used for non offload send
1579  * @txdlh: Descriptor handle. Returned by HW.
1580  * @t_code: Transfer code, as per Titan User Guide,
1581  *          Transmit Descriptor Format.
1582  *          Returned by HW.
1583  *
1584  * Retrieve the _next_ completed descriptor.
1585  * HW uses channel callback (*vxge_hw_channel_callback_f) to notifiy
1586  * driver of new completed descriptors. After that
1587  * the driver can use vxge_hw_fifo_txdl_next_completed to retrieve the rest
1588  * completions (the very first completion is passed by HW via
1589  * vxge_hw_channel_callback_f).
1590  *
1591  * Implementation-wise, the driver is free to call
1592  * vxge_hw_fifo_txdl_next_completed either immediately from inside the
1593  * channel callback, or in a deferred fashion and separate (from HW)
1594  * context.
1595  *
1596  * Non-zero @t_code means failure to process the descriptor.
1597  * The failure could happen, for instance, when the link is
1598  * down, in which case Titan completes the descriptor because it
1599  * is not able to send the data out.
1600  *
1601  * For details please refer to Titan User Guide.
1602  *
1603  * Returns: VXGE_HW_OK - success.
1604  * VXGE_HW_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors
1605  * are currently available for processing.
1606  *
1607  */
1608 enum vxge_hw_status vxge_hw_fifo_txdl_next_completed(
1609         struct __vxge_hw_fifo *fifo, void **txdlh,
1610         enum vxge_hw_fifo_tcode *t_code)
1611 {
1612         struct __vxge_hw_channel *channel;
1613         struct vxge_hw_fifo_txd *txdp;
1614         enum vxge_hw_status status = VXGE_HW_OK;
1615
1616         channel = &fifo->channel;
1617
1618         vxge_hw_channel_dtr_try_complete(channel, txdlh);
1619
1620         txdp = *txdlh;
1621         if (txdp == NULL) {
1622                 status = VXGE_HW_INF_NO_MORE_COMPLETED_DESCRIPTORS;
1623                 goto exit;
1624         }
1625
1626         /* check whether host owns it */
1627         if (!(txdp->control_0 & VXGE_HW_FIFO_TXD_LIST_OWN_ADAPTER)) {
1628
1629                 vxge_assert(txdp->host_control != 0);
1630
1631                 vxge_hw_channel_dtr_complete(channel);
1632
1633                 *t_code = (u8)VXGE_HW_FIFO_TXD_T_CODE_GET(txdp->control_0);
1634
1635                 if (fifo->stats->common_stats.usage_cnt > 0)
1636                         fifo->stats->common_stats.usage_cnt--;
1637
1638                 status = VXGE_HW_OK;
1639                 goto exit;
1640         }
1641
1642         /* no more completions */
1643         *txdlh = NULL;
1644         status = VXGE_HW_INF_NO_MORE_COMPLETED_DESCRIPTORS;
1645 exit:
1646         return status;
1647 }
1648
1649 /**
1650  * vxge_hw_fifo_handle_tcode - Handle transfer code.
1651  * @fifo: Handle to the fifo object used for non offload send
1652  * @txdlh: Descriptor handle.
1653  * @t_code: One of the enumerated (and documented in the Titan user guide)
1654  *          "transfer codes".
1655  *
1656  * Handle descriptor's transfer code. The latter comes with each completed
1657  * descriptor.
1658  *
1659  * Returns: one of the enum vxge_hw_status{} enumerated types.
1660  * VXGE_HW_OK - for success.
1661  * VXGE_HW_ERR_CRITICAL - when encounters critical error.
1662  */
1663 enum vxge_hw_status vxge_hw_fifo_handle_tcode(struct __vxge_hw_fifo *fifo,
1664                                               void *txdlh,
1665                                               enum vxge_hw_fifo_tcode t_code)
1666 {
1667         struct __vxge_hw_channel *channel;
1668
1669         enum vxge_hw_status status = VXGE_HW_OK;
1670         channel = &fifo->channel;
1671
1672         if (((t_code & 0x7) < 0) || ((t_code & 0x7) > 0x4)) {
1673                 status = VXGE_HW_ERR_INVALID_TCODE;
1674                 goto exit;
1675         }
1676
1677         fifo->stats->txd_t_code_err_cnt[t_code]++;
1678 exit:
1679         return status;
1680 }
1681
1682 /**
1683  * vxge_hw_fifo_txdl_free - Free descriptor.
1684  * @fifo: Handle to the fifo object used for non offload send
1685  * @txdlh: Descriptor handle.
1686  *
1687  * Free the reserved descriptor. This operation is "symmetrical" to
1688  * vxge_hw_fifo_txdl_reserve. The "free-ing" completes the descriptor's
1689  * lifecycle.
1690  *
1691  * After free-ing (see vxge_hw_fifo_txdl_free()) the descriptor again can
1692  * be:
1693  *
1694  * - reserved (vxge_hw_fifo_txdl_reserve);
1695  *
1696  * - posted (vxge_hw_fifo_txdl_post);
1697  *
1698  * - completed (vxge_hw_fifo_txdl_next_completed);
1699  *
1700  * - and recycled again (vxge_hw_fifo_txdl_free).
1701  *
1702  * For alternative state transitions and more details please refer to
1703  * the design doc.
1704  *
1705  */
1706 void vxge_hw_fifo_txdl_free(struct __vxge_hw_fifo *fifo, void *txdlh)
1707 {
1708         struct __vxge_hw_fifo_txdl_priv *txdl_priv;
1709         u32 max_frags;
1710         struct __vxge_hw_channel *channel;
1711
1712         channel = &fifo->channel;
1713
1714         txdl_priv = __vxge_hw_fifo_txdl_priv(fifo,
1715                         (struct vxge_hw_fifo_txd *)txdlh);
1716
1717         max_frags = fifo->config->max_frags;
1718
1719         vxge_hw_channel_dtr_free(channel, txdlh);
1720 }
1721
1722 /**
1723  * vxge_hw_vpath_mac_addr_add - Add the mac address entry for this vpath
1724  *               to MAC address table.
1725  * @vp: Vpath handle.
1726  * @macaddr: MAC address to be added for this vpath into the list
1727  * @macaddr_mask: MAC address mask for macaddr
1728  * @duplicate_mode: Duplicate MAC address add mode. Please see
1729  *             enum vxge_hw_vpath_mac_addr_add_mode{}
1730  *
1731  * Adds the given mac address and mac address mask into the list for this
1732  * vpath.
1733  * see also: vxge_hw_vpath_mac_addr_delete, vxge_hw_vpath_mac_addr_get and
1734  * vxge_hw_vpath_mac_addr_get_next
1735  *
1736  */
1737 enum vxge_hw_status
1738 vxge_hw_vpath_mac_addr_add(
1739         struct __vxge_hw_vpath_handle *vp,
1740         u8 *macaddr,
1741         u8 *macaddr_mask,
1742         enum vxge_hw_vpath_mac_addr_add_mode duplicate_mode)
1743 {
1744         u32 i;
1745         u64 data1 = 0ULL;
1746         u64 data2 = 0ULL;
1747         enum vxge_hw_status status = VXGE_HW_OK;
1748
1749         if (vp == NULL) {
1750                 status = VXGE_HW_ERR_INVALID_HANDLE;
1751                 goto exit;
1752         }
1753
1754         for (i = 0; i < ETH_ALEN; i++) {
1755                 data1 <<= 8;
1756                 data1 |= (u8)macaddr[i];
1757
1758                 data2 <<= 8;
1759                 data2 |= (u8)macaddr_mask[i];
1760         }
1761
1762         switch (duplicate_mode) {
1763         case VXGE_HW_VPATH_MAC_ADDR_ADD_DUPLICATE:
1764                 i = 0;
1765                 break;
1766         case VXGE_HW_VPATH_MAC_ADDR_DISCARD_DUPLICATE:
1767                 i = 1;
1768                 break;
1769         case VXGE_HW_VPATH_MAC_ADDR_REPLACE_DUPLICATE:
1770                 i = 2;
1771                 break;
1772         default:
1773                 i = 0;
1774                 break;
1775         }
1776
1777         status = __vxge_hw_vpath_rts_table_set(vp,
1778                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_ADD_ENTRY,
1779                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA,
1780                         0,
1781                         VXGE_HW_RTS_ACCESS_STEER_DATA0_DA_MAC_ADDR(data1),
1782                         VXGE_HW_RTS_ACCESS_STEER_DATA1_DA_MAC_ADDR_MASK(data2)|
1783                         VXGE_HW_RTS_ACCESS_STEER_DATA1_DA_MAC_ADDR_MODE(i));
1784 exit:
1785         return status;
1786 }
1787
1788 /**
1789  * vxge_hw_vpath_mac_addr_get - Get the first mac address entry for this vpath
1790  *               from MAC address table.
1791  * @vp: Vpath handle.
1792  * @macaddr: First MAC address entry for this vpath in the list
1793  * @macaddr_mask: MAC address mask for macaddr
1794  *
1795  * Returns the first mac address and mac address mask in the list for this
1796  * vpath.
1797  * see also: vxge_hw_vpath_mac_addr_get_next
1798  *
1799  */
1800 enum vxge_hw_status
1801 vxge_hw_vpath_mac_addr_get(
1802         struct __vxge_hw_vpath_handle *vp,
1803         u8 *macaddr,
1804         u8 *macaddr_mask)
1805 {
1806         u32 i;
1807         u64 data1 = 0ULL;
1808         u64 data2 = 0ULL;
1809         enum vxge_hw_status status = VXGE_HW_OK;
1810
1811         if (vp == NULL) {
1812                 status = VXGE_HW_ERR_INVALID_HANDLE;
1813                 goto exit;
1814         }
1815
1816         status = __vxge_hw_vpath_rts_table_get(vp,
1817                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_LIST_FIRST_ENTRY,
1818                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA,
1819                         0, &data1, &data2);
1820
1821         if (status != VXGE_HW_OK)
1822                 goto exit;
1823
1824         data1 = VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_DA_MAC_ADDR(data1);
1825
1826         data2 = VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_DA_MAC_ADDR_MASK(data2);
1827
1828         for (i = ETH_ALEN; i > 0; i--) {
1829                 macaddr[i-1] = (u8)(data1 & 0xFF);
1830                 data1 >>= 8;
1831
1832                 macaddr_mask[i-1] = (u8)(data2 & 0xFF);
1833                 data2 >>= 8;
1834         }
1835 exit:
1836         return status;
1837 }
1838
1839 /**
1840  * vxge_hw_vpath_mac_addr_get_next - Get the next mac address entry for this
1841  * vpath
1842  *               from MAC address table.
1843  * @vp: Vpath handle.
1844  * @macaddr: Next MAC address entry for this vpath in the list
1845  * @macaddr_mask: MAC address mask for macaddr
1846  *
1847  * Returns the next mac address and mac address mask in the list for this
1848  * vpath.
1849  * see also: vxge_hw_vpath_mac_addr_get
1850  *
1851  */
1852 enum vxge_hw_status
1853 vxge_hw_vpath_mac_addr_get_next(
1854         struct __vxge_hw_vpath_handle *vp,
1855         u8 *macaddr,
1856         u8 *macaddr_mask)
1857 {
1858         u32 i;
1859         u64 data1 = 0ULL;
1860         u64 data2 = 0ULL;
1861         enum vxge_hw_status status = VXGE_HW_OK;
1862
1863         if (vp == NULL) {
1864                 status = VXGE_HW_ERR_INVALID_HANDLE;
1865                 goto exit;
1866         }
1867
1868         status = __vxge_hw_vpath_rts_table_get(vp,
1869                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_LIST_NEXT_ENTRY,
1870                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA,
1871                         0, &data1, &data2);
1872
1873         if (status != VXGE_HW_OK)
1874                 goto exit;
1875
1876         data1 = VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_DA_MAC_ADDR(data1);
1877
1878         data2 = VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_DA_MAC_ADDR_MASK(data2);
1879
1880         for (i = ETH_ALEN; i > 0; i--) {
1881                 macaddr[i-1] = (u8)(data1 & 0xFF);
1882                 data1 >>= 8;
1883
1884                 macaddr_mask[i-1] = (u8)(data2 & 0xFF);
1885                 data2 >>= 8;
1886         }
1887
1888 exit:
1889         return status;
1890 }
1891
1892 /**
1893  * vxge_hw_vpath_mac_addr_delete - Delete the mac address entry for this vpath
1894  *               to MAC address table.
1895  * @vp: Vpath handle.
1896  * @macaddr: MAC address to be added for this vpath into the list
1897  * @macaddr_mask: MAC address mask for macaddr
1898  *
1899  * Delete the given mac address and mac address mask into the list for this
1900  * vpath.
1901  * see also: vxge_hw_vpath_mac_addr_add, vxge_hw_vpath_mac_addr_get and
1902  * vxge_hw_vpath_mac_addr_get_next
1903  *
1904  */
1905 enum vxge_hw_status
1906 vxge_hw_vpath_mac_addr_delete(
1907         struct __vxge_hw_vpath_handle *vp,
1908         u8 *macaddr,
1909         u8 *macaddr_mask)
1910 {
1911         u32 i;
1912         u64 data1 = 0ULL;
1913         u64 data2 = 0ULL;
1914         enum vxge_hw_status status = VXGE_HW_OK;
1915
1916         if (vp == NULL) {
1917                 status = VXGE_HW_ERR_INVALID_HANDLE;
1918                 goto exit;
1919         }
1920
1921         for (i = 0; i < ETH_ALEN; i++) {
1922                 data1 <<= 8;
1923                 data1 |= (u8)macaddr[i];
1924
1925                 data2 <<= 8;
1926                 data2 |= (u8)macaddr_mask[i];
1927         }
1928
1929         status = __vxge_hw_vpath_rts_table_set(vp,
1930                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_DELETE_ENTRY,
1931                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA,
1932                         0,
1933                         VXGE_HW_RTS_ACCESS_STEER_DATA0_DA_MAC_ADDR(data1),
1934                         VXGE_HW_RTS_ACCESS_STEER_DATA1_DA_MAC_ADDR_MASK(data2));
1935 exit:
1936         return status;
1937 }
1938
1939 /**
1940  * vxge_hw_vpath_vid_add - Add the vlan id entry for this vpath
1941  *               to vlan id table.
1942  * @vp: Vpath handle.
1943  * @vid: vlan id to be added for this vpath into the list
1944  *
1945  * Adds the given vlan id into the list for this  vpath.
1946  * see also: vxge_hw_vpath_vid_delete
1947  *
1948  */
1949 enum vxge_hw_status
1950 vxge_hw_vpath_vid_add(struct __vxge_hw_vpath_handle *vp, u64 vid)
1951 {
1952         enum vxge_hw_status status = VXGE_HW_OK;
1953
1954         if (vp == NULL) {
1955                 status = VXGE_HW_ERR_INVALID_HANDLE;
1956                 goto exit;
1957         }
1958
1959         status = __vxge_hw_vpath_rts_table_set(vp,
1960                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_ADD_ENTRY,
1961                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_VID,
1962                         0, VXGE_HW_RTS_ACCESS_STEER_DATA0_VLAN_ID(vid), 0);
1963 exit:
1964         return status;
1965 }
1966
1967 /**
1968  * vxge_hw_vpath_vid_delete - Delete the vlan id entry for this vpath
1969  *               to vlan id table.
1970  * @vp: Vpath handle.
1971  * @vid: vlan id to be added for this vpath into the list
1972  *
1973  * Adds the given vlan id into the list for this  vpath.
1974  * see also: vxge_hw_vpath_vid_add
1975  *
1976  */
1977 enum vxge_hw_status
1978 vxge_hw_vpath_vid_delete(struct __vxge_hw_vpath_handle *vp, u64 vid)
1979 {
1980         enum vxge_hw_status status = VXGE_HW_OK;
1981
1982         if (vp == NULL) {
1983                 status = VXGE_HW_ERR_INVALID_HANDLE;
1984                 goto exit;
1985         }
1986
1987         status = __vxge_hw_vpath_rts_table_set(vp,
1988                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_DELETE_ENTRY,
1989                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_VID,
1990                         0, VXGE_HW_RTS_ACCESS_STEER_DATA0_VLAN_ID(vid), 0);
1991 exit:
1992         return status;
1993 }
1994
1995 /**
1996  * vxge_hw_vpath_promisc_enable - Enable promiscuous mode.
1997  * @vp: Vpath handle.
1998  *
1999  * Enable promiscuous mode of Titan-e operation.
2000  *
2001  * See also: vxge_hw_vpath_promisc_disable().
2002  */
2003 enum vxge_hw_status vxge_hw_vpath_promisc_enable(
2004                         struct __vxge_hw_vpath_handle *vp)
2005 {
2006         u64 val64;
2007         struct __vxge_hw_virtualpath *vpath;
2008         enum vxge_hw_status status = VXGE_HW_OK;
2009
2010         if ((vp == NULL) || (vp->vpath->ringh == NULL)) {
2011                 status = VXGE_HW_ERR_INVALID_HANDLE;
2012                 goto exit;
2013         }
2014
2015         vpath = vp->vpath;
2016
2017         /* Enable promiscuous mode for function 0 only */
2018         if (!(vpath->hldev->access_rights &
2019                 VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM))
2020                 return VXGE_HW_OK;
2021
2022         val64 = readq(&vpath->vp_reg->rxmac_vcfg0);
2023
2024         if (!(val64 & VXGE_HW_RXMAC_VCFG0_UCAST_ALL_ADDR_EN)) {
2025
2026                 val64 |= VXGE_HW_RXMAC_VCFG0_UCAST_ALL_ADDR_EN |
2027                          VXGE_HW_RXMAC_VCFG0_MCAST_ALL_ADDR_EN |
2028                          VXGE_HW_RXMAC_VCFG0_BCAST_EN |
2029                          VXGE_HW_RXMAC_VCFG0_ALL_VID_EN;
2030
2031                 writeq(val64, &vpath->vp_reg->rxmac_vcfg0);
2032         }
2033 exit:
2034         return status;
2035 }
2036
2037 /**
2038  * vxge_hw_vpath_promisc_disable - Disable promiscuous mode.
2039  * @vp: Vpath handle.
2040  *
2041  * Disable promiscuous mode of Titan-e operation.
2042  *
2043  * See also: vxge_hw_vpath_promisc_enable().
2044  */
2045 enum vxge_hw_status vxge_hw_vpath_promisc_disable(
2046                         struct __vxge_hw_vpath_handle *vp)
2047 {
2048         u64 val64;
2049         struct __vxge_hw_virtualpath *vpath;
2050         enum vxge_hw_status status = VXGE_HW_OK;
2051
2052         if ((vp == NULL) || (vp->vpath->ringh == NULL)) {
2053                 status = VXGE_HW_ERR_INVALID_HANDLE;
2054                 goto exit;
2055         }
2056
2057         vpath = vp->vpath;
2058
2059         val64 = readq(&vpath->vp_reg->rxmac_vcfg0);
2060
2061         if (val64 & VXGE_HW_RXMAC_VCFG0_UCAST_ALL_ADDR_EN) {
2062
2063                 val64 &= ~(VXGE_HW_RXMAC_VCFG0_UCAST_ALL_ADDR_EN |
2064                            VXGE_HW_RXMAC_VCFG0_MCAST_ALL_ADDR_EN |
2065                            VXGE_HW_RXMAC_VCFG0_ALL_VID_EN);
2066
2067                 writeq(val64, &vpath->vp_reg->rxmac_vcfg0);
2068         }
2069 exit:
2070         return status;
2071 }
2072
2073 /*
2074  * vxge_hw_vpath_bcast_enable - Enable broadcast
2075  * @vp: Vpath handle.
2076  *
2077  * Enable receiving broadcasts.
2078  */
2079 enum vxge_hw_status vxge_hw_vpath_bcast_enable(
2080                         struct __vxge_hw_vpath_handle *vp)
2081 {
2082         u64 val64;
2083         struct __vxge_hw_virtualpath *vpath;
2084         enum vxge_hw_status status = VXGE_HW_OK;
2085
2086         if ((vp == NULL) || (vp->vpath->ringh == NULL)) {
2087                 status = VXGE_HW_ERR_INVALID_HANDLE;
2088                 goto exit;
2089         }
2090
2091         vpath = vp->vpath;
2092
2093         val64 = readq(&vpath->vp_reg->rxmac_vcfg0);
2094
2095         if (!(val64 & VXGE_HW_RXMAC_VCFG0_BCAST_EN)) {
2096                 val64 |= VXGE_HW_RXMAC_VCFG0_BCAST_EN;
2097                 writeq(val64, &vpath->vp_reg->rxmac_vcfg0);
2098         }
2099 exit:
2100         return status;
2101 }
2102
2103 /**
2104  * vxge_hw_vpath_mcast_enable - Enable multicast addresses.
2105  * @vp: Vpath handle.
2106  *
2107  * Enable Titan-e multicast addresses.
2108  * Returns: VXGE_HW_OK on success.
2109  *
2110  */
2111 enum vxge_hw_status vxge_hw_vpath_mcast_enable(
2112                         struct __vxge_hw_vpath_handle *vp)
2113 {
2114         u64 val64;
2115         struct __vxge_hw_virtualpath *vpath;
2116         enum vxge_hw_status status = VXGE_HW_OK;
2117
2118         if ((vp == NULL) || (vp->vpath->ringh == NULL)) {
2119                 status = VXGE_HW_ERR_INVALID_HANDLE;
2120                 goto exit;
2121         }
2122
2123         vpath = vp->vpath;
2124
2125         val64 = readq(&vpath->vp_reg->rxmac_vcfg0);
2126
2127         if (!(val64 & VXGE_HW_RXMAC_VCFG0_MCAST_ALL_ADDR_EN)) {
2128                 val64 |= VXGE_HW_RXMAC_VCFG0_MCAST_ALL_ADDR_EN;
2129                 writeq(val64, &vpath->vp_reg->rxmac_vcfg0);
2130         }
2131 exit:
2132         return status;
2133 }
2134
2135 /**
2136  * vxge_hw_vpath_mcast_disable - Disable  multicast addresses.
2137  * @vp: Vpath handle.
2138  *
2139  * Disable Titan-e multicast addresses.
2140  * Returns: VXGE_HW_OK - success.
2141  * VXGE_HW_ERR_INVALID_HANDLE - Invalid handle
2142  *
2143  */
2144 enum vxge_hw_status
2145 vxge_hw_vpath_mcast_disable(struct __vxge_hw_vpath_handle *vp)
2146 {
2147         u64 val64;
2148         struct __vxge_hw_virtualpath *vpath;
2149         enum vxge_hw_status status = VXGE_HW_OK;
2150
2151         if ((vp == NULL) || (vp->vpath->ringh == NULL)) {
2152                 status = VXGE_HW_ERR_INVALID_HANDLE;
2153                 goto exit;
2154         }
2155
2156         vpath = vp->vpath;
2157
2158         val64 = readq(&vpath->vp_reg->rxmac_vcfg0);
2159
2160         if (val64 & VXGE_HW_RXMAC_VCFG0_MCAST_ALL_ADDR_EN) {
2161                 val64 &= ~VXGE_HW_RXMAC_VCFG0_MCAST_ALL_ADDR_EN;
2162                 writeq(val64, &vpath->vp_reg->rxmac_vcfg0);
2163         }
2164 exit:
2165         return status;
2166 }
2167
2168 /*
2169  * vxge_hw_vpath_alarm_process - Process Alarms.
2170  * @vpath: Virtual Path.
2171  * @skip_alarms: Do not clear the alarms
2172  *
2173  * Process vpath alarms.
2174  *
2175  */
2176 enum vxge_hw_status vxge_hw_vpath_alarm_process(
2177                         struct __vxge_hw_vpath_handle *vp,
2178                         u32 skip_alarms)
2179 {
2180         enum vxge_hw_status status = VXGE_HW_OK;
2181
2182         if (vp == NULL) {
2183                 status = VXGE_HW_ERR_INVALID_HANDLE;
2184                 goto exit;
2185         }
2186
2187         status = __vxge_hw_vpath_alarm_process(vp->vpath, skip_alarms);
2188 exit:
2189         return status;
2190 }
2191
2192 /**
2193  * vxge_hw_vpath_msix_set - Associate MSIX vectors with TIM interrupts and
2194  *                            alrms
2195  * @vp: Virtual Path handle.
2196  * @tim_msix_id: MSIX vectors associated with VXGE_HW_MAX_INTR_PER_VP number of
2197  *             interrupts(Can be repeated). If fifo or ring are not enabled
2198  *             the MSIX vector for that should be set to 0
2199  * @alarm_msix_id: MSIX vector for alarm.
2200  *
2201  * This API will associate a given MSIX vector numbers with the four TIM
2202  * interrupts and alarm interrupt.
2203  */
2204 void
2205 vxge_hw_vpath_msix_set(struct __vxge_hw_vpath_handle *vp, int *tim_msix_id,
2206                        int alarm_msix_id)
2207 {
2208         u64 val64;
2209         struct __vxge_hw_virtualpath *vpath = vp->vpath;
2210         struct vxge_hw_vpath_reg __iomem *vp_reg = vpath->vp_reg;
2211         u32 vp_id = vp->vpath->vp_id;
2212
2213         val64 =  VXGE_HW_INTERRUPT_CFG0_GROUP0_MSIX_FOR_TXTI(
2214                   (vp_id * 4) + tim_msix_id[0]) |
2215                  VXGE_HW_INTERRUPT_CFG0_GROUP1_MSIX_FOR_TXTI(
2216                   (vp_id * 4) + tim_msix_id[1]);
2217
2218         writeq(val64, &vp_reg->interrupt_cfg0);
2219
2220         writeq(VXGE_HW_INTERRUPT_CFG2_ALARM_MAP_TO_MSG(
2221                         (vpath->hldev->first_vp_id * 4) + alarm_msix_id),
2222                         &vp_reg->interrupt_cfg2);
2223
2224         if (vpath->hldev->config.intr_mode ==
2225                                         VXGE_HW_INTR_MODE_MSIX_ONE_SHOT) {
2226                 __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(
2227                                 VXGE_HW_ONE_SHOT_VECT0_EN_ONE_SHOT_VECT0_EN,
2228                                 0, 32), &vp_reg->one_shot_vect0_en);
2229                 __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(
2230                                 VXGE_HW_ONE_SHOT_VECT1_EN_ONE_SHOT_VECT1_EN,
2231                                 0, 32), &vp_reg->one_shot_vect1_en);
2232                 __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(
2233                                 VXGE_HW_ONE_SHOT_VECT2_EN_ONE_SHOT_VECT2_EN,
2234                                 0, 32), &vp_reg->one_shot_vect2_en);
2235         }
2236 }
2237
2238 /**
2239  * vxge_hw_vpath_msix_mask - Mask MSIX Vector.
2240  * @vp: Virtual Path handle.
2241  * @msix_id:  MSIX ID
2242  *
2243  * The function masks the msix interrupt for the given msix_id
2244  *
2245  * Returns: 0,
2246  * Otherwise, VXGE_HW_ERR_WRONG_IRQ if the msix index is out of range
2247  * status.
2248  * See also:
2249  */
2250 void
2251 vxge_hw_vpath_msix_mask(struct __vxge_hw_vpath_handle *vp, int msix_id)
2252 {
2253         struct __vxge_hw_device *hldev = vp->vpath->hldev;
2254         __vxge_hw_pio_mem_write32_upper(
2255                 (u32) vxge_bVALn(vxge_mBIT(msix_id  >> 2), 0, 32),
2256                 &hldev->common_reg->set_msix_mask_vect[msix_id % 4]);
2257 }
2258
2259 /**
2260  * vxge_hw_vpath_msix_clear - Clear MSIX Vector.
2261  * @vp: Virtual Path handle.
2262  * @msix_id:  MSI ID
2263  *
2264  * The function clears the msix interrupt for the given msix_id
2265  *
2266  * Returns: 0,
2267  * Otherwise, VXGE_HW_ERR_WRONG_IRQ if the msix index is out of range
2268  * status.
2269  * See also:
2270  */
2271 void vxge_hw_vpath_msix_clear(struct __vxge_hw_vpath_handle *vp, int msix_id)
2272 {
2273         struct __vxge_hw_device *hldev = vp->vpath->hldev;
2274
2275         if ((hldev->config.intr_mode == VXGE_HW_INTR_MODE_MSIX_ONE_SHOT))
2276                 __vxge_hw_pio_mem_write32_upper(
2277                         (u32) vxge_bVALn(vxge_mBIT((msix_id >> 2)), 0, 32),
2278                         &hldev->common_reg->clr_msix_one_shot_vec[msix_id % 4]);
2279         else
2280                 __vxge_hw_pio_mem_write32_upper(
2281                         (u32) vxge_bVALn(vxge_mBIT((msix_id >> 2)), 0, 32),
2282                         &hldev->common_reg->clear_msix_mask_vect[msix_id % 4]);
2283 }
2284
2285 /**
2286  * vxge_hw_vpath_msix_unmask - Unmask the MSIX Vector.
2287  * @vp: Virtual Path handle.
2288  * @msix_id:  MSI ID
2289  *
2290  * The function unmasks the msix interrupt for the given msix_id
2291  *
2292  * Returns: 0,
2293  * Otherwise, VXGE_HW_ERR_WRONG_IRQ if the msix index is out of range
2294  * status.
2295  * See also:
2296  */
2297 void
2298 vxge_hw_vpath_msix_unmask(struct __vxge_hw_vpath_handle *vp, int msix_id)
2299 {
2300         struct __vxge_hw_device *hldev = vp->vpath->hldev;
2301         __vxge_hw_pio_mem_write32_upper(
2302                         (u32)vxge_bVALn(vxge_mBIT(msix_id >> 2), 0, 32),
2303                         &hldev->common_reg->clear_msix_mask_vect[msix_id%4]);
2304 }
2305
2306 /**
2307  * vxge_hw_vpath_inta_mask_tx_rx - Mask Tx and Rx interrupts.
2308  * @vp: Virtual Path handle.
2309  *
2310  * Mask Tx and Rx vpath interrupts.
2311  *
2312  * See also: vxge_hw_vpath_inta_mask_tx_rx()
2313  */
2314 void vxge_hw_vpath_inta_mask_tx_rx(struct __vxge_hw_vpath_handle *vp)
2315 {
2316         u64     tim_int_mask0[4] = {[0 ...3] = 0};
2317         u32     tim_int_mask1[4] = {[0 ...3] = 0};
2318         u64     val64;
2319         struct __vxge_hw_device *hldev = vp->vpath->hldev;
2320
2321         VXGE_HW_DEVICE_TIM_INT_MASK_SET(tim_int_mask0,
2322                 tim_int_mask1, vp->vpath->vp_id);
2323
2324         val64 = readq(&hldev->common_reg->tim_int_mask0);
2325
2326         if ((tim_int_mask0[VXGE_HW_VPATH_INTR_TX] != 0) ||
2327                 (tim_int_mask0[VXGE_HW_VPATH_INTR_RX] != 0)) {
2328                 writeq((tim_int_mask0[VXGE_HW_VPATH_INTR_TX] |
2329                         tim_int_mask0[VXGE_HW_VPATH_INTR_RX] | val64),
2330                         &hldev->common_reg->tim_int_mask0);
2331         }
2332
2333         val64 = readl(&hldev->common_reg->tim_int_mask1);
2334
2335         if ((tim_int_mask1[VXGE_HW_VPATH_INTR_TX] != 0) ||
2336                 (tim_int_mask1[VXGE_HW_VPATH_INTR_RX] != 0)) {
2337                 __vxge_hw_pio_mem_write32_upper(
2338                         (tim_int_mask1[VXGE_HW_VPATH_INTR_TX] |
2339                         tim_int_mask1[VXGE_HW_VPATH_INTR_RX] | val64),
2340                         &hldev->common_reg->tim_int_mask1);
2341         }
2342 }
2343
2344 /**
2345  * vxge_hw_vpath_inta_unmask_tx_rx - Unmask Tx and Rx interrupts.
2346  * @vp: Virtual Path handle.
2347  *
2348  * Unmask Tx and Rx vpath interrupts.
2349  *
2350  * See also: vxge_hw_vpath_inta_mask_tx_rx()
2351  */
2352 void vxge_hw_vpath_inta_unmask_tx_rx(struct __vxge_hw_vpath_handle *vp)
2353 {
2354         u64     tim_int_mask0[4] = {[0 ...3] = 0};
2355         u32     tim_int_mask1[4] = {[0 ...3] = 0};
2356         u64     val64;
2357         struct __vxge_hw_device *hldev = vp->vpath->hldev;
2358
2359         VXGE_HW_DEVICE_TIM_INT_MASK_SET(tim_int_mask0,
2360                 tim_int_mask1, vp->vpath->vp_id);
2361
2362         val64 = readq(&hldev->common_reg->tim_int_mask0);
2363
2364         if ((tim_int_mask0[VXGE_HW_VPATH_INTR_TX] != 0) ||
2365            (tim_int_mask0[VXGE_HW_VPATH_INTR_RX] != 0)) {
2366                 writeq((~(tim_int_mask0[VXGE_HW_VPATH_INTR_TX] |
2367                         tim_int_mask0[VXGE_HW_VPATH_INTR_RX])) & val64,
2368                         &hldev->common_reg->tim_int_mask0);
2369         }
2370
2371         if ((tim_int_mask1[VXGE_HW_VPATH_INTR_TX] != 0) ||
2372            (tim_int_mask1[VXGE_HW_VPATH_INTR_RX] != 0)) {
2373                 __vxge_hw_pio_mem_write32_upper(
2374                         (~(tim_int_mask1[VXGE_HW_VPATH_INTR_TX] |
2375                           tim_int_mask1[VXGE_HW_VPATH_INTR_RX])) & val64,
2376                         &hldev->common_reg->tim_int_mask1);
2377         }
2378 }
2379
2380 /**
2381  * vxge_hw_vpath_poll_rx - Poll Rx Virtual Path for completed
2382  * descriptors and process the same.
2383  * @ring: Handle to the ring object used for receive
2384  *
2385  * The function polls the Rx for the completed  descriptors and calls
2386  * the driver via supplied completion   callback.
2387  *
2388  * Returns: VXGE_HW_OK, if the polling is completed successful.
2389  * VXGE_HW_COMPLETIONS_REMAIN: There are still more completed
2390  * descriptors available which are yet to be processed.
2391  *
2392  * See also: vxge_hw_vpath_poll_rx()
2393  */
2394 enum vxge_hw_status vxge_hw_vpath_poll_rx(struct __vxge_hw_ring *ring)
2395 {
2396         u8 t_code;
2397         enum vxge_hw_status status = VXGE_HW_OK;
2398         void *first_rxdh;
2399         int new_count = 0;
2400
2401         ring->cmpl_cnt = 0;
2402
2403         status = vxge_hw_ring_rxd_next_completed(ring, &first_rxdh, &t_code);
2404         if (status == VXGE_HW_OK)
2405                 ring->callback(ring, first_rxdh,
2406                         t_code, ring->channel.userdata);
2407
2408         if (ring->cmpl_cnt != 0) {
2409                 ring->doorbell_cnt += ring->cmpl_cnt;
2410                 if (ring->doorbell_cnt >= ring->rxds_limit) {
2411                         /*
2412                          * Each RxD is of 4 qwords, update the number of
2413                          * qwords replenished
2414                          */
2415                         new_count = (ring->doorbell_cnt * 4);
2416
2417                         /* For each block add 4 more qwords */
2418                         ring->total_db_cnt += ring->doorbell_cnt;
2419                         if (ring->total_db_cnt >= ring->rxds_per_block) {
2420                                 new_count += 4;
2421                                 /* Reset total count */
2422                                 ring->total_db_cnt %= ring->rxds_per_block;
2423                         }
2424                         writeq(VXGE_HW_PRC_RXD_DOORBELL_NEW_QW_CNT(new_count),
2425                                 &ring->vp_reg->prc_rxd_doorbell);
2426                         readl(&ring->common_reg->titan_general_int_status);
2427                         ring->doorbell_cnt = 0;
2428                 }
2429         }
2430
2431         return status;
2432 }
2433
2434 /**
2435  * vxge_hw_vpath_poll_tx - Poll Tx for completed descriptors and process
2436  * the same.
2437  * @fifo: Handle to the fifo object used for non offload send
2438  *
2439  * The function polls the Tx for the completed descriptors and calls
2440  * the driver via supplied completion callback.
2441  *
2442  * Returns: VXGE_HW_OK, if the polling is completed successful.
2443  * VXGE_HW_COMPLETIONS_REMAIN: There are still more completed
2444  * descriptors available which are yet to be processed.
2445  */
2446 enum vxge_hw_status vxge_hw_vpath_poll_tx(struct __vxge_hw_fifo *fifo,
2447                                         struct sk_buff ***skb_ptr, int nr_skb,
2448                                         int *more)
2449 {
2450         enum vxge_hw_fifo_tcode t_code;
2451         void *first_txdlh;
2452         enum vxge_hw_status status = VXGE_HW_OK;
2453         struct __vxge_hw_channel *channel;
2454
2455         channel = &fifo->channel;
2456
2457         status = vxge_hw_fifo_txdl_next_completed(fifo,
2458                                 &first_txdlh, &t_code);
2459         if (status == VXGE_HW_OK)
2460                 if (fifo->callback(fifo, first_txdlh, t_code,
2461                         channel->userdata, skb_ptr, nr_skb, more) != VXGE_HW_OK)
2462                         status = VXGE_HW_COMPLETIONS_REMAIN;
2463
2464         return status;
2465 }