drivers/net/ethernet/cavium/liquidio/request_manager.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more
  17  * details.
  18  **********************************************************************/
  19 #include <linux/pci.h>
  20 #include <linux/netdevice.h>
  21 #include <linux/vmalloc.h>
  22 #include "liquidio_common.h"
  23 #include "octeon_droq.h"
  24 #include "octeon_iq.h"
  25 #include "response_manager.h"
  26 #include "octeon_device.h"
  27 #include "octeon_main.h"
  28 #include "octeon_network.h"
  29 #include "cn66xx_device.h"
  30 #include "cn23xx_pf_device.h"
  31 #include "cn23xx_vf_device.h"
  32
  33 struct iq_post_status {
  34         int status;
  35         int index;
  36 };
  37
  38 static void check_db_timeout(struct work_struct *work);
  39 static void  __check_db_timeout(struct octeon_device *oct, u64 iq_no);
  40
  41 static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *);
  42
  43 static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no)
  44 {
  45         struct octeon_instr_queue *iq =
  46             (struct octeon_instr_queue *)oct->instr_queue[iq_no];
  47         return iq->iqcmd_64B;
  48 }
  49
  50 #define IQ_INSTR_MODE_32B(oct, iq_no)  (!IQ_INSTR_MODE_64B(oct, iq_no))
  51
  52 /* Define this to return the request status comaptible to old code */
  53 /*#define OCTEON_USE_OLD_REQ_STATUS*/
  54
  55 /* Return 0 on success, 1 on failure */
  56 int octeon_init_instr_queue(struct octeon_device *oct,
  57                             union oct_txpciq txpciq,
  58                             u32 num_descs)
  59 {
  60         struct octeon_instr_queue *iq;
  61         struct octeon_iq_config *conf = NULL;
  62         u32 iq_no = (u32)txpciq.s.q_no;
  63         u32 q_size;
  64         struct cavium_wq *db_wq;
  65         int numa_node = dev_to_node(&oct->pci_dev->dev);
  66
  67         if (OCTEON_CN6XXX(oct))
  68                 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn6xxx)));
  69         else if (OCTEON_CN23XX_PF(oct))
  70                 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_pf)));
  71         else if (OCTEON_CN23XX_VF(oct))
  72                 conf = &(CFG_GET_IQ_CFG(CHIP_CONF(oct, cn23xx_vf)));
  73
  74         if (!conf) {
  75                 dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n",
  76                         oct->chip_id);
  77                 return 1;
  78         }
  79
  80         q_size = (u32)conf->instr_type * num_descs;
  81
  82         iq = oct->instr_queue[iq_no];
  83
  84         iq->oct_dev = oct;
  85
  86         iq->base_addr = lio_dma_alloc(oct, q_size, &iq->base_addr_dma);
  87         if (!iq->base_addr) {
  88                 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n",
  89                         iq_no);
  90                 return 1;
  91         }
  92
  93         iq->max_count = num_descs;
  94
  95         /* Initialize a list to holds requests that have been posted to Octeon
  96          * but has yet to be fetched by octeon
  97          */
  98         iq->request_list = vmalloc_node((sizeof(*iq->request_list) * num_descs),
  99                                                numa_node);
 100         if (!iq->request_list)
 101                 iq->request_list = vmalloc(sizeof(*iq->request_list) *
 102                                                   num_descs);
 103         if (!iq->request_list) {
 104                 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
 105                 dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n",
 106                         iq_no);
 107                 return 1;
 108         }
 109
 110         memset(iq->request_list, 0, sizeof(*iq->request_list) * num_descs);
 111
 112         dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %llx count: %d\n",
 113                 iq_no, iq->base_addr, iq->base_addr_dma, iq->max_count);
 114
 115         iq->txpciq.u64 = txpciq.u64;
 116         iq->fill_threshold = (u32)conf->db_min;
 117         iq->fill_cnt = 0;
 118         iq->host_write_index = 0;
 119         iq->octeon_read_index = 0;
 120         iq->flush_index = 0;
 121         iq->last_db_time = 0;
 122         iq->do_auto_flush = 1;
 123         iq->db_timeout = (u32)conf->db_timeout;
 124         atomic_set(&iq->instr_pending, 0);
 125         iq->pkts_processed = 0;
 126
 127         /* Initialize the spinlock for this instruction queue */
 128         spin_lock_init(&iq->lock);
 129         spin_lock_init(&iq->post_lock);
 130
 131         spin_lock_init(&iq->iq_flush_running_lock);
 132
 133         oct->io_qmask.iq |= BIT_ULL(iq_no);
 134
 135         /* Set the 32B/64B mode for each input queue */
 136         oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);
 137         iq->iqcmd_64B = (conf->instr_type == 64);
 138
 139         oct->fn_list.setup_iq_regs(oct, iq_no);
 140
 141         oct->check_db_wq[iq_no].wq = alloc_workqueue("check_iq_db",
 142                                                      WQ_MEM_RECLAIM,
 143                                                      0);
 144         if (!oct->check_db_wq[iq_no].wq) {
 145                 vfree(iq->request_list);
 146                 iq->request_list = NULL;
 147                 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
 148                 dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n",
 149                         iq_no);
 150                 return 1;
 151         }
 152
 153         db_wq = &oct->check_db_wq[iq_no];
 154
 155         INIT_DELAYED_WORK(&db_wq->wk.work, check_db_timeout);
 156         db_wq->wk.ctxptr = oct;
 157         db_wq->wk.ctxul = iq_no;
 158         queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1));
 159
 160         return 0;
 161 }
 162
 163 int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)
 164 {
 165         u64 desc_size = 0, q_size;
 166         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 167
 168         cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work);
 169         destroy_workqueue(oct->check_db_wq[iq_no].wq);
 170
 171         if (OCTEON_CN6XXX(oct))
 172                 desc_size =
 173                     CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn6xxx));
 174         else if (OCTEON_CN23XX_PF(oct))
 175                 desc_size =
 176                     CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_pf));
 177         else if (OCTEON_CN23XX_VF(oct))
 178                 desc_size =
 179                     CFG_GET_IQ_INSTR_TYPE(CHIP_CONF(oct, cn23xx_vf));
 180
 181         vfree(iq->request_list);
 182
 183         if (iq->base_addr) {
 184                 q_size = iq->max_count * desc_size;
 185                 lio_dma_free(oct, (u32)q_size, iq->base_addr,
 186                              iq->base_addr_dma);
 187                 oct->io_qmask.iq &= ~(1ULL << iq_no);
 188                 vfree(oct->instr_queue[iq_no]);
 189                 oct->instr_queue[iq_no] = NULL;
 190                 oct->num_iqs--;
 191                 return 0;
 192         }
 193         return 1;
 194 }
 195
 196 /* Return 0 on success, 1 on failure */
 197 int octeon_setup_iq(struct octeon_device *oct,
 198                     int ifidx,
 199                     int q_index,
 200                     union oct_txpciq txpciq,
 201                     u32 num_descs,
 202                     void *app_ctx)
 203 {
 204         u32 iq_no = (u32)txpciq.s.q_no;
 205         int numa_node = dev_to_node(&oct->pci_dev->dev);
 206
 207         if (oct->instr_queue[iq_no]) {
 208                 dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n",
 209                         iq_no);
 210                 oct->instr_queue[iq_no]->txpciq.u64 = txpciq.u64;
 211                 oct->instr_queue[iq_no]->app_ctx = app_ctx;
 212                 return 0;
 213         }
 214         oct->instr_queue[iq_no] =
 215             vmalloc_node(sizeof(struct octeon_instr_queue), numa_node);
 216         if (!oct->instr_queue[iq_no])
 217                 oct->instr_queue[iq_no] =
 218                     vmalloc(sizeof(struct octeon_instr_queue));
 219         if (!oct->instr_queue[iq_no])
 220                 return 1;
 221
 222         memset(oct->instr_queue[iq_no], 0,
 223                sizeof(struct octeon_instr_queue));
 224
 225         oct->instr_queue[iq_no]->q_index = q_index;
 226         oct->instr_queue[iq_no]->app_ctx = app_ctx;
 227         oct->instr_queue[iq_no]->ifidx = ifidx;
 228
 229         if (octeon_init_instr_queue(oct, txpciq, num_descs)) {
 230                 vfree(oct->instr_queue[iq_no]);
 231                 oct->instr_queue[iq_no] = NULL;
 232                 return 1;
 233         }
 234
 235         oct->num_iqs++;
 236         if (oct->fn_list.enable_io_queues(oct)) {
 237                 octeon_delete_instr_queue(oct, iq_no);
 238                 return 1;
 239         }
 240
 241         return 0;
 242 }
 243
 244 int lio_wait_for_instr_fetch(struct octeon_device *oct)
 245 {
 246         int i, retry = 1000, pending, instr_cnt = 0;
 247
 248         do {
 249                 instr_cnt = 0;
 250
 251                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 252                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
 253                                 continue;
 254                         pending =
 255                             atomic_read(&oct->instr_queue[i]->instr_pending);
 256                         if (pending)
 257                                 __check_db_timeout(oct, i);
 258                         instr_cnt += pending;
 259                 }
 260
 261                 if (instr_cnt == 0)
 262                         break;
 263
 264                 schedule_timeout_uninterruptible(1);
 265
 266         } while (retry-- && instr_cnt);
 267
 268         return instr_cnt;
 269 }
 270
 271 static inline void
 272 ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq)
 273 {
 274         if (atomic_read(&oct->status) == OCT_DEV_RUNNING) {
 275                 writel(iq->fill_cnt, iq->doorbell_reg);
 276                 /* make sure doorbell write goes through */
 277                 mmiowb();
 278                 iq->fill_cnt = 0;
 279                 iq->last_db_time = jiffies;
 280                 return;
 281         }
 282 }
 283
 284 static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq,
 285                                       u8 *cmd)
 286 {
 287         u8 *iqptr, cmdsize;
 288
 289         cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
 290         iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
 291
 292         memcpy(iqptr, cmd, cmdsize);
 293 }
 294
 295 static inline struct iq_post_status
 296 __post_command2(struct octeon_instr_queue *iq, u8 *cmd)
 297 {
 298         struct iq_post_status st;
 299
 300         st.status = IQ_SEND_OK;
 301
 302         /* This ensures that the read index does not wrap around to the same
 303          * position if queue gets full before Octeon could fetch any instr.
 304          */
 305         if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) {
 306                 st.status = IQ_SEND_FAILED;
 307                 st.index = -1;
 308                 return st;
 309         }
 310
 311         if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 2))
 312                 st.status = IQ_SEND_STOP;
 313
 314         __copy_cmd_into_iq(iq, cmd);
 315
 316         /* "index" is returned, host_write_index is modified. */
 317         st.index = iq->host_write_index;
 318         iq->host_write_index = incr_index(iq->host_write_index, 1,
 319                                           iq->max_count);
 320         iq->fill_cnt++;
 321
 322         /* Flush the command into memory. We need to be sure the data is in
 323          * memory before indicating that the instruction is pending.
 324          */
 325         wmb();
 326
 327         atomic_inc(&iq->instr_pending);
 328
 329         return st;
 330 }
 331
 332 int
 333 octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype,
 334                                 void (*fn)(void *))
 335 {
 336         if (reqtype > REQTYPE_LAST) {
 337                 dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n",
 338                         __func__, reqtype);
 339                 return -EINVAL;
 340         }
 341
 342         reqtype_free_fn[oct->octeon_id][reqtype] = fn;
 343
 344         return 0;
 345 }
 346
 347 static inline void
 348 __add_to_request_list(struct octeon_instr_queue *iq,
 349                       int idx, void *buf, int reqtype)
 350 {
 351         iq->request_list[idx].buf = buf;
 352         iq->request_list[idx].reqtype = reqtype;
 353 }
 354
 355 /* Can only run in process context */
 356 int
 357 lio_process_iq_request_list(struct octeon_device *oct,
 358                             struct octeon_instr_queue *iq, u32 napi_budget)
 359 {
 360         int reqtype;
 361         void *buf;
 362         u32 old = iq->flush_index;
 363         u32 inst_count = 0;
 364         unsigned int pkts_compl = 0, bytes_compl = 0;
 365         struct octeon_soft_command *sc;
 366         struct octeon_instr_irh *irh;
 367         unsigned long flags;
 368
 369         while (old != iq->octeon_read_index) {
 370                 reqtype = iq->request_list[old].reqtype;
 371                 buf     = iq->request_list[old].buf;
 372
 373                 if (reqtype == REQTYPE_NONE)
 374                         goto skip_this;
 375
 376                 octeon_update_tx_completion_counters(buf, reqtype, &pkts_compl,
 377                                                      &bytes_compl);
 378
 379                 switch (reqtype) {
 380                 case REQTYPE_NORESP_NET:
 381                 case REQTYPE_NORESP_NET_SG:
 382                 case REQTYPE_RESP_NET_SG:
 383                         reqtype_free_fn[oct->octeon_id][reqtype](buf);
 384                         break;
 385                 case REQTYPE_RESP_NET:
 386                 case REQTYPE_SOFT_COMMAND:
 387                         sc = buf;
 388
 389                         if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct))
 390                                 irh = (struct octeon_instr_irh *)
 391                                         &sc->cmd.cmd3.irh;
 392                         else
 393                                 irh = (struct octeon_instr_irh *)
 394                                         &sc->cmd.cmd2.irh;
 395                         if (irh->rflag) {
 396                                 /* We're expecting a response from Octeon.
 397                                  * It's up to lio_process_ordered_list() to
 398                                  * process  sc. Add sc to the ordered soft
 399                                  * command response list because we expect
 400                                  * a response from Octeon.
 401                                  */
 402                                 spin_lock_irqsave
 403                                         (&oct->response_list
 404                                          [OCTEON_ORDERED_SC_LIST].lock,
 405                                          flags);
 406                                 atomic_inc(&oct->response_list
 407                                         [OCTEON_ORDERED_SC_LIST].
 408                                         pending_req_count);
 409                                 list_add_tail(&sc->node, &oct->response_list
 410                                         [OCTEON_ORDERED_SC_LIST].head);
 411                                 spin_unlock_irqrestore
 412                                         (&oct->response_list
 413                                          [OCTEON_ORDERED_SC_LIST].lock,
 414                                          flags);
 415                         } else {
 416                                 if (sc->callback) {
 417                                         /* This callback must not sleep */
 418                                         sc->callback(oct, OCTEON_REQUEST_DONE,
 419                                                      sc->callback_arg);
 420                                 }
 421                         }
 422                         break;
 423                 default:
 424                         dev_err(&oct->pci_dev->dev,
 425                                 "%s Unknown reqtype: %d buf: %p at idx %d\n",
 426                                 __func__, reqtype, buf, old);
 427                 }
 428
 429                 iq->request_list[old].buf = NULL;
 430                 iq->request_list[old].reqtype = 0;
 431
 432  skip_this:
 433                 inst_count++;
 434                 old = incr_index(old, 1, iq->max_count);
 435
 436                 if ((napi_budget) && (inst_count >= napi_budget))
 437                         break;
 438         }
 439         if (bytes_compl)
 440                 octeon_report_tx_completion_to_bql(iq->app_ctx, pkts_compl,
 441                                                    bytes_compl);
 442         iq->flush_index = old;
 443
 444         return inst_count;
 445 }
 446
 447 /* Can only be called from process context */
 448 int
 449 octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,
 450                 u32 napi_budget)
 451 {
 452         u32 inst_processed = 0;
 453         u32 tot_inst_processed = 0;
 454         int tx_done = 1;
 455
 456         if (!spin_trylock(&iq->iq_flush_running_lock))
 457                 return tx_done;
 458
 459         spin_lock_bh(&iq->lock);
 460
 461         iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);
 462
 463         do {
 464                 /* Process any outstanding IQ packets. */
 465                 if (iq->flush_index == iq->octeon_read_index)
 466                         break;
 467
 468                 if (napi_budget)
 469                         inst_processed =
 470                                 lio_process_iq_request_list(oct, iq,
 471                                                             napi_budget -
 472                                                             tot_inst_processed);
 473                 else
 474                         inst_processed =
 475                                 lio_process_iq_request_list(oct, iq, 0);
 476
 477                 if (inst_processed) {
 478                         iq->pkts_processed += inst_processed;
 479                         atomic_sub(inst_processed, &iq->instr_pending);
 480                         iq->stats.instr_processed += inst_processed;
 481                 }
 482
 483                 tot_inst_processed += inst_processed;
 484                 inst_processed = 0;
 485
 486         } while (tot_inst_processed < napi_budget);
 487
 488         if (napi_budget && (tot_inst_processed >= napi_budget))
 489                 tx_done = 0;
 490
 491         iq->last_db_time = jiffies;
 492
 493         spin_unlock_bh(&iq->lock);
 494
 495         spin_unlock(&iq->iq_flush_running_lock);
 496
 497         return tx_done;
 498 }
 499
 500 /* Process instruction queue after timeout.
 501  * This routine gets called from a workqueue or when removing the module.
 502  */
 503 static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)
 504 {
 505         struct octeon_instr_queue *iq;
 506         u64 next_time;
 507
 508         if (!oct)
 509                 return;
 510
 511         iq = oct->instr_queue[iq_no];
 512         if (!iq)
 513                 return;
 514
 515         /* return immediately, if no work pending */
 516         if (!atomic_read(&iq->instr_pending))
 517                 return;
 518         /* If jiffies - last_db_time < db_timeout do nothing  */
 519         next_time = iq->last_db_time + iq->db_timeout;
 520         if (!time_after(jiffies, (unsigned long)next_time))
 521                 return;
 522         iq->last_db_time = jiffies;
 523
 524         /* Flush the instruction queue */
 525         octeon_flush_iq(oct, iq, 0);
 526
 527         lio_enable_irq(NULL, iq);
 528 }
 529
 530 /* Called by the Poll thread at regular intervals to check the instruction
 531  * queue for commands to be posted and for commands that were fetched by Octeon.
 532  */
 533 static void check_db_timeout(struct work_struct *work)
 534 {
 535         struct cavium_wk *wk = (struct cavium_wk *)work;
 536         struct octeon_device *oct = (struct octeon_device *)wk->ctxptr;
 537         u64 iq_no = wk->ctxul;
 538         struct cavium_wq *db_wq = &oct->check_db_wq[iq_no];
 539         u32 delay = 10;
 540
 541         __check_db_timeout(oct, iq_no);
 542         queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(delay));
 543 }
 544
 545 int
 546 octeon_send_command(struct octeon_device *oct, u32 iq_no,
 547                     u32 force_db, void *cmd, void *buf,
 548                     u32 datasize, u32 reqtype)
 549 {
 550         struct iq_post_status st;
 551         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 552
 553         /* Get the lock and prevent other tasks and tx interrupt handler from
 554          * running.
 555          */
 556         spin_lock_bh(&iq->post_lock);
 557
 558         st = __post_command2(iq, cmd);
 559
 560         if (st.status != IQ_SEND_FAILED) {
 561                 octeon_report_sent_bytes_to_bql(buf, reqtype);
 562                 __add_to_request_list(iq, st.index, buf, reqtype);
 563                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);
 564                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);
 565
 566                 if (force_db)
 567                         ring_doorbell(oct, iq);
 568         } else {
 569                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);
 570         }
 571
 572         spin_unlock_bh(&iq->post_lock);
 573
 574         /* This is only done here to expedite packets being flushed
 575          * for cases where there are no IQ completion interrupts.
 576          */
 577
 578         return st.status;
 579 }
 580
 581 void
 582 octeon_prepare_soft_command(struct octeon_device *oct,
 583                             struct octeon_soft_command *sc,
 584                             u8 opcode,
 585                             u8 subcode,
 586                             u32 irh_ossp,
 587                             u64 ossp0,
 588                             u64 ossp1)
 589 {
 590         struct octeon_config *oct_cfg;
 591         struct octeon_instr_ih2 *ih2;
 592         struct octeon_instr_ih3 *ih3;
 593         struct octeon_instr_pki_ih3 *pki_ih3;
 594         struct octeon_instr_irh *irh;
 595         struct octeon_instr_rdp *rdp;
 596
 597         WARN_ON(opcode > 15);
 598         WARN_ON(subcode > 127);
 599
 600         oct_cfg = octeon_get_conf(oct);
 601
 602         if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {
 603                 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
 604
 605                 ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;
 606
 607                 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
 608
 609                 pki_ih3->w           = 1;
 610                 pki_ih3->raw         = 1;
 611                 pki_ih3->utag        = 1;
 612                 pki_ih3->uqpg        =
 613                         oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
 614                 pki_ih3->utt         = 1;
 615                 pki_ih3->tag     = LIO_CONTROL;
 616                 pki_ih3->tagtype = ATOMIC_TAG;
 617                 pki_ih3->qpg         =
 618                         oct->instr_queue[sc->iq_no]->txpciq.s.qpg;
 619                 pki_ih3->pm          = 0x7;
 620                 pki_ih3->sl          = 8;
 621
 622                 if (sc->datasize)
 623                         ih3->dlengsz = sc->datasize;
 624
 625                 irh            = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
 626                 irh->opcode    = opcode;
 627                 irh->subcode   = subcode;
 628
 629                 /* opcode/subcode specific parameters (ossp) */
 630                 irh->ossp       = irh_ossp;
 631                 sc->cmd.cmd3.ossp[0] = ossp0;
 632                 sc->cmd.cmd3.ossp[1] = ossp1;
 633
 634                 if (sc->rdatasize) {
 635                         rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
 636                         rdp->pcie_port = oct->pcie_port;
 637                         rdp->rlen      = sc->rdatasize;
 638
 639                         irh->rflag =  1;
 640                         /*PKI IH3*/
 641                         /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */
 642                         ih3->fsz    = LIO_SOFTCMDRESP_IH3;
 643                 } else {
 644                         irh->rflag =  0;
 645                         /*PKI IH3*/
 646                         /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
 647                         ih3->fsz    = LIO_PCICMD_O3;
 648                 }
 649
 650         } else {
 651                 ih2          = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
 652                 ih2->tagtype = ATOMIC_TAG;
 653                 ih2->tag     = LIO_CONTROL;
 654                 ih2->raw     = 1;
 655                 ih2->grp     = CFG_GET_CTRL_Q_GRP(oct_cfg);
 656
 657                 if (sc->datasize) {
 658                         ih2->dlengsz = sc->datasize;
 659                         ih2->rs = 1;
 660                 }
 661
 662                 irh            = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
 663                 irh->opcode    = opcode;
 664                 irh->subcode   = subcode;
 665
 666                 /* opcode/subcode specific parameters (ossp) */
 667                 irh->ossp       = irh_ossp;
 668                 sc->cmd.cmd2.ossp[0] = ossp0;
 669                 sc->cmd.cmd2.ossp[1] = ossp1;
 670
 671                 if (sc->rdatasize) {
 672                         rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
 673                         rdp->pcie_port = oct->pcie_port;
 674                         rdp->rlen      = sc->rdatasize;
 675
 676                         irh->rflag =  1;
 677                         /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
 678                         ih2->fsz   = LIO_SOFTCMDRESP_IH2;
 679                 } else {
 680                         irh->rflag =  0;
 681                         /* irh + ossp[0] + ossp[1] = 24 bytes */
 682                         ih2->fsz   = LIO_PCICMD_O2;
 683                 }
 684         }
 685 }
 686
 687 int octeon_send_soft_command(struct octeon_device *oct,
 688                              struct octeon_soft_command *sc)
 689 {
 690         struct octeon_instr_ih2 *ih2;
 691         struct octeon_instr_ih3 *ih3;
 692         struct octeon_instr_irh *irh;
 693         u32 len;
 694
 695         if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {
 696                 ih3 =  (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
 697                 if (ih3->dlengsz) {
 698                         WARN_ON(!sc->dmadptr);
 699                         sc->cmd.cmd3.dptr = sc->dmadptr;
 700                 }
 701                 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
 702                 if (irh->rflag) {
 703                         WARN_ON(!sc->dmarptr);
 704                         WARN_ON(!sc->status_word);
 705                         *sc->status_word = COMPLETION_WORD_INIT;
 706                         sc->cmd.cmd3.rptr = sc->dmarptr;
 707                 }
 708                 len = (u32)ih3->dlengsz;
 709         } else {
 710                 ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
 711                 if (ih2->dlengsz) {
 712                         WARN_ON(!sc->dmadptr);
 713                         sc->cmd.cmd2.dptr = sc->dmadptr;
 714                 }
 715                 irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
 716                 if (irh->rflag) {
 717                         WARN_ON(!sc->dmarptr);
 718                         WARN_ON(!sc->status_word);
 719                         *sc->status_word = COMPLETION_WORD_INIT;
 720                         sc->cmd.cmd2.rptr = sc->dmarptr;
 721                 }
 722                 len = (u32)ih2->dlengsz;
 723         }
 724
 725         if (sc->wait_time)
 726                 sc->timeout = jiffies + sc->wait_time;
 727
 728         return (octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,
 729                                     len, REQTYPE_SOFT_COMMAND));
 730 }
 731
 732 int octeon_setup_sc_buffer_pool(struct octeon_device *oct)
 733 {
 734         int i;
 735         u64 dma_addr;
 736         struct octeon_soft_command *sc;
 737
 738         INIT_LIST_HEAD(&oct->sc_buf_pool.head);
 739         spin_lock_init(&oct->sc_buf_pool.lock);
 740         atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0);
 741
 742         for (i = 0; i < MAX_SOFT_COMMAND_BUFFERS; i++) {
 743                 sc = (struct octeon_soft_command *)
 744                         lio_dma_alloc(oct,
 745                                       SOFT_COMMAND_BUFFER_SIZE,
 746                                           (dma_addr_t *)&dma_addr);
 747                 if (!sc) {
 748                         octeon_free_sc_buffer_pool(oct);
 749                         return 1;
 750                 }
 751
 752                 sc->dma_addr = dma_addr;
 753                 sc->size = SOFT_COMMAND_BUFFER_SIZE;
 754
 755                 list_add_tail(&sc->node, &oct->sc_buf_pool.head);
 756         }
 757
 758         return 0;
 759 }
 760
 761 int octeon_free_sc_buffer_pool(struct octeon_device *oct)
 762 {
 763         struct list_head *tmp, *tmp2;
 764         struct octeon_soft_command *sc;
 765
 766         spin_lock_bh(&oct->sc_buf_pool.lock);
 767
 768         list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) {
 769                 list_del(tmp);
 770
 771                 sc = (struct octeon_soft_command *)tmp;
 772
 773                 lio_dma_free(oct, sc->size, sc, sc->dma_addr);
 774         }
 775
 776         INIT_LIST_HEAD(&oct->sc_buf_pool.head);
 777
 778         spin_unlock_bh(&oct->sc_buf_pool.lock);
 779
 780         return 0;
 781 }
 782
 783 struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct,
 784                                                       u32 datasize,
 785                                                       u32 rdatasize,
 786                                                       u32 ctxsize)
 787 {
 788         u64 dma_addr;
 789         u32 size;
 790         u32 offset = sizeof(struct octeon_soft_command);
 791         struct octeon_soft_command *sc = NULL;
 792         struct list_head *tmp;
 793
 794         WARN_ON((offset + datasize + rdatasize + ctxsize) >
 795                SOFT_COMMAND_BUFFER_SIZE);
 796
 797         spin_lock_bh(&oct->sc_buf_pool.lock);
 798
 799         if (list_empty(&oct->sc_buf_pool.head)) {
 800                 spin_unlock_bh(&oct->sc_buf_pool.lock);
 801                 return NULL;
 802         }
 803
 804         list_for_each(tmp, &oct->sc_buf_pool.head)
 805                 break;
 806
 807         list_del(tmp);
 808
 809         atomic_inc(&oct->sc_buf_pool.alloc_buf_count);
 810
 811         spin_unlock_bh(&oct->sc_buf_pool.lock);
 812
 813         sc = (struct octeon_soft_command *)tmp;
 814
 815         dma_addr = sc->dma_addr;
 816         size = sc->size;
 817
 818         memset(sc, 0, sc->size);
 819
 820         sc->dma_addr = dma_addr;
 821         sc->size = size;
 822
 823         if (ctxsize) {
 824                 sc->ctxptr = (u8 *)sc + offset;
 825                 sc->ctxsize = ctxsize;
 826         }
 827
 828         /* Start data at 128 byte boundary */
 829         offset = (offset + ctxsize + 127) & 0xffffff80;
 830
 831         if (datasize) {
 832                 sc->virtdptr = (u8 *)sc + offset;
 833                 sc->dmadptr = dma_addr + offset;
 834                 sc->datasize = datasize;
 835         }
 836
 837         /* Start rdata at 128 byte boundary */
 838         offset = (offset + datasize + 127) & 0xffffff80;
 839
 840         if (rdatasize) {
 841                 WARN_ON(rdatasize < 16);
 842                 sc->virtrptr = (u8 *)sc + offset;
 843                 sc->dmarptr = dma_addr + offset;
 844                 sc->rdatasize = rdatasize;
 845                 sc->status_word = (u64 *)((u8 *)(sc->virtrptr) + rdatasize - 8);
 846         }
 847
 848         return sc;
 849 }
 850
 851 void octeon_free_soft_command(struct octeon_device *oct,
 852                               struct octeon_soft_command *sc)
 853 {
 854         spin_lock_bh(&oct->sc_buf_pool.lock);
 855
 856         list_add_tail(&sc->node, &oct->sc_buf_pool.head);
 857
 858         atomic_dec(&oct->sc_buf_pool.alloc_buf_count);
 859
 860         spin_unlock_bh(&oct->sc_buf_pool.lock);
 861 }