drivers/net/ethernet/cavium/liquidio/request_manager.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2015 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more
  17  * details.
  18  *
  19  * This file may also be available under a different license from Cavium.
  20  * Contact Cavium, Inc. for more information
  21  **********************************************************************/
  22 #include <linux/pci.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/vmalloc.h>
  25 #include "liquidio_common.h"
  26 #include "octeon_droq.h"
  27 #include "octeon_iq.h"
  28 #include "response_manager.h"
  29 #include "octeon_device.h"
  30 #include "octeon_main.h"
  31 #include "octeon_network.h"
  32 #include "cn66xx_device.h"
  33 #include "cn23xx_pf_device.h"
  34
  35 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count)  \
  36         (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
  37
  38 struct iq_post_status {
  39         int status;
  40         int index;
  41 };
  42
  43 static void check_db_timeout(struct work_struct *work);
  44 static void  __check_db_timeout(struct octeon_device *oct, u64 iq_no);
  45
  46 static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *);
  47
  48 static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no)
  49 {
  50         struct octeon_instr_queue *iq =
  51             (struct octeon_instr_queue *)oct->instr_queue[iq_no];
  52         return iq->iqcmd_64B;
  53 }
  54
  55 #define IQ_INSTR_MODE_32B(oct, iq_no)  (!IQ_INSTR_MODE_64B(oct, iq_no))
  56
  57 /* Define this to return the request status comaptible to old code */
  58 /*#define OCTEON_USE_OLD_REQ_STATUS*/
  59
  60 /* Return 0 on success, 1 on failure */
  61 int octeon_init_instr_queue(struct octeon_device *oct,
  62                             union oct_txpciq txpciq,
  63                             u32 num_descs)
  64 {
  65         struct octeon_instr_queue *iq;
  66         struct octeon_iq_config *conf = NULL;
  67         u32 iq_no = (u32)txpciq.s.q_no;
  68         u32 q_size;
  69         struct cavium_wq *db_wq;
  70         int orig_node = dev_to_node(&oct->pci_dev->dev);
  71         int numa_node = cpu_to_node(iq_no % num_online_cpus());
  72
  73         if (OCTEON_CN6XXX(oct))
  74                 conf = &(CFG_GET_IQ_CFG(CHIP_FIELD(oct, cn6xxx, conf)));
  75         else if (OCTEON_CN23XX_PF(oct))
  76                 conf = &(CFG_GET_IQ_CFG(CHIP_FIELD(oct, cn23xx_pf, conf)));
  77         if (!conf) {
  78                 dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n",
  79                         oct->chip_id);
  80                 return 1;
  81         }
  82
  83         if (num_descs & (num_descs - 1)) {
  84                 dev_err(&oct->pci_dev->dev,
  85                         "Number of descriptors for instr queue %d not in power of 2.\n",
  86                         iq_no);
  87                 return 1;
  88         }
  89
  90         q_size = (u32)conf->instr_type * num_descs;
  91
  92         iq = oct->instr_queue[iq_no];
  93
  94         iq->oct_dev = oct;
  95
  96         set_dev_node(&oct->pci_dev->dev, numa_node);
  97         iq->base_addr = lio_dma_alloc(oct, q_size,
  98                                       (dma_addr_t *)&iq->base_addr_dma);
  99         set_dev_node(&oct->pci_dev->dev, orig_node);
 100         if (!iq->base_addr)
 101                 iq->base_addr = lio_dma_alloc(oct, q_size,
 102                                               (dma_addr_t *)&iq->base_addr_dma);
 103         if (!iq->base_addr) {
 104                 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n",
 105                         iq_no);
 106                 return 1;
 107         }
 108
 109         iq->max_count = num_descs;
 110
 111         /* Initialize a list to holds requests that have been posted to Octeon
 112          * but has yet to be fetched by octeon
 113          */
 114         iq->request_list = vmalloc_node((sizeof(*iq->request_list) * num_descs),
 115                                                numa_node);
 116         if (!iq->request_list)
 117                 iq->request_list = vmalloc(sizeof(*iq->request_list) *
 118                                                   num_descs);
 119         if (!iq->request_list) {
 120                 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
 121                 dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n",
 122                         iq_no);
 123                 return 1;
 124         }
 125
 126         memset(iq->request_list, 0, sizeof(*iq->request_list) * num_descs);
 127
 128         dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %llx count: %d\n",
 129                 iq_no, iq->base_addr, iq->base_addr_dma, iq->max_count);
 130
 131         iq->txpciq.u64 = txpciq.u64;
 132         iq->fill_threshold = (u32)conf->db_min;
 133         iq->fill_cnt = 0;
 134         iq->host_write_index = 0;
 135         iq->octeon_read_index = 0;
 136         iq->flush_index = 0;
 137         iq->last_db_time = 0;
 138         iq->do_auto_flush = 1;
 139         iq->db_timeout = (u32)conf->db_timeout;
 140         atomic_set(&iq->instr_pending, 0);
 141
 142         /* Initialize the spinlock for this instruction queue */
 143         spin_lock_init(&iq->lock);
 144         spin_lock_init(&iq->post_lock);
 145
 146         spin_lock_init(&iq->iq_flush_running_lock);
 147
 148         oct->io_qmask.iq |= (1ULL << iq_no);
 149
 150         /* Set the 32B/64B mode for each input queue */
 151         oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no);
 152         iq->iqcmd_64B = (conf->instr_type == 64);
 153
 154         oct->fn_list.setup_iq_regs(oct, iq_no);
 155
 156         oct->check_db_wq[iq_no].wq = alloc_workqueue("check_iq_db",
 157                                                      WQ_MEM_RECLAIM,
 158                                                      0);
 159         if (!oct->check_db_wq[iq_no].wq) {
 160                 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma);
 161                 dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n",
 162                         iq_no);
 163                 return 1;
 164         }
 165
 166         db_wq = &oct->check_db_wq[iq_no];
 167
 168         INIT_DELAYED_WORK(&db_wq->wk.work, check_db_timeout);
 169         db_wq->wk.ctxptr = oct;
 170         db_wq->wk.ctxul = iq_no;
 171         queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1));
 172
 173         return 0;
 174 }
 175
 176 int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)
 177 {
 178         u64 desc_size = 0, q_size;
 179         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 180
 181         cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work);
 182         destroy_workqueue(oct->check_db_wq[iq_no].wq);
 183
 184         if (OCTEON_CN6XXX(oct))
 185                 desc_size =
 186                     CFG_GET_IQ_INSTR_TYPE(CHIP_FIELD(oct, cn6xxx, conf));
 187         else if (OCTEON_CN23XX_PF(oct))
 188                 desc_size =
 189                     CFG_GET_IQ_INSTR_TYPE(CHIP_FIELD(oct, cn23xx_pf, conf));
 190
 191         vfree(iq->request_list);
 192
 193         if (iq->base_addr) {
 194                 q_size = iq->max_count * desc_size;
 195                 lio_dma_free(oct, (u32)q_size, iq->base_addr,
 196                              iq->base_addr_dma);
 197                 return 0;
 198         }
 199         return 1;
 200 }
 201
 202 /* Return 0 on success, 1 on failure */
 203 int octeon_setup_iq(struct octeon_device *oct,
 204                     int ifidx,
 205                     int q_index,
 206                     union oct_txpciq txpciq,
 207                     u32 num_descs,
 208                     void *app_ctx)
 209 {
 210         u32 iq_no = (u32)txpciq.s.q_no;
 211         int numa_node = cpu_to_node(iq_no % num_online_cpus());
 212
 213         if (oct->instr_queue[iq_no]) {
 214                 dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n",
 215                         iq_no);
 216                 oct->instr_queue[iq_no]->txpciq.u64 = txpciq.u64;
 217                 oct->instr_queue[iq_no]->app_ctx = app_ctx;
 218                 return 0;
 219         }
 220         oct->instr_queue[iq_no] =
 221             vmalloc_node(sizeof(struct octeon_instr_queue), numa_node);
 222         if (!oct->instr_queue[iq_no])
 223                 oct->instr_queue[iq_no] =
 224                     vmalloc(sizeof(struct octeon_instr_queue));
 225         if (!oct->instr_queue[iq_no])
 226                 return 1;
 227
 228         memset(oct->instr_queue[iq_no], 0,
 229                sizeof(struct octeon_instr_queue));
 230
 231         oct->instr_queue[iq_no]->q_index = q_index;
 232         oct->instr_queue[iq_no]->app_ctx = app_ctx;
 233         oct->instr_queue[iq_no]->ifidx = ifidx;
 234
 235         if (octeon_init_instr_queue(oct, txpciq, num_descs)) {
 236                 vfree(oct->instr_queue[iq_no]);
 237                 oct->instr_queue[iq_no] = NULL;
 238                 return 1;
 239         }
 240
 241         oct->num_iqs++;
 242         oct->fn_list.enable_io_queues(oct);
 243         return 0;
 244 }
 245
 246 int lio_wait_for_instr_fetch(struct octeon_device *oct)
 247 {
 248         int i, retry = 1000, pending, instr_cnt = 0;
 249
 250         do {
 251                 instr_cnt = 0;
 252
 253                 /*for (i = 0; i < oct->num_iqs; i++) {*/
 254                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 255                         if (!(oct->io_qmask.iq & (1ULL << i)))
 256                                 continue;
 257                         pending =
 258                             atomic_read(&oct->
 259                                                instr_queue[i]->instr_pending);
 260                         if (pending)
 261                                 __check_db_timeout(oct, i);
 262                         instr_cnt += pending;
 263                 }
 264
 265                 if (instr_cnt == 0)
 266                         break;
 267
 268                 schedule_timeout_uninterruptible(1);
 269
 270         } while (retry-- && instr_cnt);
 271
 272         return instr_cnt;
 273 }
 274
 275 static inline void
 276 ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq)
 277 {
 278         if (atomic_read(&oct->status) == OCT_DEV_RUNNING) {
 279                 writel(iq->fill_cnt, iq->doorbell_reg);
 280                 /* make sure doorbell write goes through */
 281                 mmiowb();
 282                 iq->fill_cnt = 0;
 283                 iq->last_db_time = jiffies;
 284                 return;
 285         }
 286 }
 287
 288 static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq,
 289                                       u8 *cmd)
 290 {
 291         u8 *iqptr, cmdsize;
 292
 293         cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
 294         iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
 295
 296         memcpy(iqptr, cmd, cmdsize);
 297 }
 298
 299 static inline struct iq_post_status
 300 __post_command2(struct octeon_instr_queue *iq, u8 *cmd)
 301 {
 302         struct iq_post_status st;
 303
 304         st.status = IQ_SEND_OK;
 305
 306         /* This ensures that the read index does not wrap around to the same
 307          * position if queue gets full before Octeon could fetch any instr.
 308          */
 309         if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) {
 310                 st.status = IQ_SEND_FAILED;
 311                 st.index = -1;
 312                 return st;
 313         }
 314
 315         if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 2))
 316                 st.status = IQ_SEND_STOP;
 317
 318         __copy_cmd_into_iq(iq, cmd);
 319
 320         /* "index" is returned, host_write_index is modified. */
 321         st.index = iq->host_write_index;
 322         INCR_INDEX_BY1(iq->host_write_index, iq->max_count);
 323         iq->fill_cnt++;
 324
 325         /* Flush the command into memory. We need to be sure the data is in
 326          * memory before indicating that the instruction is pending.
 327          */
 328         wmb();
 329
 330         atomic_inc(&iq->instr_pending);
 331
 332         return st;
 333 }
 334
 335 int
 336 octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype,
 337                                 void (*fn)(void *))
 338 {
 339         if (reqtype > REQTYPE_LAST) {
 340                 dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n",
 341                         __func__, reqtype);
 342                 return -EINVAL;
 343         }
 344
 345         reqtype_free_fn[oct->octeon_id][reqtype] = fn;
 346
 347         return 0;
 348 }
 349
 350 static inline void
 351 __add_to_request_list(struct octeon_instr_queue *iq,
 352                       int idx, void *buf, int reqtype)
 353 {
 354         iq->request_list[idx].buf = buf;
 355         iq->request_list[idx].reqtype = reqtype;
 356 }
 357
 358 /* Can only run in process context */
 359 int
 360 lio_process_iq_request_list(struct octeon_device *oct,
 361                             struct octeon_instr_queue *iq, u32 napi_budget)
 362 {
 363         int reqtype;
 364         void *buf;
 365         u32 old = iq->flush_index;
 366         u32 inst_count = 0;
 367         unsigned int pkts_compl = 0, bytes_compl = 0;
 368         struct octeon_soft_command *sc;
 369         struct octeon_instr_irh *irh;
 370         unsigned long flags;
 371
 372         while (old != iq->octeon_read_index) {
 373                 reqtype = iq->request_list[old].reqtype;
 374                 buf     = iq->request_list[old].buf;
 375
 376                 if (reqtype == REQTYPE_NONE)
 377                         goto skip_this;
 378
 379                 octeon_update_tx_completion_counters(buf, reqtype, &pkts_compl,
 380                                                      &bytes_compl);
 381
 382                 switch (reqtype) {
 383                 case REQTYPE_NORESP_NET:
 384                 case REQTYPE_NORESP_NET_SG:
 385                 case REQTYPE_RESP_NET_SG:
 386                         reqtype_free_fn[oct->octeon_id][reqtype](buf);
 387                         break;
 388                 case REQTYPE_RESP_NET:
 389                 case REQTYPE_SOFT_COMMAND:
 390                         sc = buf;
 391
 392                         if (OCTEON_CN23XX_PF(oct))
 393                                 irh = (struct octeon_instr_irh *)
 394                                         &sc->cmd.cmd3.irh;
 395                         else
 396                                 irh = (struct octeon_instr_irh *)
 397                                         &sc->cmd.cmd2.irh;
 398                         if (irh->rflag) {
 399                                 /* We're expecting a response from Octeon.
 400                                  * It's up to lio_process_ordered_list() to
 401                                  * process  sc. Add sc to the ordered soft
 402                                  * command response list because we expect
 403                                  * a response from Octeon.
 404                                  */
 405                                 spin_lock_irqsave
 406                                         (&oct->response_list
 407                                          [OCTEON_ORDERED_SC_LIST].lock,
 408                                          flags);
 409                                 atomic_inc(&oct->response_list
 410                                         [OCTEON_ORDERED_SC_LIST].
 411                                         pending_req_count);
 412                                 list_add_tail(&sc->node, &oct->response_list
 413                                         [OCTEON_ORDERED_SC_LIST].head);
 414                                 spin_unlock_irqrestore
 415                                         (&oct->response_list
 416                                          [OCTEON_ORDERED_SC_LIST].lock,
 417                                          flags);
 418                         } else {
 419                                 if (sc->callback) {
 420                                         /* This callback must not sleep */
 421                                         sc->callback(oct, OCTEON_REQUEST_DONE,
 422                                                      sc->callback_arg);
 423                                 }
 424                         }
 425                         break;
 426                 default:
 427                         dev_err(&oct->pci_dev->dev,
 428                                 "%s Unknown reqtype: %d buf: %p at idx %d\n",
 429                                 __func__, reqtype, buf, old);
 430                 }
 431
 432                 iq->request_list[old].buf = NULL;
 433                 iq->request_list[old].reqtype = 0;
 434
 435  skip_this:
 436                 inst_count++;
 437                 INCR_INDEX_BY1(old, iq->max_count);
 438
 439                 if ((napi_budget) && (inst_count >= napi_budget))
 440                         break;
 441         }
 442         if (bytes_compl)
 443                 octeon_report_tx_completion_to_bql(iq->app_ctx, pkts_compl,
 444                                                    bytes_compl);
 445         iq->flush_index = old;
 446
 447         return inst_count;
 448 }
 449
 450 /* Can only be called from process context */
 451 int
 452 octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq,
 453                 u32 pending_thresh, u32 napi_budget)
 454 {
 455         u32 inst_processed = 0;
 456         u32 tot_inst_processed = 0;
 457         int tx_done = 1;
 458
 459         if (!spin_trylock(&iq->iq_flush_running_lock))
 460                 return tx_done;
 461
 462         spin_lock_bh(&iq->lock);
 463
 464         iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq);
 465
 466         if (atomic_read(&iq->instr_pending) >= (s32)pending_thresh) {
 467                 do {
 468                         /* Process any outstanding IQ packets. */
 469                         if (iq->flush_index == iq->octeon_read_index)
 470                                 break;
 471
 472                         if (napi_budget)
 473                                 inst_processed = lio_process_iq_request_list
 474                                         (oct, iq,
 475                                          napi_budget - tot_inst_processed);
 476                         else
 477                                 inst_processed =
 478                                         lio_process_iq_request_list(oct, iq, 0);
 479
 480                         if (inst_processed) {
 481                                 atomic_sub(inst_processed, &iq->instr_pending);
 482                                 iq->stats.instr_processed += inst_processed;
 483                         }
 484
 485                         tot_inst_processed += inst_processed;
 486                         inst_processed = 0;
 487
 488                 } while (tot_inst_processed < napi_budget);
 489
 490                 if (napi_budget && (tot_inst_processed >= napi_budget))
 491                         tx_done = 0;
 492         }
 493
 494         iq->last_db_time = jiffies;
 495
 496         spin_unlock_bh(&iq->lock);
 497
 498         spin_unlock(&iq->iq_flush_running_lock);
 499
 500         return tx_done;
 501 }
 502
 503 /* Process instruction queue after timeout.
 504  * This routine gets called from a workqueue or when removing the module.
 505  */
 506 static void __check_db_timeout(struct octeon_device *oct, u64 iq_no)
 507 {
 508         struct octeon_instr_queue *iq;
 509         u64 next_time;
 510
 511         if (!oct)
 512                 return;
 513
 514         iq = oct->instr_queue[iq_no];
 515         if (!iq)
 516                 return;
 517
 518         /* return immediately, if no work pending */
 519         if (!atomic_read(&iq->instr_pending))
 520                 return;
 521         /* If jiffies - last_db_time < db_timeout do nothing  */
 522         next_time = iq->last_db_time + iq->db_timeout;
 523         if (!time_after(jiffies, (unsigned long)next_time))
 524                 return;
 525         iq->last_db_time = jiffies;
 526
 527         /* Flush the instruction queue */
 528         octeon_flush_iq(oct, iq, 1, 0);
 529
 530         lio_enable_irq(NULL, iq);
 531 }
 532
 533 /* Called by the Poll thread at regular intervals to check the instruction
 534  * queue for commands to be posted and for commands that were fetched by Octeon.
 535  */
 536 static void check_db_timeout(struct work_struct *work)
 537 {
 538         struct cavium_wk *wk = (struct cavium_wk *)work;
 539         struct octeon_device *oct = (struct octeon_device *)wk->ctxptr;
 540         u64 iq_no = wk->ctxul;
 541         struct cavium_wq *db_wq = &oct->check_db_wq[iq_no];
 542         u32 delay = 10;
 543
 544         __check_db_timeout(oct, iq_no);
 545         queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(delay));
 546 }
 547
 548 int
 549 octeon_send_command(struct octeon_device *oct, u32 iq_no,
 550                     u32 force_db, void *cmd, void *buf,
 551                     u32 datasize, u32 reqtype)
 552 {
 553         struct iq_post_status st;
 554         struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
 555
 556         /* Get the lock and prevent other tasks and tx interrupt handler from
 557          * running.
 558          */
 559         spin_lock_bh(&iq->post_lock);
 560
 561         st = __post_command2(iq, cmd);
 562
 563         if (st.status != IQ_SEND_FAILED) {
 564                 octeon_report_sent_bytes_to_bql(buf, reqtype);
 565                 __add_to_request_list(iq, st.index, buf, reqtype);
 566                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize);
 567                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1);
 568
 569                 if (force_db)
 570                         ring_doorbell(oct, iq);
 571         } else {
 572                 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1);
 573         }
 574
 575         spin_unlock_bh(&iq->post_lock);
 576
 577         /* This is only done here to expedite packets being flushed
 578          * for cases where there are no IQ completion interrupts.
 579          */
 580         /*if (iq->do_auto_flush)*/
 581         /*      octeon_flush_iq(oct, iq, 2, 0);*/
 582
 583         return st.status;
 584 }
 585
 586 void
 587 octeon_prepare_soft_command(struct octeon_device *oct,
 588                             struct octeon_soft_command *sc,
 589                             u8 opcode,
 590                             u8 subcode,
 591                             u32 irh_ossp,
 592                             u64 ossp0,
 593                             u64 ossp1)
 594 {
 595         struct octeon_config *oct_cfg;
 596         struct octeon_instr_ih2 *ih2;
 597         struct octeon_instr_ih3 *ih3;
 598         struct octeon_instr_pki_ih3 *pki_ih3;
 599         struct octeon_instr_irh *irh;
 600         struct octeon_instr_rdp *rdp;
 601
 602         WARN_ON(opcode > 15);
 603         WARN_ON(subcode > 127);
 604
 605         oct_cfg = octeon_get_conf(oct);
 606
 607         if (OCTEON_CN23XX_PF(oct)) {
 608                 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
 609
 610                 ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind;
 611
 612                 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
 613
 614                 pki_ih3->w           = 1;
 615                 pki_ih3->raw         = 1;
 616                 pki_ih3->utag        = 1;
 617                 pki_ih3->uqpg        =
 618                         oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
 619                 pki_ih3->utt         = 1;
 620                 pki_ih3->tag     = LIO_CONTROL;
 621                 pki_ih3->tagtype = ATOMIC_TAG;
 622                 pki_ih3->qpg         =
 623                         oct->instr_queue[sc->iq_no]->txpciq.s.qpg;
 624                 pki_ih3->pm          = 0x7;
 625                 pki_ih3->sl          = 8;
 626
 627                 if (sc->datasize)
 628                         ih3->dlengsz = sc->datasize;
 629
 630                 irh            = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
 631                 irh->opcode    = opcode;
 632                 irh->subcode   = subcode;
 633
 634                 /* opcode/subcode specific parameters (ossp) */
 635                 irh->ossp       = irh_ossp;
 636                 sc->cmd.cmd3.ossp[0] = ossp0;
 637                 sc->cmd.cmd3.ossp[1] = ossp1;
 638
 639                 if (sc->rdatasize) {
 640                         rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
 641                         rdp->pcie_port = oct->pcie_port;
 642                         rdp->rlen      = sc->rdatasize;
 643
 644                         irh->rflag =  1;
 645                         /*PKI IH3*/
 646                         /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */
 647                         ih3->fsz    = LIO_SOFTCMDRESP_IH3;
 648                 } else {
 649                         irh->rflag =  0;
 650                         /*PKI IH3*/
 651                         /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
 652                         ih3->fsz    = LIO_PCICMD_O3;
 653                 }
 654
 655         } else {
 656                 ih2          = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
 657                 ih2->tagtype = ATOMIC_TAG;
 658                 ih2->tag     = LIO_CONTROL;
 659                 ih2->raw     = 1;
 660                 ih2->grp     = CFG_GET_CTRL_Q_GRP(oct_cfg);
 661
 662                 if (sc->datasize) {
 663                         ih2->dlengsz = sc->datasize;
 664                         ih2->rs = 1;
 665                 }
 666
 667                 irh            = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
 668                 irh->opcode    = opcode;
 669                 irh->subcode   = subcode;
 670
 671                 /* opcode/subcode specific parameters (ossp) */
 672                 irh->ossp       = irh_ossp;
 673                 sc->cmd.cmd2.ossp[0] = ossp0;
 674                 sc->cmd.cmd2.ossp[1] = ossp1;
 675
 676                 if (sc->rdatasize) {
 677                         rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd2.rdp;
 678                         rdp->pcie_port = oct->pcie_port;
 679                         rdp->rlen      = sc->rdatasize;
 680
 681                         irh->rflag =  1;
 682                         /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */
 683                         ih2->fsz   = LIO_SOFTCMDRESP_IH2;
 684                 } else {
 685                         irh->rflag =  0;
 686                         /* irh + ossp[0] + ossp[1] = 24 bytes */
 687                         ih2->fsz   = LIO_PCICMD_O2;
 688                 }
 689         }
 690 }
 691
 692 int octeon_send_soft_command(struct octeon_device *oct,
 693                              struct octeon_soft_command *sc)
 694 {
 695         struct octeon_instr_ih2 *ih2;
 696         struct octeon_instr_ih3 *ih3;
 697         struct octeon_instr_irh *irh;
 698         u32 len;
 699
 700         if (OCTEON_CN23XX_PF(oct)) {
 701                 ih3 =  (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
 702                 if (ih3->dlengsz) {
 703                         WARN_ON(!sc->dmadptr);
 704                         sc->cmd.cmd3.dptr = sc->dmadptr;
 705                 }
 706                 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
 707                 if (irh->rflag) {
 708                         WARN_ON(!sc->dmarptr);
 709                         WARN_ON(!sc->status_word);
 710                         *sc->status_word = COMPLETION_WORD_INIT;
 711                         sc->cmd.cmd3.rptr = sc->dmarptr;
 712                 }
 713                 len = (u32)ih3->dlengsz;
 714         } else {
 715                 ih2 = (struct octeon_instr_ih2 *)&sc->cmd.cmd2.ih2;
 716                 if (ih2->dlengsz) {
 717                         WARN_ON(!sc->dmadptr);
 718                         sc->cmd.cmd2.dptr = sc->dmadptr;
 719                 }
 720                 irh = (struct octeon_instr_irh *)&sc->cmd.cmd2.irh;
 721                 if (irh->rflag) {
 722                         WARN_ON(!sc->dmarptr);
 723                         WARN_ON(!sc->status_word);
 724                         *sc->status_word = COMPLETION_WORD_INIT;
 725                         sc->cmd.cmd2.rptr = sc->dmarptr;
 726                 }
 727                 len = (u32)ih2->dlengsz;
 728         }
 729
 730         if (sc->wait_time)
 731                 sc->timeout = jiffies + sc->wait_time;
 732
 733         return (octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc,
 734                                     len, REQTYPE_SOFT_COMMAND));
 735 }
 736
 737 int octeon_setup_sc_buffer_pool(struct octeon_device *oct)
 738 {
 739         int i;
 740         u64 dma_addr;
 741         struct octeon_soft_command *sc;
 742
 743         INIT_LIST_HEAD(&oct->sc_buf_pool.head);
 744         spin_lock_init(&oct->sc_buf_pool.lock);
 745         atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0);
 746
 747         for (i = 0; i < MAX_SOFT_COMMAND_BUFFERS; i++) {
 748                 sc = (struct octeon_soft_command *)
 749                         lio_dma_alloc(oct,
 750                                       SOFT_COMMAND_BUFFER_SIZE,
 751                                           (dma_addr_t *)&dma_addr);
 752                 if (!sc)
 753                         return 1;
 754
 755                 sc->dma_addr = dma_addr;
 756                 sc->size = SOFT_COMMAND_BUFFER_SIZE;
 757
 758                 list_add_tail(&sc->node, &oct->sc_buf_pool.head);
 759         }
 760
 761         return 0;
 762 }
 763
 764 int octeon_free_sc_buffer_pool(struct octeon_device *oct)
 765 {
 766         struct list_head *tmp, *tmp2;
 767         struct octeon_soft_command *sc;
 768
 769         spin_lock_bh(&oct->sc_buf_pool.lock);
 770
 771         list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) {
 772                 list_del(tmp);
 773
 774                 sc = (struct octeon_soft_command *)tmp;
 775
 776                 lio_dma_free(oct, sc->size, sc, sc->dma_addr);
 777         }
 778
 779         INIT_LIST_HEAD(&oct->sc_buf_pool.head);
 780
 781         spin_unlock_bh(&oct->sc_buf_pool.lock);
 782
 783         return 0;
 784 }
 785
 786 struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct,
 787                                                       u32 datasize,
 788                                                       u32 rdatasize,
 789                                                       u32 ctxsize)
 790 {
 791         u64 dma_addr;
 792         u32 size;
 793         u32 offset = sizeof(struct octeon_soft_command);
 794         struct octeon_soft_command *sc = NULL;
 795         struct list_head *tmp;
 796
 797         WARN_ON((offset + datasize + rdatasize + ctxsize) >
 798                SOFT_COMMAND_BUFFER_SIZE);
 799
 800         spin_lock_bh(&oct->sc_buf_pool.lock);
 801
 802         if (list_empty(&oct->sc_buf_pool.head)) {
 803                 spin_unlock_bh(&oct->sc_buf_pool.lock);
 804                 return NULL;
 805         }
 806
 807         list_for_each(tmp, &oct->sc_buf_pool.head)
 808                 break;
 809
 810         list_del(tmp);
 811
 812         atomic_inc(&oct->sc_buf_pool.alloc_buf_count);
 813
 814         spin_unlock_bh(&oct->sc_buf_pool.lock);
 815
 816         sc = (struct octeon_soft_command *)tmp;
 817
 818         dma_addr = sc->dma_addr;
 819         size = sc->size;
 820
 821         memset(sc, 0, sc->size);
 822
 823         sc->dma_addr = dma_addr;
 824         sc->size = size;
 825
 826         if (ctxsize) {
 827                 sc->ctxptr = (u8 *)sc + offset;
 828                 sc->ctxsize = ctxsize;
 829         }
 830
 831         /* Start data at 128 byte boundary */
 832         offset = (offset + ctxsize + 127) & 0xffffff80;
 833
 834         if (datasize) {
 835                 sc->virtdptr = (u8 *)sc + offset;
 836                 sc->dmadptr = dma_addr + offset;
 837                 sc->datasize = datasize;
 838         }
 839
 840         /* Start rdata at 128 byte boundary */
 841         offset = (offset + datasize + 127) & 0xffffff80;
 842
 843         if (rdatasize) {
 844                 WARN_ON(rdatasize < 16);
 845                 sc->virtrptr = (u8 *)sc + offset;
 846                 sc->dmarptr = dma_addr + offset;
 847                 sc->rdatasize = rdatasize;
 848                 sc->status_word = (u64 *)((u8 *)(sc->virtrptr) + rdatasize - 8);
 849         }
 850
 851         return sc;
 852 }
 853
 854 void octeon_free_soft_command(struct octeon_device *oct,
 855                               struct octeon_soft_command *sc)
 856 {
 857         spin_lock_bh(&oct->sc_buf_pool.lock);
 858
 859         list_add_tail(&sc->node, &oct->sc_buf_pool.head);
 860
 861         atomic_dec(&oct->sc_buf_pool.alloc_buf_count);
 862
 863         spin_unlock_bh(&oct->sc_buf_pool.lock);
 864 }