drivers/soc/ti/knav_qmss_acc.c

   1 /*
   2  * Keystone accumulator queue manager
   3  *
   4  * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
   5  * Author:      Sandeep Nair <sandeep_n@ti.com>
   6  *              Cyril Chemparathy <cyril@ti.com>
   7  *              Santosh Shilimkar <santosh.shilimkar@ti.com>
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * version 2 as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/device.h>
  22 #include <linux/io.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/bitops.h>
  25 #include <linux/slab.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/soc/ti/knav_qmss.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/dma-mapping.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_address.h>
  33 #include <linux/firmware.h>
  34
  35 #include "knav_qmss.h"
  36
  37 #define knav_range_offset_to_inst(kdev, range, q)       \
  38         (range->queue_base_inst + (q << kdev->inst_shift))
  39
  40 static void __knav_acc_notify(struct knav_range_info *range,
  41                                 struct knav_acc_channel *acc)
  42 {
  43         struct knav_device *kdev = range->kdev;
  44         struct knav_queue_inst *inst;
  45         int range_base, queue;
  46
  47         range_base = kdev->base_id + range->queue_base;
  48
  49         if (range->flags & RANGE_MULTI_QUEUE) {
  50                 for (queue = 0; queue < range->num_queues; queue++) {
  51                         inst = knav_range_offset_to_inst(kdev, range,
  52                                                                 queue);
  53                         if (inst->notify_needed) {
  54                                 inst->notify_needed = 0;
  55                                 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  56                                         range_base + queue);
  57                                 knav_queue_notify(inst);
  58                         }
  59                 }
  60         } else {
  61                 queue = acc->channel - range->acc_info.start_channel;
  62                 inst = knav_range_offset_to_inst(kdev, range, queue);
  63                 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  64                         range_base + queue);
  65                 knav_queue_notify(inst);
  66         }
  67 }
  68
  69 static int knav_acc_set_notify(struct knav_range_info *range,
  70                                 struct knav_queue_inst *kq,
  71                                 bool enabled)
  72 {
  73         struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
  74         struct knav_device *kdev = range->kdev;
  75         u32 mask, offset;
  76
  77         /*
  78          * when enabling, we need to re-trigger an interrupt if we
  79          * have descriptors pending
  80          */
  81         if (!enabled || atomic_read(&kq->desc_count) <= 0)
  82                 return 0;
  83
  84         kq->notify_needed = 1;
  85         atomic_inc(&kq->acc->retrigger_count);
  86         mask = BIT(kq->acc->channel % 32);
  87         offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
  88         dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
  89                 kq->acc->name);
  90         writel_relaxed(mask, pdsp->intd + offset);
  91         return 0;
  92 }
  93
  94 static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
  95 {
  96         struct knav_acc_channel *acc;
  97         struct knav_queue_inst *kq = NULL;
  98         struct knav_range_info *range;
  99         struct knav_pdsp_info *pdsp;
 100         struct knav_acc_info *info;
 101         struct knav_device *kdev;
 102
 103         u32 *list, *list_cpu, val, idx, notifies;
 104         int range_base, channel, queue = 0;
 105         dma_addr_t list_dma;
 106
 107         range = _instdata;
 108         info  = &range->acc_info;
 109         kdev  = range->kdev;
 110         pdsp  = range->acc_info.pdsp;
 111         acc   = range->acc;
 112
 113         range_base = kdev->base_id + range->queue_base;
 114         if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
 115                 for (queue = 0; queue < range->num_irqs; queue++)
 116                         if (range->irqs[queue].irq == irq)
 117                                 break;
 118                 kq = knav_range_offset_to_inst(kdev, range, queue);
 119                 acc += queue;
 120         }
 121
 122         channel = acc->channel;
 123         list_dma = acc->list_dma[acc->list_index];
 124         list_cpu = acc->list_cpu[acc->list_index];
 125         dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n",
 126                 channel, acc->list_index, list_cpu, &list_dma);
 127         if (atomic_read(&acc->retrigger_count)) {
 128                 atomic_dec(&acc->retrigger_count);
 129                 __knav_acc_notify(range, acc);
 130                 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 131                 /* ack the interrupt */
 132                 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
 133                                pdsp->intd + ACC_INTD_OFFSET_EOI);
 134
 135                 return IRQ_HANDLED;
 136         }
 137
 138         notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 139         WARN_ON(!notifies);
 140         dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
 141                                 DMA_FROM_DEVICE);
 142
 143         for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
 144              list += ACC_LIST_ENTRY_WORDS) {
 145                 if (ACC_LIST_ENTRY_WORDS == 1) {
 146                         dev_dbg(kdev->dev,
 147                                 "acc-irq: list %d, entry @%p, %08x\n",
 148                                 acc->list_index, list, list[0]);
 149                 } else if (ACC_LIST_ENTRY_WORDS == 2) {
 150                         dev_dbg(kdev->dev,
 151                                 "acc-irq: list %d, entry @%p, %08x %08x\n",
 152                                 acc->list_index, list, list[0], list[1]);
 153                 } else if (ACC_LIST_ENTRY_WORDS == 4) {
 154                         dev_dbg(kdev->dev,
 155                                 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
 156                                 acc->list_index, list, list[0], list[1],
 157                                 list[2], list[3]);
 158                 }
 159
 160                 val = list[ACC_LIST_ENTRY_DESC_IDX];
 161                 if (!val)
 162                         break;
 163
 164                 if (range->flags & RANGE_MULTI_QUEUE) {
 165                         queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
 166                         if (queue < range_base ||
 167                             queue >= range_base + range->num_queues) {
 168                                 dev_err(kdev->dev,
 169                                         "bad queue %d, expecting %d-%d\n",
 170                                         queue, range_base,
 171                                         range_base + range->num_queues);
 172                                 break;
 173                         }
 174                         queue -= range_base;
 175                         kq = knav_range_offset_to_inst(kdev, range,
 176                                                                 queue);
 177                 }
 178
 179                 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
 180                         atomic_dec(&kq->desc_count);
 181                         dev_err(kdev->dev,
 182                                 "acc-irq: queue %d full, entry dropped\n",
 183                                 queue + range_base);
 184                         continue;
 185                 }
 186
 187                 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
 188                 kq->descs[idx] = val;
 189                 kq->notify_needed = 1;
 190                 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
 191                         val, idx, queue + range_base);
 192         }
 193
 194         __knav_acc_notify(range, acc);
 195         memset(list_cpu, 0, info->list_size);
 196         dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
 197                                    DMA_TO_DEVICE);
 198
 199         /* flip to the other list */
 200         acc->list_index ^= 1;
 201
 202         /* reset the interrupt counter */
 203         writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 204
 205         /* ack the interrupt */
 206         writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
 207                        pdsp->intd + ACC_INTD_OFFSET_EOI);
 208
 209         return IRQ_HANDLED;
 210 }
 211
 212 static int knav_range_setup_acc_irq(struct knav_range_info *range,
 213                                 int queue, bool enabled)
 214 {
 215         struct knav_device *kdev = range->kdev;
 216         struct knav_acc_channel *acc;
 217         unsigned long cpu_map;
 218         int ret = 0, irq;
 219         u32 old, new;
 220
 221         if (range->flags & RANGE_MULTI_QUEUE) {
 222                 acc = range->acc;
 223                 irq = range->irqs[0].irq;
 224                 cpu_map = range->irqs[0].cpu_map;
 225         } else {
 226                 acc = range->acc + queue;
 227                 irq = range->irqs[queue].irq;
 228                 cpu_map = range->irqs[queue].cpu_map;
 229         }
 230
 231         old = acc->open_mask;
 232         if (enabled)
 233                 new = old | BIT(queue);
 234         else
 235                 new = old & ~BIT(queue);
 236         acc->open_mask = new;
 237
 238         dev_dbg(kdev->dev,
 239                 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
 240                 old, new, acc->name);
 241
 242         if (likely(new == old))
 243                 return 0;
 244
 245         if (new && !old) {
 246                 dev_dbg(kdev->dev,
 247                         "setup-acc-irq: requesting %s for channel %s\n",
 248                         acc->name, acc->name);
 249                 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
 250                                   range);
 251                 if (!ret && cpu_map) {
 252                         ret = irq_set_affinity_hint(irq, to_cpumask(&cpu_map));
 253                         if (ret) {
 254                                 dev_warn(range->kdev->dev,
 255                                          "Failed to set IRQ affinity\n");
 256                                 return ret;
 257                         }
 258                 }
 259         }
 260
 261         if (old && !new) {
 262                 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
 263                         acc->name, acc->name);
 264                 ret = irq_set_affinity_hint(irq, NULL);
 265                 if (ret)
 266                         dev_warn(range->kdev->dev,
 267                                  "Failed to set IRQ affinity\n");
 268                 free_irq(irq, range);
 269         }
 270
 271         return ret;
 272 }
 273
 274 static const char *knav_acc_result_str(enum knav_acc_result result)
 275 {
 276         static const char * const result_str[] = {
 277                 [ACC_RET_IDLE]                  = "idle",
 278                 [ACC_RET_SUCCESS]               = "success",
 279                 [ACC_RET_INVALID_COMMAND]       = "invalid command",
 280                 [ACC_RET_INVALID_CHANNEL]       = "invalid channel",
 281                 [ACC_RET_INACTIVE_CHANNEL]      = "inactive channel",
 282                 [ACC_RET_ACTIVE_CHANNEL]        = "active channel",
 283                 [ACC_RET_INVALID_QUEUE]         = "invalid queue",
 284                 [ACC_RET_INVALID_RET]           = "invalid return code",
 285         };
 286
 287         if (result >= ARRAY_SIZE(result_str))
 288                 return result_str[ACC_RET_INVALID_RET];
 289         else
 290                 return result_str[result];
 291 }
 292
 293 static enum knav_acc_result
 294 knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
 295                 struct knav_reg_acc_command *cmd)
 296 {
 297         u32 result;
 298
 299         dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
 300                 cmd->command, cmd->queue_mask, cmd->list_dma,
 301                 cmd->queue_num, cmd->timer_config);
 302
 303         writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
 304         writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
 305         writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma);
 306         writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
 307         writel_relaxed(cmd->command, &pdsp->acc_command->command);
 308
 309         /* wait for the command to clear */
 310         do {
 311                 result = readl_relaxed(&pdsp->acc_command->command);
 312         } while ((result >> 8) & 0xff);
 313
 314         return (result >> 24) & 0xff;
 315 }
 316
 317 static void knav_acc_setup_cmd(struct knav_device *kdev,
 318                                 struct knav_range_info *range,
 319                                 struct knav_reg_acc_command *cmd,
 320                                 int queue)
 321 {
 322         struct knav_acc_info *info = &range->acc_info;
 323         struct knav_acc_channel *acc;
 324         int queue_base;
 325         u32 queue_mask;
 326
 327         if (range->flags & RANGE_MULTI_QUEUE) {
 328                 acc = range->acc;
 329                 queue_base = range->queue_base;
 330                 queue_mask = BIT(range->num_queues) - 1;
 331         } else {
 332                 acc = range->acc + queue;
 333                 queue_base = range->queue_base + queue;
 334                 queue_mask = 0;
 335         }
 336
 337         memset(cmd, 0, sizeof(*cmd));
 338         cmd->command    = acc->channel;
 339         cmd->queue_mask = queue_mask;
 340         cmd->list_dma   = (u32)acc->list_dma[0];
 341         cmd->queue_num  = info->list_entries << 16;
 342         cmd->queue_num |= queue_base;
 343
 344         cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
 345         if (range->flags & RANGE_MULTI_QUEUE)
 346                 cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
 347         cmd->timer_config |= info->pacing_mode << 16;
 348         cmd->timer_config |= info->timer_count;
 349 }
 350
 351 static void knav_acc_stop(struct knav_device *kdev,
 352                                 struct knav_range_info *range,
 353                                 int queue)
 354 {
 355         struct knav_reg_acc_command cmd;
 356         struct knav_acc_channel *acc;
 357         enum knav_acc_result result;
 358
 359         acc = range->acc + queue;
 360
 361         knav_acc_setup_cmd(kdev, range, &cmd, queue);
 362         cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
 363         result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
 364
 365         dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
 366                 acc->name, knav_acc_result_str(result));
 367 }
 368
 369 static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
 370                                                 struct knav_range_info *range,
 371                                                 int queue)
 372 {
 373         struct knav_reg_acc_command cmd;
 374         struct knav_acc_channel *acc;
 375         enum knav_acc_result result;
 376
 377         acc = range->acc + queue;
 378
 379         knav_acc_setup_cmd(kdev, range, &cmd, queue);
 380         cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
 381         result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
 382
 383         dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
 384                 acc->name, knav_acc_result_str(result));
 385
 386         return result;
 387 }
 388
 389 static int knav_acc_init_range(struct knav_range_info *range)
 390 {
 391         struct knav_device *kdev = range->kdev;
 392         struct knav_acc_channel *acc;
 393         enum knav_acc_result result;
 394         int queue;
 395
 396         for (queue = 0; queue < range->num_queues; queue++) {
 397                 acc = range->acc + queue;
 398
 399                 knav_acc_stop(kdev, range, queue);
 400                 acc->list_index = 0;
 401                 result = knav_acc_start(kdev, range, queue);
 402
 403                 if (result != ACC_RET_SUCCESS)
 404                         return -EIO;
 405
 406                 if (range->flags & RANGE_MULTI_QUEUE)
 407                         return 0;
 408         }
 409         return 0;
 410 }
 411
 412 static int knav_acc_init_queue(struct knav_range_info *range,
 413                                 struct knav_queue_inst *kq)
 414 {
 415         unsigned id = kq->id - range->queue_base;
 416
 417         kq->descs = devm_kzalloc(range->kdev->dev,
 418                                  ACC_DESCS_MAX * sizeof(u32), GFP_KERNEL);
 419         if (!kq->descs)
 420                 return -ENOMEM;
 421
 422         kq->acc = range->acc;
 423         if ((range->flags & RANGE_MULTI_QUEUE) == 0)
 424                 kq->acc += id;
 425         return 0;
 426 }
 427
 428 static int knav_acc_open_queue(struct knav_range_info *range,
 429                                 struct knav_queue_inst *inst, unsigned flags)
 430 {
 431         unsigned id = inst->id - range->queue_base;
 432
 433         return knav_range_setup_acc_irq(range, id, true);
 434 }
 435
 436 static int knav_acc_close_queue(struct knav_range_info *range,
 437                                         struct knav_queue_inst *inst)
 438 {
 439         unsigned id = inst->id - range->queue_base;
 440
 441         return knav_range_setup_acc_irq(range, id, false);
 442 }
 443
 444 static int knav_acc_free_range(struct knav_range_info *range)
 445 {
 446         struct knav_device *kdev = range->kdev;
 447         struct knav_acc_channel *acc;
 448         struct knav_acc_info *info;
 449         int channel, channels;
 450
 451         info = &range->acc_info;
 452
 453         if (range->flags & RANGE_MULTI_QUEUE)
 454                 channels = 1;
 455         else
 456                 channels = range->num_queues;
 457
 458         for (channel = 0; channel < channels; channel++) {
 459                 acc = range->acc + channel;
 460                 if (!acc->list_cpu[0])
 461                         continue;
 462                 dma_unmap_single(kdev->dev, acc->list_dma[0],
 463                                  info->mem_size, DMA_BIDIRECTIONAL);
 464                 free_pages_exact(acc->list_cpu[0], info->mem_size);
 465         }
 466         devm_kfree(range->kdev->dev, range->acc);
 467         return 0;
 468 }
 469
 470 struct knav_range_ops knav_acc_range_ops = {
 471         .set_notify     = knav_acc_set_notify,
 472         .init_queue     = knav_acc_init_queue,
 473         .open_queue     = knav_acc_open_queue,
 474         .close_queue    = knav_acc_close_queue,
 475         .init_range     = knav_acc_init_range,
 476         .free_range     = knav_acc_free_range,
 477 };
 478
 479 /**
 480  * knav_init_acc_range: Initialise accumulator ranges
 481  *
 482  * @kdev:               qmss device
 483  * @node:               device node
 484  * @range:              qmms range information
 485  *
 486  * Return 0 on success or error
 487  */
 488 int knav_init_acc_range(struct knav_device *kdev,
 489                         struct device_node *node,
 490                         struct knav_range_info *range)
 491 {
 492         struct knav_acc_channel *acc;
 493         struct knav_pdsp_info *pdsp;
 494         struct knav_acc_info *info;
 495         int ret, channel, channels;
 496         int list_size, mem_size;
 497         dma_addr_t list_dma;
 498         void *list_mem;
 499         u32 config[5];
 500
 501         range->flags |= RANGE_HAS_ACCUMULATOR;
 502         info = &range->acc_info;
 503
 504         ret = of_property_read_u32_array(node, "accumulator", config, 5);
 505         if (ret)
 506                 return ret;
 507
 508         info->pdsp_id           = config[0];
 509         info->start_channel     = config[1];
 510         info->list_entries      = config[2];
 511         info->pacing_mode       = config[3];
 512         info->timer_count       = config[4] / ACC_DEFAULT_PERIOD;
 513
 514         if (info->start_channel > ACC_MAX_CHANNEL) {
 515                 dev_err(kdev->dev, "channel %d invalid for range %s\n",
 516                         info->start_channel, range->name);
 517                 return -EINVAL;
 518         }
 519
 520         if (info->pacing_mode > 3) {
 521                 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
 522                         info->pacing_mode, range->name);
 523                 return -EINVAL;
 524         }
 525
 526         pdsp = knav_find_pdsp(kdev, info->pdsp_id);
 527         if (!pdsp) {
 528                 dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
 529                         info->pdsp_id, range->name);
 530                 return -EINVAL;
 531         }
 532
 533         if (!pdsp->started) {
 534                 dev_err(kdev->dev, "pdsp id %d not started for range %s\n",
 535                         info->pdsp_id, range->name);
 536                 return -ENODEV;
 537         }
 538
 539         info->pdsp = pdsp;
 540         channels = range->num_queues;
 541         if (of_get_property(node, "multi-queue", NULL)) {
 542                 range->flags |= RANGE_MULTI_QUEUE;
 543                 channels = 1;
 544                 if (range->queue_base & (32 - 1)) {
 545                         dev_err(kdev->dev,
 546                                 "misaligned multi-queue accumulator range %s\n",
 547                                 range->name);
 548                         return -EINVAL;
 549                 }
 550                 if (range->num_queues > 32) {
 551                         dev_err(kdev->dev,
 552                                 "too many queues in accumulator range %s\n",
 553                                 range->name);
 554                         return -EINVAL;
 555                 }
 556         }
 557
 558         /* figure out list size */
 559         list_size  = info->list_entries;
 560         list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
 561         info->list_size = list_size;
 562         mem_size   = PAGE_ALIGN(list_size * 2);
 563         info->mem_size  = mem_size;
 564         range->acc = devm_kzalloc(kdev->dev, channels * sizeof(*range->acc),
 565                                   GFP_KERNEL);
 566         if (!range->acc)
 567                 return -ENOMEM;
 568
 569         for (channel = 0; channel < channels; channel++) {
 570                 acc = range->acc + channel;
 571                 acc->channel = info->start_channel + channel;
 572
 573                 /* allocate memory for the two lists */
 574                 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
 575                 if (!list_mem)
 576                         return -ENOMEM;
 577
 578                 list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
 579                                           DMA_BIDIRECTIONAL);
 580                 if (dma_mapping_error(kdev->dev, list_dma)) {
 581                         free_pages_exact(list_mem, mem_size);
 582                         return -ENOMEM;
 583                 }
 584
 585                 memset(list_mem, 0, mem_size);
 586                 dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
 587                                            DMA_TO_DEVICE);
 588                 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
 589                           acc->channel);
 590                 acc->list_cpu[0] = list_mem;
 591                 acc->list_cpu[1] = list_mem + list_size;
 592                 acc->list_dma[0] = list_dma;
 593                 acc->list_dma[1] = list_dma + list_size;
 594                 dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n",
 595                         acc->name, acc->channel, &list_dma, list_mem);
 596         }
 597
 598         range->ops = &knav_acc_range_ops;
 599         return 0;
 600 }
 601 EXPORT_SYMBOL_GPL(knav_init_acc_range);