drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c

   1 /**
   2  * Copyright (c) 2010-2012 Broadcom. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions, and the following disclaimer,
   9  *    without modification.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. The names of the above-listed copyright holders may not be used
  14  *    to endorse or promote products derived from this software without
  15  *    specific prior written permission.
  16  *
  17  * ALTERNATIVELY, this software may be distributed under the terms of the
  18  * GNU General Public License ("GPL") version 2, as published by the Free
  19  * Software Foundation.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  22  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  23  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  26  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  27  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  28  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  29  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  30  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  31  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <linux/kernel.h>
  35 #include <linux/types.h>
  36 #include <linux/errno.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/pagemap.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/version.h>
  41 #include <linux/io.h>
  42 #include <linux/platform_device.h>
  43 #include <linux/uaccess.h>
  44 #include <linux/of.h>
  45 #include <asm/pgtable.h>
  46 #include <soc/bcm2835/raspberrypi-firmware.h>
  47
  48 #define dmac_map_area                   __glue(_CACHE,_dma_map_area)
  49 #define dmac_unmap_area                 __glue(_CACHE,_dma_unmap_area)
  50
  51 extern void dmac_map_area(const void *, size_t, int);
  52 extern void dmac_unmap_area(const void *, size_t, int);
  53
  54 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
  55
  56 #define VCHIQ_ARM_ADDRESS(x) ((void *)((char *)x + g_virt_to_bus_offset))
  57
  58 #include "vchiq_arm.h"
  59 #include "vchiq_2835.h"
  60 #include "vchiq_connected.h"
  61 #include "vchiq_killable.h"
  62
  63 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
  64
  65 #define BELL0   0x00
  66 #define BELL2   0x08
  67
  68 typedef struct vchiq_2835_state_struct {
  69    int inited;
  70    VCHIQ_ARM_STATE_T arm_state;
  71 } VCHIQ_2835_ARM_STATE_T;
  72
  73 static void __iomem *g_regs;
  74 static unsigned int g_cache_line_size = sizeof(CACHE_LINE_SIZE);
  75 static unsigned int g_fragments_size;
  76 static char *g_fragments_base;
  77 static char *g_free_fragments;
  78 static struct semaphore g_free_fragments_sema;
  79 static unsigned long g_virt_to_bus_offset;
  80
  81 extern int vchiq_arm_log_level;
  82
  83 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
  84
  85 static irqreturn_t
  86 vchiq_doorbell_irq(int irq, void *dev_id);
  87
  88 static int
  89 create_pagelist(char __user *buf, size_t count, unsigned short type,
  90                 struct task_struct *task, PAGELIST_T ** ppagelist);
  91
  92 static void
  93 free_pagelist(PAGELIST_T *pagelist, int actual);
  94
  95 int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state)
  96 {
  97         struct device *dev = &pdev->dev;
  98         struct rpi_firmware *fw = platform_get_drvdata(pdev);
  99         VCHIQ_SLOT_ZERO_T *vchiq_slot_zero;
 100         struct resource *res;
 101         void *slot_mem;
 102         dma_addr_t slot_phys;
 103         u32 channelbase;
 104         int slot_mem_size, frag_mem_size;
 105         int err, irq, i;
 106
 107         g_virt_to_bus_offset = virt_to_dma(dev, (void *)0);
 108
 109         err = of_property_read_u32(dev->of_node, "cache-line-size",
 110                                    &g_cache_line_size);
 111
 112         if (err) {
 113                 dev_err(dev, "Missing cache-line-size property\n");
 114                 return -ENODEV;
 115         }
 116
 117         g_fragments_size = 2 * g_cache_line_size;
 118
 119         /* Allocate space for the channels in coherent memory */
 120         slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
 121         frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
 122
 123         slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
 124                                        &slot_phys, GFP_KERNEL);
 125         if (!slot_mem) {
 126                 dev_err(dev, "could not allocate DMA memory\n");
 127                 return -ENOMEM;
 128         }
 129
 130         WARN_ON(((int)slot_mem & (PAGE_SIZE - 1)) != 0);
 131
 132         vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
 133         if (!vchiq_slot_zero)
 134                 return -EINVAL;
 135
 136         vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
 137                 (int)slot_phys + slot_mem_size;
 138         vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
 139                 MAX_FRAGMENTS;
 140
 141         g_fragments_base = (char *)slot_mem + slot_mem_size;
 142         slot_mem_size += frag_mem_size;
 143
 144         g_free_fragments = g_fragments_base;
 145         for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
 146                 *(char **)&g_fragments_base[i*g_fragments_size] =
 147                         &g_fragments_base[(i + 1)*g_fragments_size];
 148         }
 149         *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
 150         sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
 151
 152         if (vchiq_init_state(state, vchiq_slot_zero, 0) != VCHIQ_SUCCESS)
 153                 return -EINVAL;
 154
 155         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 156         g_regs = devm_ioremap_resource(&pdev->dev, res);
 157         if (IS_ERR(g_regs))
 158                 return PTR_ERR(g_regs);
 159
 160         irq = platform_get_irq(pdev, 0);
 161         if (irq <= 0) {
 162                 dev_err(dev, "failed to get IRQ\n");
 163                 return irq;
 164         }
 165
 166         err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
 167                                "VCHIQ doorbell", state);
 168         if (err) {
 169                 dev_err(dev, "failed to register irq=%d\n", irq);
 170                 return err;
 171         }
 172
 173         /* Send the base address of the slots to VideoCore */
 174         channelbase = slot_phys;
 175         err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
 176                                     &channelbase, sizeof(channelbase));
 177         if (err || channelbase) {
 178                 dev_err(dev, "failed to set channelbase\n");
 179                 return err ? : -ENXIO;
 180         }
 181
 182         vchiq_log_info(vchiq_arm_log_level,
 183                 "vchiq_init - done (slots %x, phys %pad)",
 184                 (unsigned int)vchiq_slot_zero, &slot_phys);
 185
 186         vchiq_call_connected_callbacks();
 187
 188    return 0;
 189 }
 190
 191 VCHIQ_STATUS_T
 192 vchiq_platform_init_state(VCHIQ_STATE_T *state)
 193 {
 194    VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
 195    state->platform_state = kzalloc(sizeof(VCHIQ_2835_ARM_STATE_T), GFP_KERNEL);
 196    ((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited = 1;
 197    status = vchiq_arm_init_state(state, &((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->arm_state);
 198    if(status != VCHIQ_SUCCESS)
 199    {
 200       ((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited = 0;
 201    }
 202    return status;
 203 }
 204
 205 VCHIQ_ARM_STATE_T*
 206 vchiq_platform_get_arm_state(VCHIQ_STATE_T *state)
 207 {
 208    if(!((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited)
 209    {
 210       BUG();
 211    }
 212    return &((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->arm_state;
 213 }
 214
 215 void
 216 remote_event_signal(REMOTE_EVENT_T *event)
 217 {
 218         wmb();
 219
 220         event->fired = 1;
 221
 222         dsb();         /* data barrier operation */
 223
 224         if (event->armed)
 225                 writel(0, g_regs + BELL2); /* trigger vc interrupt */
 226 }
 227
 228 int
 229 vchiq_copy_from_user(void *dst, const void *src, int size)
 230 {
 231         if ((uint32_t)src < TASK_SIZE) {
 232                 return copy_from_user(dst, src, size);
 233         } else {
 234                 memcpy(dst, src, size);
 235                 return 0;
 236         }
 237 }
 238
 239 VCHIQ_STATUS_T
 240 vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle,
 241         void *offset, int size, int dir)
 242 {
 243         PAGELIST_T *pagelist;
 244         int ret;
 245
 246         WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID);
 247
 248         ret = create_pagelist((char __user *)offset, size,
 249                         (dir == VCHIQ_BULK_RECEIVE)
 250                         ? PAGELIST_READ
 251                         : PAGELIST_WRITE,
 252                         current,
 253                         &pagelist);
 254         if (ret != 0)
 255                 return VCHIQ_ERROR;
 256
 257         bulk->handle = memhandle;
 258         bulk->data = VCHIQ_ARM_ADDRESS(pagelist);
 259
 260         /* Store the pagelist address in remote_data, which isn't used by the
 261            slave. */
 262         bulk->remote_data = pagelist;
 263
 264         return VCHIQ_SUCCESS;
 265 }
 266
 267 void
 268 vchiq_complete_bulk(VCHIQ_BULK_T *bulk)
 269 {
 270         if (bulk && bulk->remote_data && bulk->actual)
 271                 free_pagelist((PAGELIST_T *)bulk->remote_data, bulk->actual);
 272 }
 273
 274 void
 275 vchiq_transfer_bulk(VCHIQ_BULK_T *bulk)
 276 {
 277         /*
 278          * This should only be called on the master (VideoCore) side, but
 279          * provide an implementation to avoid the need for ifdefery.
 280          */
 281         BUG();
 282 }
 283
 284 void
 285 vchiq_dump_platform_state(void *dump_context)
 286 {
 287         char buf[80];
 288         int len;
 289         len = snprintf(buf, sizeof(buf),
 290                 "  Platform: 2835 (VC master)");
 291         vchiq_dump(dump_context, buf, len + 1);
 292 }
 293
 294 VCHIQ_STATUS_T
 295 vchiq_platform_suspend(VCHIQ_STATE_T *state)
 296 {
 297    return VCHIQ_ERROR;
 298 }
 299
 300 VCHIQ_STATUS_T
 301 vchiq_platform_resume(VCHIQ_STATE_T *state)
 302 {
 303    return VCHIQ_SUCCESS;
 304 }
 305
 306 void
 307 vchiq_platform_paused(VCHIQ_STATE_T *state)
 308 {
 309 }
 310
 311 void
 312 vchiq_platform_resumed(VCHIQ_STATE_T *state)
 313 {
 314 }
 315
 316 int
 317 vchiq_platform_videocore_wanted(VCHIQ_STATE_T* state)
 318 {
 319    return 1; // autosuspend not supported - videocore always wanted
 320 }
 321
 322 int
 323 vchiq_platform_use_suspend_timer(void)
 324 {
 325    return 0;
 326 }
 327 void
 328 vchiq_dump_platform_use_state(VCHIQ_STATE_T *state)
 329 {
 330         vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use");
 331 }
 332 void
 333 vchiq_platform_handle_timeout(VCHIQ_STATE_T *state)
 334 {
 335         (void)state;
 336 }
 337 /*
 338  * Local functions
 339  */
 340
 341 static irqreturn_t
 342 vchiq_doorbell_irq(int irq, void *dev_id)
 343 {
 344         VCHIQ_STATE_T *state = dev_id;
 345         irqreturn_t ret = IRQ_NONE;
 346         unsigned int status;
 347
 348         /* Read (and clear) the doorbell */
 349         status = readl(g_regs + BELL0);
 350
 351         if (status & 0x4) {  /* Was the doorbell rung? */
 352                 remote_event_pollall(state);
 353                 ret = IRQ_HANDLED;
 354         }
 355
 356         return ret;
 357 }
 358
 359 /* There is a potential problem with partial cache lines (pages?)
 360 ** at the ends of the block when reading. If the CPU accessed anything in
 361 ** the same line (page?) then it may have pulled old data into the cache,
 362 ** obscuring the new data underneath. We can solve this by transferring the
 363 ** partial cache lines separately, and allowing the ARM to copy into the
 364 ** cached area.
 365
 366 ** N.B. This implementation plays slightly fast and loose with the Linux
 367 ** driver programming rules, e.g. its use of dmac_map_area instead of
 368 ** dma_map_single, but it isn't a multi-platform driver and it benefits
 369 ** from increased speed as a result.
 370 */
 371
 372 static int
 373 create_pagelist(char __user *buf, size_t count, unsigned short type,
 374         struct task_struct *task, PAGELIST_T ** ppagelist)
 375 {
 376         PAGELIST_T *pagelist;
 377         struct page **pages;
 378         unsigned long *addrs;
 379         unsigned int num_pages, offset, i;
 380         char *addr, *base_addr, *next_addr;
 381         int run, addridx, actual_pages;
 382         unsigned long *need_release;
 383
 384         offset = (unsigned int)buf & (PAGE_SIZE - 1);
 385         num_pages = (count + offset + PAGE_SIZE - 1) / PAGE_SIZE;
 386
 387         *ppagelist = NULL;
 388
 389         /* Allocate enough storage to hold the page pointers and the page
 390         ** list
 391         */
 392         pagelist = kmalloc(sizeof(PAGELIST_T) +
 393                            (num_pages * sizeof(unsigned long)) +
 394                            sizeof(unsigned long) +
 395                            (num_pages * sizeof(pages[0])),
 396                            GFP_KERNEL);
 397
 398         vchiq_log_trace(vchiq_arm_log_level,
 399                 "create_pagelist - %x", (unsigned int)pagelist);
 400         if (!pagelist)
 401                 return -ENOMEM;
 402
 403         addrs = pagelist->addrs;
 404         need_release = (unsigned long *)(addrs + num_pages);
 405         pages = (struct page **)(addrs + num_pages + 1);
 406
 407         if (is_vmalloc_addr(buf)) {
 408                 int dir = (type == PAGELIST_WRITE) ?
 409                         DMA_TO_DEVICE : DMA_FROM_DEVICE;
 410                 unsigned long length = count;
 411                 unsigned int off = offset;
 412
 413                 for (actual_pages = 0; actual_pages < num_pages;
 414                      actual_pages++) {
 415                         struct page *pg = vmalloc_to_page(buf + (actual_pages *
 416                                                                  PAGE_SIZE));
 417                         size_t bytes = PAGE_SIZE - off;
 418
 419                         if (bytes > length)
 420                                 bytes = length;
 421                         pages[actual_pages] = pg;
 422                         dmac_map_area(page_address(pg) + off, bytes, dir);
 423                         length -= bytes;
 424                         off = 0;
 425                 }
 426                 *need_release = 0; /* do not try and release vmalloc pages */
 427         } else {
 428                 down_read(&task->mm->mmap_sem);
 429                 actual_pages = get_user_pages(task, task->mm,
 430                                           (unsigned long)buf & ~(PAGE_SIZE - 1),
 431                                           num_pages,
 432                                           (type == PAGELIST_READ) ? FOLL_WRITE : 0,
 433                                           pages,
 434                                           NULL /*vmas */);
 435                 up_read(&task->mm->mmap_sem);
 436
 437                 if (actual_pages != num_pages) {
 438                         vchiq_log_info(vchiq_arm_log_level,
 439                                        "create_pagelist - only %d/%d pages locked",
 440                                        actual_pages,
 441                                        num_pages);
 442
 443                         /* This is probably due to the process being killed */
 444                         while (actual_pages > 0)
 445                         {
 446                                 actual_pages--;
 447                                 page_cache_release(pages[actual_pages]);
 448                         }
 449                         kfree(pagelist);
 450                         if (actual_pages == 0)
 451                                 actual_pages = -ENOMEM;
 452                         return actual_pages;
 453                 }
 454                 *need_release = 1; /* release user pages */
 455         }
 456
 457         pagelist->length = count;
 458         pagelist->type = type;
 459         pagelist->offset = offset;
 460
 461         /* Group the pages into runs of contiguous pages */
 462
 463         base_addr = VCHIQ_ARM_ADDRESS(page_address(pages[0]));
 464         next_addr = base_addr + PAGE_SIZE;
 465         addridx = 0;
 466         run = 0;
 467
 468         for (i = 1; i < num_pages; i++) {
 469                 addr = VCHIQ_ARM_ADDRESS(page_address(pages[i]));
 470                 if ((addr == next_addr) && (run < (PAGE_SIZE - 1))) {
 471                         next_addr += PAGE_SIZE;
 472                         run++;
 473                 } else {
 474                         addrs[addridx] = (unsigned long)base_addr + run;
 475                         addridx++;
 476                         base_addr = addr;
 477                         next_addr = addr + PAGE_SIZE;
 478                         run = 0;
 479                 }
 480         }
 481
 482         addrs[addridx] = (unsigned long)base_addr + run;
 483         addridx++;
 484
 485         /* Partial cache lines (fragments) require special measures */
 486         if ((type == PAGELIST_READ) &&
 487                 ((pagelist->offset & (g_cache_line_size - 1)) ||
 488                 ((pagelist->offset + pagelist->length) &
 489                 (g_cache_line_size - 1)))) {
 490                 char *fragments;
 491
 492                 if (down_interruptible(&g_free_fragments_sema) != 0) {
 493                         kfree(pagelist);
 494                         return -EINTR;
 495                 }
 496
 497                 WARN_ON(g_free_fragments == NULL);
 498
 499                 down(&g_free_fragments_mutex);
 500                 fragments = g_free_fragments;
 501                 WARN_ON(fragments == NULL);
 502                 g_free_fragments = *(char **) g_free_fragments;
 503                 up(&g_free_fragments_mutex);
 504                 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
 505                         (fragments - g_fragments_base) / g_fragments_size;
 506         }
 507
 508         dmac_flush_range(pagelist, addrs + num_pages);
 509
 510         *ppagelist = pagelist;
 511
 512         return 0;
 513 }
 514
 515 static void
 516 free_pagelist(PAGELIST_T *pagelist, int actual)
 517 {
 518         unsigned long *need_release;
 519         struct page **pages;
 520         unsigned int num_pages, i;
 521
 522         vchiq_log_trace(vchiq_arm_log_level,
 523                 "free_pagelist - %x, %d", (unsigned int)pagelist, actual);
 524
 525         num_pages =
 526                 (pagelist->length + pagelist->offset + PAGE_SIZE - 1) /
 527                 PAGE_SIZE;
 528
 529         need_release = (unsigned long *)(pagelist->addrs + num_pages);
 530         pages = (struct page **)(pagelist->addrs + num_pages + 1);
 531
 532         /* Deal with any partial cache lines (fragments) */
 533         if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
 534                 char *fragments = g_fragments_base +
 535                         (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
 536                         g_fragments_size;
 537                 int head_bytes, tail_bytes;
 538                 head_bytes = (g_cache_line_size - pagelist->offset) &
 539                         (g_cache_line_size - 1);
 540                 tail_bytes = (pagelist->offset + actual) &
 541                         (g_cache_line_size - 1);
 542
 543                 if ((actual >= 0) && (head_bytes != 0)) {
 544                         if (head_bytes > actual)
 545                                 head_bytes = actual;
 546
 547                         memcpy((char *)kmap(pages[0]) +
 548                                 pagelist->offset,
 549                                 fragments,
 550                                 head_bytes);
 551                         kunmap(pages[0]);
 552                 }
 553                 if ((actual >= 0) && (head_bytes < actual) &&
 554                         (tail_bytes != 0)) {
 555                         memcpy((char *)kmap(pages[num_pages - 1]) +
 556                                 ((pagelist->offset + actual) &
 557                                 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
 558                                 fragments + g_cache_line_size,
 559                                 tail_bytes);
 560                         kunmap(pages[num_pages - 1]);
 561                 }
 562
 563                 down(&g_free_fragments_mutex);
 564                 *(char **)fragments = g_free_fragments;
 565                 g_free_fragments = fragments;
 566                 up(&g_free_fragments_mutex);
 567                 up(&g_free_fragments_sema);
 568         }
 569
 570         if (*need_release) {
 571                 unsigned int length = pagelist->length;
 572                 unsigned int offset = pagelist->offset;
 573
 574                 for (i = 0; i < num_pages; i++) {
 575                         struct page *pg = pages[i];
 576
 577                         if (pagelist->type != PAGELIST_WRITE) {
 578                                 unsigned int bytes = PAGE_SIZE - offset;
 579
 580                                 if (bytes > length)
 581                                         bytes = length;
 582                                 dmac_unmap_area(page_address(pg) + offset,
 583                                                 bytes, DMA_FROM_DEVICE);
 584                                 length -= bytes;
 585                                 offset = 0;
 586                                 set_page_dirty(pg);
 587                         }
 588                         page_cache_release(pg);
 589                 }
 590         }
 591
 592         kfree(pagelist);
 593 }