2 * Copyright (c) 2010-2012 Broadcom. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions, and the following disclaimer,
9 * without modification.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The names of the above-listed copyright holders may not be used
14 * to endorse or promote products derived from this software without
15 * specific prior written permission.
17 * ALTERNATIVELY, this software may be distributed under the terms of the
18 * GNU General Public License ("GPL") version 2, as published by the Free
19 * Software Foundation.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
22 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
23 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
26 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <linux/kernel.h>
35 #include <linux/types.h>
36 #include <linux/errno.h>
37 #include <linux/interrupt.h>
38 #include <linux/pagemap.h>
39 #include <linux/dma-mapping.h>
40 #include <linux/version.h>
42 #include <linux/platform_device.h>
43 #include <linux/uaccess.h>
45 #include <asm/pgtable.h>
46 #include <soc/bcm2835/raspberrypi-firmware.h>
48 #define dmac_map_area __glue(_CACHE,_dma_map_area)
49 #define dmac_unmap_area __glue(_CACHE,_dma_unmap_area)
51 extern void dmac_map_area(const void *, size_t, int);
52 extern void dmac_unmap_area(const void *, size_t, int);
54 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
56 #define VCHIQ_ARM_ADDRESS(x) ((void *)((char *)x + g_virt_to_bus_offset))
58 #include "vchiq_arm.h"
59 #include "vchiq_2835.h"
60 #include "vchiq_connected.h"
61 #include "vchiq_killable.h"
63 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
68 typedef struct vchiq_2835_state_struct {
70 VCHIQ_ARM_STATE_T arm_state;
71 } VCHIQ_2835_ARM_STATE_T;
73 static void __iomem *g_regs;
74 static unsigned int g_cache_line_size = sizeof(CACHE_LINE_SIZE);
75 static unsigned int g_fragments_size;
76 static char *g_fragments_base;
77 static char *g_free_fragments;
78 static struct semaphore g_free_fragments_sema;
79 static unsigned long g_virt_to_bus_offset;
81 extern int vchiq_arm_log_level;
83 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
86 vchiq_doorbell_irq(int irq, void *dev_id);
89 create_pagelist(char __user *buf, size_t count, unsigned short type,
90 struct task_struct *task, PAGELIST_T ** ppagelist);
93 free_pagelist(PAGELIST_T *pagelist, int actual);
95 int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state)
97 struct device *dev = &pdev->dev;
98 struct rpi_firmware *fw = platform_get_drvdata(pdev);
99 VCHIQ_SLOT_ZERO_T *vchiq_slot_zero;
100 struct resource *res;
102 dma_addr_t slot_phys;
104 int slot_mem_size, frag_mem_size;
107 g_virt_to_bus_offset = virt_to_dma(dev, (void *)0);
109 err = of_property_read_u32(dev->of_node, "cache-line-size",
113 dev_err(dev, "Missing cache-line-size property\n");
117 g_fragments_size = 2 * g_cache_line_size;
119 /* Allocate space for the channels in coherent memory */
120 slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
121 frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
123 slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
124 &slot_phys, GFP_KERNEL);
126 dev_err(dev, "could not allocate DMA memory\n");
130 WARN_ON(((int)slot_mem & (PAGE_SIZE - 1)) != 0);
132 vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
133 if (!vchiq_slot_zero)
136 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
137 (int)slot_phys + slot_mem_size;
138 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
141 g_fragments_base = (char *)slot_mem + slot_mem_size;
142 slot_mem_size += frag_mem_size;
144 g_free_fragments = g_fragments_base;
145 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
146 *(char **)&g_fragments_base[i*g_fragments_size] =
147 &g_fragments_base[(i + 1)*g_fragments_size];
149 *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
150 sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
152 if (vchiq_init_state(state, vchiq_slot_zero, 0) != VCHIQ_SUCCESS)
155 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
156 g_regs = devm_ioremap_resource(&pdev->dev, res);
158 return PTR_ERR(g_regs);
160 irq = platform_get_irq(pdev, 0);
162 dev_err(dev, "failed to get IRQ\n");
166 err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
167 "VCHIQ doorbell", state);
169 dev_err(dev, "failed to register irq=%d\n", irq);
173 /* Send the base address of the slots to VideoCore */
174 channelbase = slot_phys;
175 err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
176 &channelbase, sizeof(channelbase));
177 if (err || channelbase) {
178 dev_err(dev, "failed to set channelbase\n");
179 return err ? : -ENXIO;
182 vchiq_log_info(vchiq_arm_log_level,
183 "vchiq_init - done (slots %x, phys %pad)",
184 (unsigned int)vchiq_slot_zero, &slot_phys);
186 vchiq_call_connected_callbacks();
192 vchiq_platform_init_state(VCHIQ_STATE_T *state)
194 VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
195 state->platform_state = kzalloc(sizeof(VCHIQ_2835_ARM_STATE_T), GFP_KERNEL);
196 ((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited = 1;
197 status = vchiq_arm_init_state(state, &((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->arm_state);
198 if(status != VCHIQ_SUCCESS)
200 ((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited = 0;
206 vchiq_platform_get_arm_state(VCHIQ_STATE_T *state)
208 if(!((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->inited)
212 return &((VCHIQ_2835_ARM_STATE_T*)state->platform_state)->arm_state;
216 remote_event_signal(REMOTE_EVENT_T *event)
222 dsb(); /* data barrier operation */
225 writel(0, g_regs + BELL2); /* trigger vc interrupt */
229 vchiq_copy_from_user(void *dst, const void *src, int size)
231 if ((uint32_t)src < TASK_SIZE) {
232 return copy_from_user(dst, src, size);
234 memcpy(dst, src, size);
240 vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle,
241 void *offset, int size, int dir)
243 PAGELIST_T *pagelist;
246 WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID);
248 ret = create_pagelist((char __user *)offset, size,
249 (dir == VCHIQ_BULK_RECEIVE)
257 bulk->handle = memhandle;
258 bulk->data = VCHIQ_ARM_ADDRESS(pagelist);
260 /* Store the pagelist address in remote_data, which isn't used by the
262 bulk->remote_data = pagelist;
264 return VCHIQ_SUCCESS;
268 vchiq_complete_bulk(VCHIQ_BULK_T *bulk)
270 if (bulk && bulk->remote_data && bulk->actual)
271 free_pagelist((PAGELIST_T *)bulk->remote_data, bulk->actual);
275 vchiq_transfer_bulk(VCHIQ_BULK_T *bulk)
278 * This should only be called on the master (VideoCore) side, but
279 * provide an implementation to avoid the need for ifdefery.
285 vchiq_dump_platform_state(void *dump_context)
289 len = snprintf(buf, sizeof(buf),
290 " Platform: 2835 (VC master)");
291 vchiq_dump(dump_context, buf, len + 1);
295 vchiq_platform_suspend(VCHIQ_STATE_T *state)
301 vchiq_platform_resume(VCHIQ_STATE_T *state)
303 return VCHIQ_SUCCESS;
307 vchiq_platform_paused(VCHIQ_STATE_T *state)
312 vchiq_platform_resumed(VCHIQ_STATE_T *state)
317 vchiq_platform_videocore_wanted(VCHIQ_STATE_T* state)
319 return 1; // autosuspend not supported - videocore always wanted
323 vchiq_platform_use_suspend_timer(void)
328 vchiq_dump_platform_use_state(VCHIQ_STATE_T *state)
330 vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use");
333 vchiq_platform_handle_timeout(VCHIQ_STATE_T *state)
342 vchiq_doorbell_irq(int irq, void *dev_id)
344 VCHIQ_STATE_T *state = dev_id;
345 irqreturn_t ret = IRQ_NONE;
348 /* Read (and clear) the doorbell */
349 status = readl(g_regs + BELL0);
351 if (status & 0x4) { /* Was the doorbell rung? */
352 remote_event_pollall(state);
359 /* There is a potential problem with partial cache lines (pages?)
360 ** at the ends of the block when reading. If the CPU accessed anything in
361 ** the same line (page?) then it may have pulled old data into the cache,
362 ** obscuring the new data underneath. We can solve this by transferring the
363 ** partial cache lines separately, and allowing the ARM to copy into the
366 ** N.B. This implementation plays slightly fast and loose with the Linux
367 ** driver programming rules, e.g. its use of dmac_map_area instead of
368 ** dma_map_single, but it isn't a multi-platform driver and it benefits
369 ** from increased speed as a result.
373 create_pagelist(char __user *buf, size_t count, unsigned short type,
374 struct task_struct *task, PAGELIST_T ** ppagelist)
376 PAGELIST_T *pagelist;
378 unsigned long *addrs;
379 unsigned int num_pages, offset, i;
380 char *addr, *base_addr, *next_addr;
381 int run, addridx, actual_pages;
382 unsigned long *need_release;
384 offset = (unsigned int)buf & (PAGE_SIZE - 1);
385 num_pages = (count + offset + PAGE_SIZE - 1) / PAGE_SIZE;
389 /* Allocate enough storage to hold the page pointers and the page
392 pagelist = kmalloc(sizeof(PAGELIST_T) +
393 (num_pages * sizeof(unsigned long)) +
394 sizeof(unsigned long) +
395 (num_pages * sizeof(pages[0])),
398 vchiq_log_trace(vchiq_arm_log_level,
399 "create_pagelist - %x", (unsigned int)pagelist);
403 addrs = pagelist->addrs;
404 need_release = (unsigned long *)(addrs + num_pages);
405 pages = (struct page **)(addrs + num_pages + 1);
407 if (is_vmalloc_addr(buf)) {
408 int dir = (type == PAGELIST_WRITE) ?
409 DMA_TO_DEVICE : DMA_FROM_DEVICE;
410 unsigned long length = count;
411 unsigned int off = offset;
413 for (actual_pages = 0; actual_pages < num_pages;
415 struct page *pg = vmalloc_to_page(buf + (actual_pages *
417 size_t bytes = PAGE_SIZE - off;
421 pages[actual_pages] = pg;
422 dmac_map_area(page_address(pg) + off, bytes, dir);
426 *need_release = 0; /* do not try and release vmalloc pages */
428 down_read(&task->mm->mmap_sem);
429 actual_pages = get_user_pages(task, task->mm,
430 (unsigned long)buf & ~(PAGE_SIZE - 1),
432 (type == PAGELIST_READ) ? FOLL_WRITE : 0,
435 up_read(&task->mm->mmap_sem);
437 if (actual_pages != num_pages) {
438 vchiq_log_info(vchiq_arm_log_level,
439 "create_pagelist - only %d/%d pages locked",
443 /* This is probably due to the process being killed */
444 while (actual_pages > 0)
447 page_cache_release(pages[actual_pages]);
450 if (actual_pages == 0)
451 actual_pages = -ENOMEM;
454 *need_release = 1; /* release user pages */
457 pagelist->length = count;
458 pagelist->type = type;
459 pagelist->offset = offset;
461 /* Group the pages into runs of contiguous pages */
463 base_addr = VCHIQ_ARM_ADDRESS(page_address(pages[0]));
464 next_addr = base_addr + PAGE_SIZE;
468 for (i = 1; i < num_pages; i++) {
469 addr = VCHIQ_ARM_ADDRESS(page_address(pages[i]));
470 if ((addr == next_addr) && (run < (PAGE_SIZE - 1))) {
471 next_addr += PAGE_SIZE;
474 addrs[addridx] = (unsigned long)base_addr + run;
477 next_addr = addr + PAGE_SIZE;
482 addrs[addridx] = (unsigned long)base_addr + run;
485 /* Partial cache lines (fragments) require special measures */
486 if ((type == PAGELIST_READ) &&
487 ((pagelist->offset & (g_cache_line_size - 1)) ||
488 ((pagelist->offset + pagelist->length) &
489 (g_cache_line_size - 1)))) {
492 if (down_interruptible(&g_free_fragments_sema) != 0) {
497 WARN_ON(g_free_fragments == NULL);
499 down(&g_free_fragments_mutex);
500 fragments = g_free_fragments;
501 WARN_ON(fragments == NULL);
502 g_free_fragments = *(char **) g_free_fragments;
503 up(&g_free_fragments_mutex);
504 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
505 (fragments - g_fragments_base) / g_fragments_size;
508 dmac_flush_range(pagelist, addrs + num_pages);
510 *ppagelist = pagelist;
516 free_pagelist(PAGELIST_T *pagelist, int actual)
518 unsigned long *need_release;
520 unsigned int num_pages, i;
522 vchiq_log_trace(vchiq_arm_log_level,
523 "free_pagelist - %x, %d", (unsigned int)pagelist, actual);
526 (pagelist->length + pagelist->offset + PAGE_SIZE - 1) /
529 need_release = (unsigned long *)(pagelist->addrs + num_pages);
530 pages = (struct page **)(pagelist->addrs + num_pages + 1);
532 /* Deal with any partial cache lines (fragments) */
533 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
534 char *fragments = g_fragments_base +
535 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
537 int head_bytes, tail_bytes;
538 head_bytes = (g_cache_line_size - pagelist->offset) &
539 (g_cache_line_size - 1);
540 tail_bytes = (pagelist->offset + actual) &
541 (g_cache_line_size - 1);
543 if ((actual >= 0) && (head_bytes != 0)) {
544 if (head_bytes > actual)
547 memcpy((char *)kmap(pages[0]) +
553 if ((actual >= 0) && (head_bytes < actual) &&
555 memcpy((char *)kmap(pages[num_pages - 1]) +
556 ((pagelist->offset + actual) &
557 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
558 fragments + g_cache_line_size,
560 kunmap(pages[num_pages - 1]);
563 down(&g_free_fragments_mutex);
564 *(char **)fragments = g_free_fragments;
565 g_free_fragments = fragments;
566 up(&g_free_fragments_mutex);
567 up(&g_free_fragments_sema);
571 unsigned int length = pagelist->length;
572 unsigned int offset = pagelist->offset;
574 for (i = 0; i < num_pages; i++) {
575 struct page *pg = pages[i];
577 if (pagelist->type != PAGELIST_WRITE) {
578 unsigned int bytes = PAGE_SIZE - offset;
582 dmac_unmap_area(page_address(pg) + offset,
583 bytes, DMA_FROM_DEVICE);
588 page_cache_release(pg);