1 // SPDX-License-Identifier: GPL-2.0
3 * Implementation of Gasket page table support.
5 * Copyright (C) 2018 Google, Inc.
9 * Implementation of Gasket page table support.
11 * This file assumes 4kB pages throughout; can be factored out when necessary.
13 * Address format is as follows:
14 * Simple addresses - those whose containing pages are directly placed in the
15 * device's address translation registers - are laid out as:
16 * [ 63 - 40: Unused | 39 - 28: 0 | 27 - 12: page index | 11 - 0: page offset ]
17 * page index: The index of the containing page in the device's address
18 * translation registers.
19 * page offset: The index of the address into the containing page.
21 * Extended address - those whose containing pages are contained in a second-
22 * level page table whose address is present in the device's address translation
23 * registers - are laid out as:
24 * [ 63 - 40: Unused | 39: flag | 38 - 37: 0 | 36 - 21: dev/level 0 index |
25 * 20 - 12: host/level 1 index | 11 - 0: page offset ]
26 * flag: Marker indicating that this is an extended address. Always 1.
27 * dev index: The index of the first-level page in the device's extended
28 * address translation registers.
29 * host index: The index of the containing page in the [host-resident] second-
31 * page offset: The index of the address into the containing [second-level]
34 #include "gasket_page_table.h"
36 #include <linux/device.h>
37 #include <linux/file.h>
38 #include <linux/init.h>
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/moduleparam.h>
42 #include <linux/pagemap.h>
43 #include <linux/vmalloc.h>
45 #include "gasket_constants.h"
46 #include "gasket_core.h"
48 /* Constants & utility macros */
49 /* The number of pages that can be mapped into each second-level page table. */
50 #define GASKET_PAGES_PER_SUBTABLE 512
52 /* The starting position of the page index in a simple virtual address. */
53 #define GASKET_SIMPLE_PAGE_SHIFT 12
55 /* Flag indicating that a [device] slot is valid for use. */
56 #define GASKET_VALID_SLOT_FLAG 1
59 * The starting position of the level 0 page index (i.e., the entry in the
60 * device's extended address registers) in an extended address.
61 * Also can be thought of as (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE)),
64 #define GASKET_EXTENDED_LVL0_SHIFT 21
67 * Number of first level pages that Gasket chips support. Equivalent to
68 * log2(NUM_LVL0_PAGE_TABLES)
70 * At a maximum, allowing for a 34 bits address space (or 16GB)
71 * = GASKET_EXTENDED_LVL0_WIDTH + (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE)
74 #define GASKET_EXTENDED_LVL0_WIDTH 13
77 * The starting position of the level 1 page index (i.e., the entry in the
78 * host second-level/sub- table) in an extended address.
80 #define GASKET_EXTENDED_LVL1_SHIFT 12
82 /* Type declarations */
83 /* Valid states for a struct gasket_page_table_entry. */
90 * Mapping metadata for a single page.
92 * In this file, host-side page table entries are referred to as that (or PTEs).
93 * Where device vs. host entries are differentiated, device-side or -visible
94 * entries are called "slots". A slot may be either an entry in the device's
95 * address translation table registers or an entry in a second-level page
98 * The full data in this structure is visible on the host [of course]. Only
99 * the address contained in dma_addr is communicated to the device; that points
100 * to the actual page mapped and described by this structure.
102 struct gasket_page_table_entry {
103 /* The status of this entry/slot: free or in use. */
104 enum pte_status status;
106 /* Address of the page in DMA space. */
109 /* Linux page descriptor for the page described by this structure. */
113 * Index for alignment into host vaddrs.
114 * When a user specifies a host address for a mapping, that address may
115 * not be page-aligned. Offset is the index into the containing page of
116 * the host address (i.e., host_vaddr & (PAGE_SIZE - 1)).
117 * This is necessary for translating between user-specified addresses
118 * and page-aligned addresses.
123 * If this is an extended and first-level entry, sublevel points
124 * to the second-level entries underneath this entry.
126 struct gasket_page_table_entry *sublevel;
130 * Maintains virtual to physical address mapping for a coherent page that is
131 * allocated by this module for a given device.
132 * Note that coherent pages mappings virt mapping cannot be tracked by the
133 * Linux kernel, and coherent pages don't have a struct page associated,
134 * hence Linux kernel cannot perform a get_user_page_xx() on a phys address
135 * that was allocated coherent.
136 * This structure trivially implements this mechanism.
138 struct gasket_coherent_page_entry {
139 /* Phys address, dma'able by the owner device */
142 /* Kernel virtual address */
145 /* User virtual address that was mapped by the mmap kernel subsystem */
149 * Whether this page has been mapped into a user land process virtual
156 * [Host-side] page table descriptor.
158 * This structure tracks the metadata necessary to manage both simple and
159 * extended page tables.
161 struct gasket_page_table {
162 /* The config used to create this page table. */
163 struct gasket_page_table_config config;
165 /* The number of simple (single-level) entries in the page table. */
166 uint num_simple_entries;
168 /* The number of extended (two-level) entries in the page table. */
169 uint num_extended_entries;
171 /* Array of [host-side] page table entries. */
172 struct gasket_page_table_entry *entries;
174 /* Number of actively mapped kernel pages in this table. */
175 uint num_active_pages;
177 /* Device register: base of/first slot in the page table. */
178 u64 __iomem *base_slot;
180 /* Device register: holds the offset indicating the start of the
181 * extended address region of the device's address translation table.
183 u64 __iomem *extended_offset_reg;
185 /* Device structure for the underlying device. Only used for logging. */
186 struct device *device;
188 /* PCI system descriptor for the underlying device. */
189 struct pci_dev *pci_dev;
191 /* Location of the extended address bit for this Gasket device. */
194 /* Mutex to protect page table internals. */
197 /* Number of coherent pages accessible thru by this page table */
198 int num_coherent_pages;
201 * List of coherent memory (physical) allocated for a device.
203 * This structure also remembers the user virtual mapping, this is
204 * hacky, but we need to do this because the kernel doesn't keep track
205 * of the user coherent pages (pfn pages), and virt to coherent page
207 * TODO: use find_vma() APIs to convert host address to vm_area, to
208 * dma_addr_t instead of storing user virtu address in
209 * gasket_coherent_page_entry
211 * Note that the user virtual mapping is created by the driver, in
212 * gasket_mmap function, so user_virt belongs in the driver anyhow.
214 struct gasket_coherent_page_entry *coherent_pages;
217 /* See gasket_page_table.h for description. */
218 int gasket_page_table_init(struct gasket_page_table **ppg_tbl,
219 const struct gasket_bar_data *bar_data,
220 const struct gasket_page_table_config *page_table_config,
221 struct device *device, struct pci_dev *pci_dev)
224 struct gasket_page_table *pg_tbl;
225 ulong total_entries = page_table_config->total_entries;
228 * TODO: Verify config->total_entries against value read from the
229 * hardware register that contains the page table size.
231 if (total_entries == ULONG_MAX) {
232 dev_dbg(device, "Error reading page table size. "
233 "Initializing page table with size 0\n");
238 "Attempting to initialize page table of size 0x%lx\n",
242 "Table has base reg 0x%x, extended offset reg 0x%x\n",
243 page_table_config->base_reg,
244 page_table_config->extended_reg);
246 *ppg_tbl = kzalloc(sizeof(**ppg_tbl), GFP_KERNEL);
248 dev_dbg(device, "No memory for page table\n");
253 bytes = total_entries * sizeof(struct gasket_page_table_entry);
255 pg_tbl->entries = vzalloc(bytes);
256 if (!pg_tbl->entries) {
258 "No memory for address translation metadata\n");
265 mutex_init(&pg_tbl->mutex);
266 memcpy(&pg_tbl->config, page_table_config, sizeof(*page_table_config));
267 if (pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_NORMAL ||
268 pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_SIMPLE) {
269 pg_tbl->num_simple_entries = total_entries;
270 pg_tbl->num_extended_entries = 0;
271 pg_tbl->extended_flag = 1ull << page_table_config->extended_bit;
273 pg_tbl->num_simple_entries = 0;
274 pg_tbl->num_extended_entries = total_entries;
275 pg_tbl->extended_flag = 0;
277 pg_tbl->num_active_pages = 0;
279 (u64 __iomem *)&bar_data->virt_base[page_table_config->base_reg];
280 pg_tbl->extended_offset_reg =
281 (u64 __iomem *)&bar_data->virt_base[page_table_config->extended_reg];
282 pg_tbl->device = get_device(device);
283 pg_tbl->pci_dev = pci_dev;
285 dev_dbg(device, "Page table initialized successfully\n");
291 * Check if a range of PTEs is free.
292 * The page table mutex must be held by the caller.
294 static bool gasket_is_pte_range_free(struct gasket_page_table_entry *ptes,
299 for (i = 0; i < num_entries; i++) {
300 if (ptes[i].status != PTE_FREE)
308 * Free a second level page [sub]table.
309 * The page table mutex must be held before this call.
311 static void gasket_free_extended_subtable(struct gasket_page_table *pg_tbl,
312 struct gasket_page_table_entry *pte,
315 /* Release the page table from the driver */
316 pte->status = PTE_FREE;
318 /* Release the page table from the device */
320 /* Force sync around the address release. */
324 dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE,
327 vfree(pte->sublevel);
330 free_page((ulong)page_address(pte->page));
332 memset(pte, 0, sizeof(struct gasket_page_table_entry));
336 * Actually perform collection.
337 * The page table mutex must be held by the caller.
340 gasket_page_table_garbage_collect_nolock(struct gasket_page_table *pg_tbl)
342 struct gasket_page_table_entry *pte;
345 /* XXX FIX ME XXX -- more efficient to keep a usage count */
346 /* rather than scanning the second level page tables */
348 for (pte = pg_tbl->entries + pg_tbl->num_simple_entries,
349 slot = pg_tbl->base_slot + pg_tbl->num_simple_entries;
350 pte < pg_tbl->entries + pg_tbl->config.total_entries;
352 if (pte->status == PTE_INUSE) {
353 if (gasket_is_pte_range_free(pte->sublevel,
354 GASKET_PAGES_PER_SUBTABLE))
355 gasket_free_extended_subtable(pg_tbl, pte,
361 /* See gasket_page_table.h for description. */
362 void gasket_page_table_garbage_collect(struct gasket_page_table *pg_tbl)
364 mutex_lock(&pg_tbl->mutex);
365 gasket_page_table_garbage_collect_nolock(pg_tbl);
366 mutex_unlock(&pg_tbl->mutex);
369 /* See gasket_page_table.h for description. */
370 void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl)
372 /* Deallocate free second-level tables. */
373 gasket_page_table_garbage_collect(pg_tbl);
375 /* TODO: Check that all PTEs have been freed? */
377 vfree(pg_tbl->entries);
378 pg_tbl->entries = NULL;
380 put_device(pg_tbl->device);
384 /* See gasket_page_table.h for description. */
385 int gasket_page_table_partition(struct gasket_page_table *pg_tbl,
386 uint num_simple_entries)
390 mutex_lock(&pg_tbl->mutex);
391 if (num_simple_entries > pg_tbl->config.total_entries) {
392 mutex_unlock(&pg_tbl->mutex);
396 gasket_page_table_garbage_collect_nolock(pg_tbl);
398 start = min(pg_tbl->num_simple_entries, num_simple_entries);
400 for (i = start; i < pg_tbl->config.total_entries; i++) {
401 if (pg_tbl->entries[i].status != PTE_FREE) {
402 dev_err(pg_tbl->device, "entry %d is not free\n", i);
403 mutex_unlock(&pg_tbl->mutex);
408 pg_tbl->num_simple_entries = num_simple_entries;
409 pg_tbl->num_extended_entries =
410 pg_tbl->config.total_entries - num_simple_entries;
411 writeq(num_simple_entries, pg_tbl->extended_offset_reg);
413 mutex_unlock(&pg_tbl->mutex);
416 EXPORT_SYMBOL(gasket_page_table_partition);
419 * Return whether a host buffer was mapped as coherent memory.
421 * A Gasket page_table currently support one contiguous dma range, mapped to one
422 * contiguous virtual memory range. Check if the host_addr is within that range.
424 static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr)
428 /* whether the host address is within user virt range */
429 if (!pg_tbl->coherent_pages)
432 min = (u64)pg_tbl->coherent_pages[0].user_virt;
433 max = min + PAGE_SIZE * pg_tbl->num_coherent_pages;
435 return min <= host_addr && host_addr < max;
439 * Get and map last level page table buffers.
441 * slots is the location(s) to write device-mapped page address. If this is a
442 * simple mapping, these will be address translation registers. If this is
443 * an extended mapping, these will be within a second-level page table
444 * allocated by the host and so must have their __iomem attribute casted away.
446 static int gasket_perform_mapping(struct gasket_page_table *pg_tbl,
447 struct gasket_page_table_entry *ptes,
448 u64 __iomem *slots, ulong host_addr,
449 uint num_pages, int is_simple_mapping)
458 for (i = 0; i < num_pages; i++) {
459 page_addr = host_addr + i * PAGE_SIZE;
460 offset = page_addr & (PAGE_SIZE - 1);
461 dev_dbg(pg_tbl->device, "%s i %d\n", __func__, i);
462 if (is_coherent(pg_tbl, host_addr)) {
465 (u64)pg_tbl->coherent_pages[0].user_virt;
467 ptes[i].offset = offset;
468 ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr +
471 ret = get_user_pages_fast(page_addr - offset, 1, 1,
475 dev_err(pg_tbl->device,
476 "get user pages failed for addr=0x%lx, "
477 "offset=0x%lx [ret=%d]\n",
478 page_addr, offset, ret);
479 return ret ? ret : -ENOMEM;
481 ++pg_tbl->num_active_pages;
484 ptes[i].offset = offset;
486 /* Map the page into DMA space. */
488 dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE,
490 dev_dbg(pg_tbl->device,
491 "%s i %d pte %p pfn %p -> mapped %llx\n",
492 __func__, i, &ptes[i],
493 (void *)page_to_pfn(page),
494 (unsigned long long)ptes[i].dma_addr);
496 if (ptes[i].dma_addr == -1) {
497 dev_dbg(pg_tbl->device,
498 "%s i %d -> fail to map page %llx "
499 "[pfn %p ohys %p]\n",
501 (unsigned long long)ptes[i].dma_addr,
502 (void *)page_to_pfn(page),
503 (void *)page_to_phys(page));
506 /* Wait until the page is mapped. */
510 /* Make the DMA-space address available to the device. */
511 dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG;
513 if (is_simple_mapping) {
514 writeq(dma_addr, &slots[i]);
516 ((u64 __force *)slots)[i] = dma_addr;
517 /* Extended page table vectors are in DRAM,
518 * and so need to be synced each time they are updated.
520 dma_map_single(pg_tbl->device,
521 (void *)&((u64 __force *)slots)[i],
522 sizeof(u64), DMA_TO_DEVICE);
524 ptes[i].status = PTE_INUSE;
530 * Return the index of the page for the address in the simple table.
531 * Does not perform validity checking.
533 static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl,
536 return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) &
537 (pg_tbl->config.total_entries - 1);
541 * Return the level 0 page index for the given address.
542 * Does not perform validity checking.
544 static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl,
547 return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) &
548 ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1);
552 * Return the level 1 page index for the given address.
553 * Does not perform validity checking.
555 static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl,
558 return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) &
559 (GASKET_PAGES_PER_SUBTABLE - 1);
563 * Allocate page table entries in a simple table.
564 * The page table mutex must be held by the caller.
566 static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl,
567 ulong dev_addr, uint num_pages)
569 if (!gasket_is_pte_range_free(pg_tbl->entries +
570 gasket_simple_page_idx(pg_tbl, dev_addr),
577 /* Safely return a page to the OS. */
578 static bool gasket_release_page(struct page *page)
583 if (!PageReserved(page))
591 * Unmap and release mapped pages.
592 * The page table mutex must be held by the caller.
594 static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl,
595 struct gasket_page_table_entry *ptes,
596 u64 __iomem *slots, uint num_pages,
597 int is_simple_mapping)
601 * For each page table entry and corresponding entry in the device's
602 * address translation table:
604 for (i = 0; i < num_pages; i++) {
605 /* release the address from the device, */
606 if (is_simple_mapping || ptes[i].status == PTE_INUSE)
607 writeq(0, &slots[i]);
609 ((u64 __force *)slots)[i] = 0;
610 /* Force sync around the address release. */
613 /* release the address from the driver, */
614 if (ptes[i].status == PTE_INUSE) {
615 if (ptes[i].dma_addr) {
616 dma_unmap_page(pg_tbl->device, ptes[i].dma_addr,
617 PAGE_SIZE, DMA_FROM_DEVICE);
619 if (gasket_release_page(ptes[i].page))
620 --pg_tbl->num_active_pages;
622 ptes[i].status = PTE_FREE;
624 /* and clear the PTE. */
625 memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry));
630 * Unmap and release pages mapped to simple addresses.
631 * The page table mutex must be held by the caller.
633 static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl,
634 ulong dev_addr, uint num_pages)
636 uint slot = gasket_simple_page_idx(pg_tbl, dev_addr);
638 gasket_perform_unmapping(pg_tbl, pg_tbl->entries + slot,
639 pg_tbl->base_slot + slot, num_pages, 1);
643 * Unmap and release buffers to extended addresses.
644 * The page table mutex must be held by the caller.
646 static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl,
647 ulong dev_addr, uint num_pages)
649 uint slot_idx, remain, len;
650 struct gasket_page_table_entry *pte;
651 u64 __iomem *slot_base;
654 slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
655 pte = pg_tbl->entries + pg_tbl->num_simple_entries +
656 gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
659 /* TODO: Add check to ensure pte remains valid? */
660 len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx);
662 if (pte->status == PTE_INUSE) {
663 slot_base = (u64 __iomem *)(page_address(pte->page) +
665 gasket_perform_unmapping(pg_tbl,
666 pte->sublevel + slot_idx,
667 slot_base + slot_idx, len, 0);
676 /* Evaluates to nonzero if the specified virtual address is simple. */
677 static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl,
680 return !((addr) & (pg_tbl)->extended_flag);
684 * Convert (simple, page, offset) into a device address.
686 * Simple page 0, offset 32:
687 * Input (0, 0, 32), Output 0x20
688 * Simple page 1000, offset 511:
689 * Input (0, 1000, 512), Output 0x3E81FF
690 * Extended page 0, offset 32:
691 * Input (0, 0, 32), Output 0x8000000020
692 * Extended page 1000, offset 511:
693 * Input (1, 1000, 512), Output 0x8003E81FF
695 static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl,
696 int is_simple, uint page_index,
699 ulong lvl0_index, lvl1_index;
702 /* Return simple addresses directly. */
703 lvl0_index = page_index & (pg_tbl->config.total_entries - 1);
704 return (lvl0_index << GASKET_SIMPLE_PAGE_SHIFT) | offset;
708 * This could be compressed into fewer statements, but
709 * A) the compiler should optimize it
710 * B) this is not slow
711 * C) this is an uncommon operation
712 * D) this is actually readable this way.
714 lvl0_index = page_index / GASKET_PAGES_PER_SUBTABLE;
715 lvl1_index = page_index & (GASKET_PAGES_PER_SUBTABLE - 1);
716 return (pg_tbl)->extended_flag |
717 (lvl0_index << GASKET_EXTENDED_LVL0_SHIFT) |
718 (lvl1_index << GASKET_EXTENDED_LVL1_SHIFT) | offset;
722 * Validity checking for simple addresses.
724 * Verify that address translation commutes (from address to/from page + offset)
725 * and that the requested page range starts and ends within the set of
726 * currently-partitioned simple pages.
728 static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl,
729 ulong dev_addr, uint num_pages)
731 ulong page_offset = dev_addr & (PAGE_SIZE - 1);
733 (dev_addr / PAGE_SIZE) & (pg_tbl->config.total_entries - 1);
735 if (gasket_components_to_dev_address(pg_tbl, 1, page_index,
736 page_offset) != dev_addr) {
737 dev_err(pg_tbl->device, "address is invalid, 0x%lX\n",
742 if (page_index >= pg_tbl->num_simple_entries) {
743 dev_err(pg_tbl->device,
744 "starting slot at %lu is too large, max is < %u\n",
745 page_index, pg_tbl->num_simple_entries);
749 if (page_index + num_pages > pg_tbl->num_simple_entries) {
750 dev_err(pg_tbl->device,
751 "ending slot at %lu is too large, max is <= %u\n",
752 page_index + num_pages, pg_tbl->num_simple_entries);
760 * Validity checking for extended addresses.
762 * Verify that address translation commutes (from address to/from page +
763 * offset) and that the requested page range starts and ends within the set of
764 * currently-partitioned extended pages.
766 static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl,
767 ulong dev_addr, uint num_pages)
769 /* Starting byte index of dev_addr into the first mapped page */
770 ulong page_offset = dev_addr & (PAGE_SIZE - 1);
771 ulong page_global_idx, page_lvl0_idx;
772 ulong num_lvl0_pages;
775 /* check if the device address is out of bound */
776 addr = dev_addr & ~((pg_tbl)->extended_flag);
777 if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) {
778 dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n",
783 /* Find the starting sub-page index in the space of all sub-pages. */
784 page_global_idx = (dev_addr / PAGE_SIZE) &
785 (pg_tbl->config.total_entries * GASKET_PAGES_PER_SUBTABLE - 1);
787 /* Find the starting level 0 index. */
788 page_lvl0_idx = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
790 /* Get the count of affected level 0 pages. */
791 num_lvl0_pages = (num_pages + GASKET_PAGES_PER_SUBTABLE - 1) /
792 GASKET_PAGES_PER_SUBTABLE;
794 if (gasket_components_to_dev_address(pg_tbl, 0, page_global_idx,
795 page_offset) != dev_addr) {
796 dev_err(pg_tbl->device, "address is invalid: 0x%lx\n",
801 if (page_lvl0_idx >= pg_tbl->num_extended_entries) {
802 dev_err(pg_tbl->device,
803 "starting level 0 slot at %lu is too large, max is < "
804 "%u\n", page_lvl0_idx, pg_tbl->num_extended_entries);
808 if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) {
809 dev_err(pg_tbl->device,
810 "ending level 0 slot at %lu is too large, max is <= %u\n",
811 page_lvl0_idx + num_lvl0_pages,
812 pg_tbl->num_extended_entries);
820 * Non-locking entry to unmapping routines.
821 * The page table mutex must be held by the caller.
823 static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl,
824 ulong dev_addr, uint num_pages)
829 if (gasket_addr_is_simple(pg_tbl, dev_addr))
830 gasket_unmap_simple_pages(pg_tbl, dev_addr, num_pages);
832 gasket_unmap_extended_pages(pg_tbl, dev_addr, num_pages);
836 * Allocate and map pages to simple addresses.
837 * If there is an error, no pages are mapped.
839 static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl,
840 ulong host_addr, ulong dev_addr,
844 uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr);
846 ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages);
848 dev_err(pg_tbl->device,
849 "page table slots %u (@ 0x%lx) to %u are not available\n",
850 slot_idx, dev_addr, slot_idx + num_pages - 1);
854 ret = gasket_perform_mapping(pg_tbl, pg_tbl->entries + slot_idx,
855 pg_tbl->base_slot + slot_idx, host_addr,
859 gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages);
860 dev_err(pg_tbl->device, "gasket_perform_mapping %d\n", ret);
866 * Allocate a second level page table.
867 * The page table mutex must be held by the caller.
869 static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl,
870 struct gasket_page_table_entry *pte,
873 ulong page_addr, subtable_bytes;
876 /* XXX FIX ME XXX this is inefficient for non-4K page sizes */
878 /* GFP_DMA flag must be passed to architectures for which
879 * part of the memory range is not considered DMA'able.
880 * This seems to be the case for Juno board with 4.5.0 Linaro kernel
882 page_addr = get_zeroed_page(GFP_KERNEL | GFP_DMA);
885 pte->page = virt_to_page((void *)page_addr);
888 subtable_bytes = sizeof(struct gasket_page_table_entry) *
889 GASKET_PAGES_PER_SUBTABLE;
890 pte->sublevel = vzalloc(subtable_bytes);
891 if (!pte->sublevel) {
892 free_page(page_addr);
893 memset(pte, 0, sizeof(struct gasket_page_table_entry));
897 /* Map the page into DMA space. */
898 pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE,
900 /* Wait until the page is mapped. */
903 /* make the addresses available to the device */
904 dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG;
905 writeq(dma_addr, slot);
907 pte->status = PTE_INUSE;
913 * Allocate slots in an extended page table. Check to see if a range of page
914 * table slots are available. If necessary, memory is allocated for second level
917 * Note that memory for second level page tables is allocated as needed, but
918 * that memory is only freed on the final close of the device file, when the
919 * page tables are repartitioned, or the the device is removed. If there is an
920 * error or if the full range of slots is not available, any memory
921 * allocated for second level page tables remains allocated until final close,
922 * repartition, or device removal.
924 * The page table mutex must be held by the caller.
926 static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl,
927 ulong dev_addr, uint num_entries)
930 uint remain, subtable_slot_idx, len;
931 struct gasket_page_table_entry *pte;
934 remain = num_entries;
935 subtable_slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
936 pte = pg_tbl->entries + pg_tbl->num_simple_entries +
937 gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
938 slot = pg_tbl->base_slot + pg_tbl->num_simple_entries +
939 gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
943 GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx);
945 if (pte->status == PTE_FREE) {
946 ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot);
948 dev_err(pg_tbl->device,
949 "no memory for extended addr subtable\n");
953 if (!gasket_is_pte_range_free(pte->sublevel +
954 subtable_slot_idx, len))
959 subtable_slot_idx = 0;
968 * gasket_map_extended_pages - Get and map buffers to extended addresses.
969 * If there is an error, no pages are mapped.
971 static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl,
972 ulong host_addr, ulong dev_addr,
977 uint slot_idx, remain, len;
978 struct gasket_page_table_entry *pte;
979 u64 __iomem *slot_base;
981 ret = gasket_alloc_extended_entries(pg_tbl, dev_addr, num_pages);
983 dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1;
984 dev_err(pg_tbl->device,
985 "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are "
987 gasket_extended_lvl0_page_idx(pg_tbl, dev_addr),
989 gasket_extended_lvl1_page_idx(pg_tbl, dev_addr),
990 gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end),
991 gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end));
996 slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
997 pte = pg_tbl->entries + pg_tbl->num_simple_entries +
998 gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
1000 while (remain > 0) {
1001 len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx);
1004 (u64 __iomem *)(page_address(pte->page) + pte->offset);
1005 ret = gasket_perform_mapping(pg_tbl, pte->sublevel + slot_idx,
1006 slot_base + slot_idx, host_addr,
1009 gasket_page_table_unmap_nolock(pg_tbl, dev_addr,
1017 host_addr += len * PAGE_SIZE;
1024 * See gasket_page_table.h for general description.
1026 * gasket_page_table_map calls either gasket_map_simple_pages() or
1027 * gasket_map_extended_pages() to actually perform the mapping.
1029 * The page table mutex is held for the entire operation.
1031 int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr,
1032 ulong dev_addr, uint num_pages)
1039 mutex_lock(&pg_tbl->mutex);
1041 if (gasket_addr_is_simple(pg_tbl, dev_addr)) {
1042 ret = gasket_map_simple_pages(pg_tbl, host_addr, dev_addr,
1045 ret = gasket_map_extended_pages(pg_tbl, host_addr, dev_addr,
1049 mutex_unlock(&pg_tbl->mutex);
1051 dev_dbg(pg_tbl->device,
1052 "%s done: ha %llx daddr %llx num %d, ret %d\n",
1053 __func__, (unsigned long long)host_addr,
1054 (unsigned long long)dev_addr, num_pages, ret);
1057 EXPORT_SYMBOL(gasket_page_table_map);
1060 * See gasket_page_table.h for general description.
1062 * gasket_page_table_unmap takes the page table lock and calls either
1063 * gasket_unmap_simple_pages() or gasket_unmap_extended_pages() to
1064 * actually unmap the pages from device space.
1066 * The page table mutex is held for the entire operation.
1068 void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, ulong dev_addr,
1074 mutex_lock(&pg_tbl->mutex);
1075 gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages);
1076 mutex_unlock(&pg_tbl->mutex);
1078 EXPORT_SYMBOL(gasket_page_table_unmap);
1080 static void gasket_page_table_unmap_all_nolock(struct gasket_page_table *pg_tbl)
1082 gasket_unmap_simple_pages(pg_tbl,
1083 gasket_components_to_dev_address(pg_tbl, 1, 0,
1085 pg_tbl->num_simple_entries);
1086 gasket_unmap_extended_pages(pg_tbl,
1087 gasket_components_to_dev_address(pg_tbl, 0,
1089 pg_tbl->num_extended_entries *
1090 GASKET_PAGES_PER_SUBTABLE);
1093 /* See gasket_page_table.h for description. */
1094 void gasket_page_table_unmap_all(struct gasket_page_table *pg_tbl)
1096 mutex_lock(&pg_tbl->mutex);
1097 gasket_page_table_unmap_all_nolock(pg_tbl);
1098 mutex_unlock(&pg_tbl->mutex);
1100 EXPORT_SYMBOL(gasket_page_table_unmap_all);
1102 /* See gasket_page_table.h for description. */
1103 void gasket_page_table_reset(struct gasket_page_table *pg_tbl)
1105 mutex_lock(&pg_tbl->mutex);
1106 gasket_page_table_unmap_all_nolock(pg_tbl);
1107 writeq(pg_tbl->config.total_entries, pg_tbl->extended_offset_reg);
1108 mutex_unlock(&pg_tbl->mutex);
1111 /* See gasket_page_table.h for description. */
1112 int gasket_page_table_lookup_page(
1113 struct gasket_page_table *pg_tbl, ulong dev_addr, struct page **ppage,
1117 struct gasket_page_table_entry *pte;
1119 mutex_lock(&pg_tbl->mutex);
1120 if (gasket_addr_is_simple(pg_tbl, dev_addr)) {
1121 page_num = gasket_simple_page_idx(pg_tbl, dev_addr);
1122 if (page_num >= pg_tbl->num_simple_entries)
1125 pte = pg_tbl->entries + page_num;
1126 if (pte->status != PTE_INUSE)
1129 /* Find the level 0 entry, */
1130 page_num = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
1131 if (page_num >= pg_tbl->num_extended_entries)
1134 pte = pg_tbl->entries + pg_tbl->num_simple_entries + page_num;
1135 if (pte->status != PTE_INUSE)
1138 /* and its contained level 1 entry. */
1139 page_num = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
1140 pte = pte->sublevel + page_num;
1141 if (pte->status != PTE_INUSE)
1146 *poffset = pte->offset;
1147 mutex_unlock(&pg_tbl->mutex);
1153 mutex_unlock(&pg_tbl->mutex);
1157 /* See gasket_page_table.h for description. */
1158 bool gasket_page_table_are_addrs_bad(
1159 struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr,
1162 if (host_addr & (PAGE_SIZE - 1)) {
1163 dev_err(pg_tbl->device,
1164 "host mapping address 0x%lx must be page aligned\n",
1169 return gasket_page_table_is_dev_addr_bad(pg_tbl, dev_addr, bytes);
1171 EXPORT_SYMBOL(gasket_page_table_are_addrs_bad);
1173 /* See gasket_page_table.h for description. */
1174 bool gasket_page_table_is_dev_addr_bad(
1175 struct gasket_page_table *pg_tbl, ulong dev_addr, ulong bytes)
1177 uint num_pages = bytes / PAGE_SIZE;
1179 if (bytes & (PAGE_SIZE - 1)) {
1180 dev_err(pg_tbl->device,
1181 "mapping size 0x%lX must be page aligned\n", bytes);
1185 if (num_pages == 0) {
1186 dev_err(pg_tbl->device,
1187 "requested mapping is less than one page: %lu / %lu\n",
1192 if (gasket_addr_is_simple(pg_tbl, dev_addr))
1193 return gasket_is_simple_dev_addr_bad(pg_tbl, dev_addr,
1195 return gasket_is_extended_dev_addr_bad(pg_tbl, dev_addr, num_pages);
1197 EXPORT_SYMBOL(gasket_page_table_is_dev_addr_bad);
1199 /* See gasket_page_table.h for description. */
1200 uint gasket_page_table_max_size(struct gasket_page_table *page_table)
1204 return page_table->config.total_entries;
1206 EXPORT_SYMBOL(gasket_page_table_max_size);
1208 /* See gasket_page_table.h for description. */
1209 uint gasket_page_table_num_entries(struct gasket_page_table *pg_tbl)
1213 return pg_tbl->num_simple_entries + pg_tbl->num_extended_entries;
1215 EXPORT_SYMBOL(gasket_page_table_num_entries);
1217 /* See gasket_page_table.h for description. */
1218 uint gasket_page_table_num_simple_entries(struct gasket_page_table *pg_tbl)
1222 return pg_tbl->num_simple_entries;
1224 EXPORT_SYMBOL(gasket_page_table_num_simple_entries);
1226 /* See gasket_page_table.h for description. */
1227 uint gasket_page_table_num_active_pages(struct gasket_page_table *pg_tbl)
1231 return pg_tbl->num_active_pages;
1233 EXPORT_SYMBOL(gasket_page_table_num_active_pages);
1235 /* See gasket_page_table.h */
1236 int gasket_page_table_system_status(struct gasket_page_table *page_table)
1239 return GASKET_STATUS_LAMED;
1241 if (gasket_page_table_num_entries(page_table) == 0) {
1242 dev_dbg(page_table->device, "Page table size is 0\n");
1243 return GASKET_STATUS_LAMED;
1246 return GASKET_STATUS_ALIVE;
1249 /* Record the host_addr to coherent dma memory mapping. */
1250 int gasket_set_user_virt(
1251 struct gasket_dev *gasket_dev, u64 size, dma_addr_t dma_address,
1255 struct gasket_page_table *pg_tbl;
1257 unsigned int num_pages = size / PAGE_SIZE;
1260 * TODO: for future chipset, better handling of the case where multiple
1261 * page tables are supported on a given device
1263 pg_tbl = gasket_dev->page_table[0];
1265 dev_dbg(gasket_dev->dev, "%s: invalid page table index\n",
1269 for (j = 0; j < num_pages; j++) {
1270 pg_tbl->coherent_pages[j].user_virt =
1271 (u64)vma + j * PAGE_SIZE;
1276 /* Allocate a block of coherent memory. */
1277 int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size,
1278 dma_addr_t *dma_address, u64 index)
1283 unsigned int num_pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
1284 const struct gasket_driver_desc *driver_desc =
1285 gasket_get_driver_desc(gasket_dev);
1287 if (!gasket_dev->page_table[index])
1293 mem = dma_alloc_coherent(gasket_get_device(gasket_dev),
1294 num_pages * PAGE_SIZE, &handle, 0);
1298 gasket_dev->page_table[index]->num_coherent_pages = num_pages;
1300 /* allocate the physical memory block */
1301 gasket_dev->page_table[index]->coherent_pages =
1302 kcalloc(num_pages, sizeof(struct gasket_coherent_page_entry),
1304 if (!gasket_dev->page_table[index]->coherent_pages)
1308 gasket_dev->coherent_buffer.length_bytes =
1309 PAGE_SIZE * (num_pages);
1310 gasket_dev->coherent_buffer.phys_base = handle;
1311 gasket_dev->coherent_buffer.virt_base = mem;
1313 *dma_address = driver_desc->coherent_buffer_description.base;
1314 for (j = 0; j < num_pages; j++) {
1315 gasket_dev->page_table[index]->coherent_pages[j].paddr =
1316 handle + j * PAGE_SIZE;
1317 gasket_dev->page_table[index]->coherent_pages[j].kernel_virt =
1318 (u64)mem + j * PAGE_SIZE;
1321 if (*dma_address == 0)
1327 dma_free_coherent(gasket_get_device(gasket_dev),
1328 num_pages * PAGE_SIZE, mem, handle);
1331 if (gasket_dev->page_table[index]->coherent_pages) {
1332 kfree(gasket_dev->page_table[index]->coherent_pages);
1333 gasket_dev->page_table[index]->coherent_pages = NULL;
1335 gasket_dev->page_table[index]->num_coherent_pages = 0;
1339 /* Free a block of coherent memory. */
1340 int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size,
1341 dma_addr_t dma_address, u64 index)
1343 const struct gasket_driver_desc *driver_desc;
1345 if (!gasket_dev->page_table[index])
1348 driver_desc = gasket_get_driver_desc(gasket_dev);
1350 if (driver_desc->coherent_buffer_description.base != dma_address)
1351 return -EADDRNOTAVAIL;
1353 if (gasket_dev->coherent_buffer.length_bytes) {
1354 dma_free_coherent(gasket_get_device(gasket_dev),
1355 gasket_dev->coherent_buffer.length_bytes,
1356 gasket_dev->coherent_buffer.virt_base,
1357 gasket_dev->coherent_buffer.phys_base);
1358 gasket_dev->coherent_buffer.length_bytes = 0;
1359 gasket_dev->coherent_buffer.virt_base = NULL;
1360 gasket_dev->coherent_buffer.phys_base = 0;
1365 /* Release all coherent memory. */
1366 void gasket_free_coherent_memory_all(
1367 struct gasket_dev *gasket_dev, u64 index)
1369 if (!gasket_dev->page_table[index])
1372 if (gasket_dev->coherent_buffer.length_bytes) {
1373 dma_free_coherent(gasket_get_device(gasket_dev),
1374 gasket_dev->coherent_buffer.length_bytes,
1375 gasket_dev->coherent_buffer.virt_base,
1376 gasket_dev->coherent_buffer.phys_base);
1377 gasket_dev->coherent_buffer.length_bytes = 0;
1378 gasket_dev->coherent_buffer.virt_base = NULL;
1379 gasket_dev->coherent_buffer.phys_base = 0;