1 // SPDX-License-Identifier: GPL-2.0-only
2 /**************************************************************************
3 * Copyright (c) 2007, Intel Corporation.
5 **************************************************************************/
7 #include <linux/highmem.h>
14 * Code for the SGX MMU:
18 * clflush on one processor only:
19 * clflush should apparently flush the cache line on all processors in an
25 * The usage of the slots must be completely encapsulated within a spinlock, and
26 * no other functions that may be using the locks for other purposed may be
27 * called from within the locked region.
28 * Since the slots are per processor, this will guarantee that we are the only
33 * TODO: Inserting ptes from an interrupt handler:
34 * This may be desirable for some SGX functionality where the GPU can fault in
35 * needed pages. For that, we need to make an atomic insert_pages function, that
37 * If it fails, the caller need to insert the page using a workqueue function,
38 * but on average it should be fast.
41 static inline uint32_t psb_mmu_pt_index(uint32_t offset)
43 return (offset >> PSB_PTE_SHIFT) & 0x3FF;
46 static inline uint32_t psb_mmu_pd_index(uint32_t offset)
48 return offset >> PSB_PDE_SHIFT;
51 #if defined(CONFIG_X86)
52 static inline void psb_clflush(void *addr)
54 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
57 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
59 if (!driver->has_clflush)
68 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
74 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
76 struct drm_device *dev = driver->dev;
77 struct drm_psb_private *dev_priv = dev->dev_private;
79 if (atomic_read(&driver->needs_tlbflush) || force) {
80 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
81 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
83 /* Make sure data cache is turned off before enabling it */
85 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
86 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
87 if (driver->msvdx_mmu_invaldc)
88 atomic_set(driver->msvdx_mmu_invaldc, 1);
90 atomic_set(&driver->needs_tlbflush, 0);
94 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
96 down_write(&driver->sem);
97 psb_mmu_flush_pd_locked(driver, force);
98 up_write(&driver->sem);
102 void psb_mmu_flush(struct psb_mmu_driver *driver)
104 struct drm_device *dev = driver->dev;
105 struct drm_psb_private *dev_priv = dev->dev_private;
108 down_write(&driver->sem);
109 val = PSB_RSGX32(PSB_CR_BIF_CTRL);
110 if (atomic_read(&driver->needs_tlbflush))
111 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
113 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
115 /* Make sure data cache is turned off and MMU is flushed before
116 restoring bank interface control register */
118 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
120 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
122 atomic_set(&driver->needs_tlbflush, 0);
123 if (driver->msvdx_mmu_invaldc)
124 atomic_set(driver->msvdx_mmu_invaldc, 1);
125 up_write(&driver->sem);
128 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
130 struct drm_device *dev = pd->driver->dev;
131 struct drm_psb_private *dev_priv = dev->dev_private;
132 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
133 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
135 down_write(&pd->driver->sem);
136 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
138 psb_mmu_flush_pd_locked(pd->driver, 1);
139 pd->hw_context = hw_context;
140 up_write(&pd->driver->sem);
144 static inline unsigned long psb_pd_addr_end(unsigned long addr,
147 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
148 return (addr < end) ? addr : end;
151 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
153 uint32_t mask = PSB_PTE_VALID;
155 if (type & PSB_MMU_CACHED_MEMORY)
156 mask |= PSB_PTE_CACHED;
157 if (type & PSB_MMU_RO_MEMORY)
159 if (type & PSB_MMU_WO_MEMORY)
162 return (pfn << PAGE_SHIFT) | mask;
165 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
166 int trap_pagefaults, int invalid_type)
168 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
175 pd->p = alloc_page(GFP_DMA32);
178 pd->dummy_pt = alloc_page(GFP_DMA32);
181 pd->dummy_page = alloc_page(GFP_DMA32);
185 if (!trap_pagefaults) {
186 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
188 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
195 v = kmap(pd->dummy_pt);
196 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
197 v[i] = pd->invalid_pte;
199 kunmap(pd->dummy_pt);
202 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
203 v[i] = pd->invalid_pde;
207 clear_page(kmap(pd->dummy_page));
208 kunmap(pd->dummy_page);
210 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
215 pd->pd_mask = PSB_PTE_VALID;
221 __free_page(pd->dummy_page);
223 __free_page(pd->dummy_pt);
231 static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
237 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
239 struct psb_mmu_driver *driver = pd->driver;
240 struct drm_device *dev = driver->dev;
241 struct drm_psb_private *dev_priv = dev->dev_private;
242 struct psb_mmu_pt *pt;
245 down_write(&driver->sem);
246 if (pd->hw_context != -1) {
247 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
248 psb_mmu_flush_pd_locked(driver, 1);
251 /* Should take the spinlock here, but we don't need to do that
252 since we have the semaphore in write mode. */
254 for (i = 0; i < 1024; ++i) {
261 __free_page(pd->dummy_page);
262 __free_page(pd->dummy_pt);
265 up_write(&driver->sem);
268 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
270 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
272 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
273 uint32_t clflush_count = PAGE_SIZE / clflush_add;
274 spinlock_t *lock = &pd->driver->lock;
282 pt->p = alloc_page(GFP_DMA32);
290 v = kmap_atomic(pt->p);
292 ptes = (uint32_t *) v;
293 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
294 *ptes++ = pd->invalid_pte;
296 #if defined(CONFIG_X86)
297 if (pd->driver->has_clflush && pd->hw_context != -1) {
299 for (i = 0; i < clflush_count; ++i) {
316 struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
319 uint32_t index = psb_mmu_pd_index(addr);
320 struct psb_mmu_pt *pt;
322 spinlock_t *lock = &pd->driver->lock;
325 pt = pd->tables[index];
328 pt = psb_mmu_alloc_pt(pd);
333 if (pd->tables[index]) {
337 pt = pd->tables[index];
341 v = kmap_atomic(pd->p);
342 pd->tables[index] = pt;
343 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
345 kunmap_atomic((void *) v);
347 if (pd->hw_context != -1) {
348 psb_mmu_clflush(pd->driver, (void *)&v[index]);
349 atomic_set(&pd->driver->needs_tlbflush, 1);
352 pt->v = kmap_atomic(pt->p);
356 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
359 uint32_t index = psb_mmu_pd_index(addr);
360 struct psb_mmu_pt *pt;
361 spinlock_t *lock = &pd->driver->lock;
364 pt = pd->tables[index];
369 pt->v = kmap_atomic(pt->p);
373 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
375 struct psb_mmu_pd *pd = pt->pd;
378 kunmap_atomic(pt->v);
379 if (pt->count == 0) {
380 v = kmap_atomic(pd->p);
381 v[pt->index] = pd->invalid_pde;
382 pd->tables[pt->index] = NULL;
384 if (pd->hw_context != -1) {
385 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
386 atomic_set(&pd->driver->needs_tlbflush, 1);
389 spin_unlock(&pd->driver->lock);
393 spin_unlock(&pd->driver->lock);
396 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
399 pt->v[psb_mmu_pt_index(addr)] = pte;
402 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
405 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
408 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
410 struct psb_mmu_pd *pd;
412 down_read(&driver->sem);
413 pd = driver->default_pd;
414 up_read(&driver->sem);
419 /* Returns the physical address of the PD shared by sgx/msvdx */
420 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)
422 struct psb_mmu_pd *pd;
424 pd = psb_mmu_get_default_pd(driver);
425 return page_to_pfn(pd->p) << PAGE_SHIFT;
428 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
430 struct drm_device *dev = driver->dev;
431 struct drm_psb_private *dev_priv = dev->dev_private;
433 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
434 psb_mmu_free_pagedir(driver->default_pd);
438 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
441 atomic_t *msvdx_mmu_invaldc)
443 struct psb_mmu_driver *driver;
444 struct drm_psb_private *dev_priv = dev->dev_private;
446 driver = kmalloc(sizeof(*driver), GFP_KERNEL);
452 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
454 if (!driver->default_pd)
457 spin_lock_init(&driver->lock);
458 init_rwsem(&driver->sem);
459 down_write(&driver->sem);
460 atomic_set(&driver->needs_tlbflush, 1);
461 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
463 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
464 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
466 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
469 driver->has_clflush = 0;
471 #if defined(CONFIG_X86)
472 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
473 uint32_t tfms, misc, cap0, cap4, clflush_size;
476 * clflush size is determined at kernel setup for x86_64 but not
477 * for i386. We have to do it here.
480 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
481 clflush_size = ((misc >> 8) & 0xff) * 8;
482 driver->has_clflush = 1;
483 driver->clflush_add =
484 PAGE_SIZE * clflush_size / sizeof(uint32_t);
485 driver->clflush_mask = driver->clflush_add - 1;
486 driver->clflush_mask = ~driver->clflush_mask;
490 up_write(&driver->sem);
498 #if defined(CONFIG_X86)
499 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
500 uint32_t num_pages, uint32_t desired_tile_stride,
501 uint32_t hw_tile_stride)
503 struct psb_mmu_pt *pt;
510 unsigned long row_add;
511 unsigned long clflush_add = pd->driver->clflush_add;
512 unsigned long clflush_mask = pd->driver->clflush_mask;
514 if (!pd->driver->has_clflush)
518 rows = num_pages / desired_tile_stride;
520 desired_tile_stride = num_pages;
522 add = desired_tile_stride << PAGE_SHIFT;
523 row_add = hw_tile_stride << PAGE_SHIFT;
525 for (i = 0; i < rows; ++i) {
531 next = psb_pd_addr_end(addr, end);
532 pt = psb_mmu_pt_map_lock(pd, addr);
536 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
537 } while (addr += clflush_add,
538 (addr & clflush_mask) < next);
540 psb_mmu_pt_unmap_unlock(pt);
541 } while (addr = next, next != end);
547 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
548 uint32_t num_pages, uint32_t desired_tile_stride,
549 uint32_t hw_tile_stride)
551 drm_ttm_cache_flush();
555 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
556 unsigned long address, uint32_t num_pages)
558 struct psb_mmu_pt *pt;
562 unsigned long f_address = address;
564 down_read(&pd->driver->sem);
567 end = addr + (num_pages << PAGE_SHIFT);
570 next = psb_pd_addr_end(addr, end);
571 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
575 psb_mmu_invalidate_pte(pt, addr);
577 } while (addr += PAGE_SIZE, addr < next);
578 psb_mmu_pt_unmap_unlock(pt);
580 } while (addr = next, next != end);
583 if (pd->hw_context != -1)
584 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
586 up_read(&pd->driver->sem);
588 if (pd->hw_context != -1)
589 psb_mmu_flush(pd->driver);
594 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
595 uint32_t num_pages, uint32_t desired_tile_stride,
596 uint32_t hw_tile_stride)
598 struct psb_mmu_pt *pt;
605 unsigned long row_add;
606 unsigned long f_address = address;
609 rows = num_pages / desired_tile_stride;
611 desired_tile_stride = num_pages;
613 add = desired_tile_stride << PAGE_SHIFT;
614 row_add = hw_tile_stride << PAGE_SHIFT;
616 down_read(&pd->driver->sem);
618 /* Make sure we only need to flush this processor's cache */
620 for (i = 0; i < rows; ++i) {
626 next = psb_pd_addr_end(addr, end);
627 pt = psb_mmu_pt_map_lock(pd, addr);
631 psb_mmu_invalidate_pte(pt, addr);
634 } while (addr += PAGE_SIZE, addr < next);
635 psb_mmu_pt_unmap_unlock(pt);
637 } while (addr = next, next != end);
640 if (pd->hw_context != -1)
641 psb_mmu_flush_ptes(pd, f_address, num_pages,
642 desired_tile_stride, hw_tile_stride);
644 up_read(&pd->driver->sem);
646 if (pd->hw_context != -1)
647 psb_mmu_flush(pd->driver);
650 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
651 unsigned long address, uint32_t num_pages,
654 struct psb_mmu_pt *pt;
659 unsigned long f_address = address;
662 down_read(&pd->driver->sem);
665 end = addr + (num_pages << PAGE_SHIFT);
668 next = psb_pd_addr_end(addr, end);
669 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
675 pte = psb_mmu_mask_pte(start_pfn++, type);
676 psb_mmu_set_pte(pt, addr, pte);
678 } while (addr += PAGE_SIZE, addr < next);
679 psb_mmu_pt_unmap_unlock(pt);
681 } while (addr = next, next != end);
685 if (pd->hw_context != -1)
686 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
688 up_read(&pd->driver->sem);
690 if (pd->hw_context != -1)
691 psb_mmu_flush(pd->driver);
696 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
697 unsigned long address, uint32_t num_pages,
698 uint32_t desired_tile_stride, uint32_t hw_tile_stride,
701 struct psb_mmu_pt *pt;
709 unsigned long row_add;
710 unsigned long f_address = address;
713 if (hw_tile_stride) {
714 if (num_pages % desired_tile_stride != 0)
716 rows = num_pages / desired_tile_stride;
718 desired_tile_stride = num_pages;
721 add = desired_tile_stride << PAGE_SHIFT;
722 row_add = hw_tile_stride << PAGE_SHIFT;
724 down_read(&pd->driver->sem);
726 for (i = 0; i < rows; ++i) {
732 next = psb_pd_addr_end(addr, end);
733 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
737 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
739 psb_mmu_set_pte(pt, addr, pte);
741 } while (addr += PAGE_SIZE, addr < next);
742 psb_mmu_pt_unmap_unlock(pt);
744 } while (addr = next, next != end);
751 if (pd->hw_context != -1)
752 psb_mmu_flush_ptes(pd, f_address, num_pages,
753 desired_tile_stride, hw_tile_stride);
755 up_read(&pd->driver->sem);
757 if (pd->hw_context != -1)
758 psb_mmu_flush(pd->driver);
763 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
767 struct psb_mmu_pt *pt;
769 spinlock_t *lock = &pd->driver->lock;
771 down_read(&pd->driver->sem);
772 pt = psb_mmu_pt_map_lock(pd, virtual);
777 v = kmap_atomic(pd->p);
778 tmp = v[psb_mmu_pd_index(virtual)];
782 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
783 !(pd->invalid_pte & PSB_PTE_VALID)) {
788 *pfn = pd->invalid_pte >> PAGE_SHIFT;
791 tmp = pt->v[psb_mmu_pt_index(virtual)];
792 if (!(tmp & PSB_PTE_VALID)) {
796 *pfn = tmp >> PAGE_SHIFT;
798 psb_mmu_pt_unmap_unlock(pt);
800 up_read(&pd->driver->sem);