1 // SPDX-License-Identifier: GPL-2.0
4 * Handling Page Tables through page fragments
8 #include <linux/kernel.h>
11 #include <linux/percpu.h>
12 #include <linux/hardirq.h>
13 #include <linux/hugetlb.h>
14 #include <asm/pgalloc.h>
15 #include <asm/tlbflush.h>
18 void pte_frag_destroy(void *pte_frag)
23 page = virt_to_page(pte_frag);
24 /* drop all the pending references */
25 count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
26 /* We allow PTE_FRAG_NR fragments from a PTE page */
27 if (atomic_sub_and_test(PTE_FRAG_NR - count, &page->pt_frag_refcount)) {
28 pgtable_pte_page_dtor(page);
33 static pte_t *get_pte_from_cache(struct mm_struct *mm)
40 spin_lock(&mm->page_table_lock);
41 ret = pte_frag_get(&mm->context);
43 pte_frag = ret + PTE_FRAG_SIZE;
45 * If we have taken up all the fragments mark PTE page NULL
47 if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
49 pte_frag_set(&mm->context, pte_frag);
51 spin_unlock(&mm->page_table_lock);
55 static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel)
61 page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
64 if (!pgtable_pte_page_ctor(page)) {
69 page = alloc_page(PGALLOC_GFP);
74 atomic_set(&page->pt_frag_refcount, 1);
76 ret = page_address(page);
78 * if we support only one fragment just return the
83 spin_lock(&mm->page_table_lock);
85 * If we find pgtable_page set, we return
86 * the allocated page with single fragment
89 if (likely(!pte_frag_get(&mm->context))) {
90 atomic_set(&page->pt_frag_refcount, PTE_FRAG_NR);
91 pte_frag_set(&mm->context, ret + PTE_FRAG_SIZE);
93 spin_unlock(&mm->page_table_lock);
98 pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel)
102 pte = get_pte_from_cache(mm);
106 return __alloc_for_ptecache(mm, kernel);
109 void pte_fragment_free(unsigned long *table, int kernel)
111 struct page *page = virt_to_page(table);
113 if (PageReserved(page))
114 return free_reserved_page(page);
116 BUG_ON(atomic_read(&page->pt_frag_refcount) <= 0);
117 if (atomic_dec_and_test(&page->pt_frag_refcount)) {
119 pgtable_pte_page_dtor(page);