2 #include <linux/highmem.h>
3 #include <linux/kernel.h>
4 #include <linux/mmdebug.h>
5 #include <linux/mm_types.h>
6 #include <linux/pagemap.h>
7 #include <linux/rcupdate.h>
9 #include <linux/swap.h>
11 #include <asm/pgalloc.h>
14 #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
16 static bool tlb_next_batch(struct mmu_gather *tlb)
18 struct mmu_gather_batch *batch;
22 tlb->active = batch->next;
26 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
29 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
36 batch->max = MAX_GATHER_BATCH;
38 tlb->active->next = batch;
44 static void tlb_batch_pages_flush(struct mmu_gather *tlb)
46 struct mmu_gather_batch *batch;
48 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
49 free_pages_and_swap_cache(batch->pages, batch->nr);
52 tlb->active = &tlb->local;
55 static void tlb_batch_list_free(struct mmu_gather *tlb)
57 struct mmu_gather_batch *batch, *next;
59 for (batch = tlb->local.next; batch; batch = next) {
61 free_pages((unsigned long)batch, 0);
63 tlb->local.next = NULL;
66 bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
68 struct mmu_gather_batch *batch;
72 #ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
73 VM_WARN_ON(tlb->page_size != page_size);
78 * Add the page and check if we are full. If so
81 batch->pages[batch->nr++] = page;
82 if (batch->nr == batch->max) {
83 if (!tlb_next_batch(tlb))
87 VM_BUG_ON_PAGE(batch->nr > batch->max, page);
92 #endif /* HAVE_MMU_GATHER_NO_GATHER */
94 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
97 * See the comment near struct mmu_table_batch.
101 * If we want tlb_remove_table() to imply TLB invalidates.
103 static inline void tlb_table_invalidate(struct mmu_gather *tlb)
105 if (tlb_needs_table_invalidate()) {
107 * Invalidate page-table caches used by hardware walkers. Then
108 * we still need to RCU-sched wait while freeing the pages
109 * because software walkers can still be in-flight.
111 tlb_flush_mmu_tlbonly(tlb);
115 static void tlb_remove_table_smp_sync(void *arg)
117 /* Simply deliver the interrupt */
120 void tlb_remove_table_sync_one(void)
122 smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
125 static void tlb_remove_table_one(void *table)
128 * This isn't an RCU grace period and hence the page-tables cannot be
129 * assumed to be actually RCU-freed.
131 * It is however sufficient for software page-table walkers that rely on
132 * IRQ disabling. See the comment near struct mmu_table_batch.
134 smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
135 __tlb_remove_table(table);
138 static void tlb_remove_table_rcu(struct rcu_head *head)
140 struct mmu_table_batch *batch;
143 batch = container_of(head, struct mmu_table_batch, rcu);
145 for (i = 0; i < batch->nr; i++)
146 __tlb_remove_table(batch->tables[i]);
148 free_page((unsigned long)batch);
151 static void tlb_table_flush(struct mmu_gather *tlb)
153 struct mmu_table_batch **batch = &tlb->batch;
156 tlb_table_invalidate(tlb);
157 call_rcu(&(*batch)->rcu, tlb_remove_table_rcu);
162 void tlb_remove_table(struct mmu_gather *tlb, void *table)
164 struct mmu_table_batch **batch = &tlb->batch;
166 if (*batch == NULL) {
167 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
168 if (*batch == NULL) {
169 tlb_table_invalidate(tlb);
170 tlb_remove_table_one(table);
176 (*batch)->tables[(*batch)->nr++] = table;
177 if ((*batch)->nr == MAX_TABLE_BATCH)
178 tlb_table_flush(tlb);
181 #endif /* CONFIG_HAVE_RCU_TABLE_FREE */
183 static void tlb_flush_mmu_free(struct mmu_gather *tlb)
185 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
186 tlb_table_flush(tlb);
188 #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
189 tlb_batch_pages_flush(tlb);
193 void tlb_flush_mmu(struct mmu_gather *tlb)
195 tlb_flush_mmu_tlbonly(tlb);
196 tlb_flush_mmu_free(tlb);
200 * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
201 * @tlb: the mmu_gather structure to initialize
202 * @mm: the mm_struct of the target address space
203 * @start: start of the region that will be removed from the page-table
204 * @end: end of the region that will be removed from the page-table
206 * Called to initialize an (on-stack) mmu_gather structure for page-table
207 * tear-down from @mm. The @start and @end are set to 0 and -1
208 * respectively when @mm is without users and we're going to destroy
209 * the full address space (exit/execve).
211 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
212 unsigned long start, unsigned long end)
216 /* Is it from 0 to ~0? */
217 tlb->fullmm = !(start | (end+1));
219 #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
220 tlb->need_flush_all = 0;
221 tlb->local.next = NULL;
223 tlb->local.max = ARRAY_SIZE(tlb->__pages);
224 tlb->active = &tlb->local;
225 tlb->batch_count = 0;
228 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
231 #ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
235 __tlb_reset_range(tlb);
236 inc_tlb_flush_pending(tlb->mm);
240 * tlb_finish_mmu - finish an mmu_gather structure
241 * @tlb: the mmu_gather structure to finish
242 * @start: start of the region that will be removed from the page-table
243 * @end: end of the region that will be removed from the page-table
245 * Called at the end of the shootdown operation to free up any resources that
248 void tlb_finish_mmu(struct mmu_gather *tlb,
249 unsigned long start, unsigned long end)
252 * If there are parallel threads are doing PTE changes on same range
253 * under non-exclusive lock (e.g., mmap_sem read-side) but defer TLB
254 * flush by batching, one thread may end up seeing inconsistent PTEs
255 * and result in having stale TLB entries. So flush TLB forcefully
256 * if we detect parallel PTE batching threads.
258 * However, some syscalls, e.g. munmap(), may free page tables, this
259 * needs force flush everything in the given range. Otherwise this
260 * may result in having stale TLB entries for some architectures,
261 * e.g. aarch64, that could specify flush what level TLB.
263 if (mm_tlb_flush_nested(tlb->mm)) {
265 * The aarch64 yields better performance with fullmm by
266 * avoiding multiple CPUs spamming TLBI messages at the
269 * On x86 non-fullmm doesn't yield significant difference
273 __tlb_reset_range(tlb);
274 tlb->freed_tables = 1;
279 #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
280 tlb_batch_list_free(tlb);
282 dec_tlb_flush_pending(tlb->mm);