1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * TLB flush routines for radix kernels.
5 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
9 #include <linux/hugetlb.h>
10 #include <linux/memblock.h>
11 #include <linux/mmu_context.h>
12 #include <linux/sched/mm.h>
13 #include <linux/debugfs.h>
15 #include <asm/ppc-opcode.h>
17 #include <asm/tlbflush.h>
18 #include <asm/trace.h>
19 #include <asm/cputhreads.h>
20 #include <asm/plpar_wrappers.h>
25 * tlbiel instruction for radix, set invalidation
26 * i.e., r=1 and is=01 or is=10 or is=11
28 static __always_inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is,
30 unsigned int ric, unsigned int prs)
35 rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
36 rs = ((unsigned long)pid << PPC_BITLSHIFT(31));
38 asm volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
39 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
43 static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)
47 asm volatile("ptesync": : :"memory");
50 * Flush the first set of the TLB, and the entire Page Walk Cache
51 * and partition table entries. Then flush the remaining sets of the
55 if (early_cpu_has_feature(CPU_FTR_HVMODE)) {
56 /* MSR[HV] should flush partition scope translations first. */
57 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
59 if (!early_cpu_has_feature(CPU_FTR_ARCH_31)) {
60 for (set = 1; set < num_sets; set++)
61 tlbiel_radix_set_isa300(set, is, 0,
66 /* Flush process scoped entries. */
67 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
69 if (!early_cpu_has_feature(CPU_FTR_ARCH_31)) {
70 for (set = 1; set < num_sets; set++)
71 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
74 ppc_after_tlbiel_barrier();
77 void radix__tlbiel_all(unsigned int action)
82 case TLB_INVAL_SCOPE_GLOBAL:
85 case TLB_INVAL_SCOPE_LPID:
92 if (early_cpu_has_feature(CPU_FTR_ARCH_300))
93 tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is);
95 WARN(1, "%s called on pre-POWER9 CPU\n", __func__);
97 asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory");
100 static __always_inline void __tlbiel_pid(unsigned long pid, int set,
103 unsigned long rb,rs,prs,r;
105 rb = PPC_BIT(53); /* IS = 1 */
106 rb |= set << PPC_BITLSHIFT(51);
107 rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
108 prs = 1; /* process scoped */
109 r = 1; /* radix format */
111 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
112 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
113 trace_tlbie(0, 1, rb, rs, ric, prs, r);
116 static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric)
118 unsigned long rb,rs,prs,r;
120 rb = PPC_BIT(53); /* IS = 1 */
121 rs = pid << PPC_BITLSHIFT(31);
122 prs = 1; /* process scoped */
123 r = 1; /* radix format */
125 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
126 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
127 trace_tlbie(0, 0, rb, rs, ric, prs, r);
130 static __always_inline void __tlbie_pid_lpid(unsigned long pid,
134 unsigned long rb, rs, prs, r;
136 rb = PPC_BIT(53); /* IS = 1 */
137 rs = (pid << PPC_BITLSHIFT(31)) | (lpid & ~(PPC_BITMASK(0, 31)));
138 prs = 1; /* process scoped */
139 r = 1; /* radix format */
141 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
142 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
143 trace_tlbie(0, 0, rb, rs, ric, prs, r);
145 static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric)
147 unsigned long rb,rs,prs,r;
149 rb = PPC_BIT(52); /* IS = 2 */
151 prs = 0; /* partition scoped */
152 r = 1; /* radix format */
154 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
155 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
156 trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
159 static __always_inline void __tlbie_lpid_guest(unsigned long lpid, unsigned long ric)
161 unsigned long rb,rs,prs,r;
163 rb = PPC_BIT(52); /* IS = 2 */
165 prs = 1; /* process scoped */
166 r = 1; /* radix format */
168 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
169 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
170 trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
173 static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid,
174 unsigned long ap, unsigned long ric)
176 unsigned long rb,rs,prs,r;
178 rb = va & ~(PPC_BITMASK(52, 63));
179 rb |= ap << PPC_BITLSHIFT(58);
180 rs = pid << PPC_BITLSHIFT(31);
181 prs = 1; /* process scoped */
182 r = 1; /* radix format */
184 asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
185 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
186 trace_tlbie(0, 1, rb, rs, ric, prs, r);
189 static __always_inline void __tlbie_va(unsigned long va, unsigned long pid,
190 unsigned long ap, unsigned long ric)
192 unsigned long rb,rs,prs,r;
194 rb = va & ~(PPC_BITMASK(52, 63));
195 rb |= ap << PPC_BITLSHIFT(58);
196 rs = pid << PPC_BITLSHIFT(31);
197 prs = 1; /* process scoped */
198 r = 1; /* radix format */
200 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
201 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
202 trace_tlbie(0, 0, rb, rs, ric, prs, r);
205 static __always_inline void __tlbie_va_lpid(unsigned long va, unsigned long pid,
207 unsigned long ap, unsigned long ric)
209 unsigned long rb, rs, prs, r;
211 rb = va & ~(PPC_BITMASK(52, 63));
212 rb |= ap << PPC_BITLSHIFT(58);
213 rs = (pid << PPC_BITLSHIFT(31)) | (lpid & ~(PPC_BITMASK(0, 31)));
214 prs = 1; /* process scoped */
215 r = 1; /* radix format */
217 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
218 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
219 trace_tlbie(0, 0, rb, rs, ric, prs, r);
222 static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid,
223 unsigned long ap, unsigned long ric)
225 unsigned long rb,rs,prs,r;
227 rb = va & ~(PPC_BITMASK(52, 63));
228 rb |= ap << PPC_BITLSHIFT(58);
230 prs = 0; /* partition scoped */
231 r = 1; /* radix format */
233 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
234 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
235 trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
239 static inline void fixup_tlbie_va(unsigned long va, unsigned long pid,
242 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
243 asm volatile("ptesync": : :"memory");
244 __tlbie_va(va, 0, ap, RIC_FLUSH_TLB);
247 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
248 asm volatile("ptesync": : :"memory");
249 __tlbie_va(va, pid, ap, RIC_FLUSH_TLB);
253 static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid,
256 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
257 asm volatile("ptesync": : :"memory");
258 __tlbie_pid(0, RIC_FLUSH_TLB);
261 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
262 asm volatile("ptesync": : :"memory");
263 __tlbie_va(va, pid, ap, RIC_FLUSH_TLB);
267 static inline void fixup_tlbie_va_range_lpid(unsigned long va,
272 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
273 asm volatile("ptesync" : : : "memory");
274 __tlbie_pid_lpid(0, lpid, RIC_FLUSH_TLB);
277 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
278 asm volatile("ptesync" : : : "memory");
279 __tlbie_va_lpid(va, pid, lpid, ap, RIC_FLUSH_TLB);
283 static inline void fixup_tlbie_pid(unsigned long pid)
286 * We can use any address for the invalidation, pick one which is
287 * probably unused as an optimisation.
289 unsigned long va = ((1UL << 52) - 1);
291 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
292 asm volatile("ptesync": : :"memory");
293 __tlbie_pid(0, RIC_FLUSH_TLB);
296 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
297 asm volatile("ptesync": : :"memory");
298 __tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
302 static inline void fixup_tlbie_pid_lpid(unsigned long pid, unsigned long lpid)
305 * We can use any address for the invalidation, pick one which is
306 * probably unused as an optimisation.
308 unsigned long va = ((1UL << 52) - 1);
310 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
311 asm volatile("ptesync" : : : "memory");
312 __tlbie_pid_lpid(0, lpid, RIC_FLUSH_TLB);
315 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
316 asm volatile("ptesync" : : : "memory");
317 __tlbie_va_lpid(va, pid, lpid, mmu_get_ap(MMU_PAGE_64K),
322 static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid,
325 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
326 asm volatile("ptesync": : :"memory");
327 __tlbie_lpid_va(va, 0, ap, RIC_FLUSH_TLB);
330 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
331 asm volatile("ptesync": : :"memory");
332 __tlbie_lpid_va(va, lpid, ap, RIC_FLUSH_TLB);
336 static inline void fixup_tlbie_lpid(unsigned long lpid)
339 * We can use any address for the invalidation, pick one which is
340 * probably unused as an optimisation.
342 unsigned long va = ((1UL << 52) - 1);
344 if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
345 asm volatile("ptesync": : :"memory");
346 __tlbie_lpid(0, RIC_FLUSH_TLB);
349 if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
350 asm volatile("ptesync": : :"memory");
351 __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
356 * We use 128 set in radix mode and 256 set in hpt mode.
358 static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
362 asm volatile("ptesync": : :"memory");
367 /* For PWC, only one flush is needed */
368 __tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
369 ppc_after_tlbiel_barrier();
372 __tlbiel_pid(pid, 0, RIC_FLUSH_TLB);
377 * Flush the first set of the TLB, and if
378 * we're doing a RIC_FLUSH_ALL, also flush
379 * the entire Page Walk Cache.
381 __tlbiel_pid(pid, 0, RIC_FLUSH_ALL);
384 if (!cpu_has_feature(CPU_FTR_ARCH_31)) {
385 /* For the remaining sets, just flush the TLB */
386 for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
387 __tlbiel_pid(pid, set, RIC_FLUSH_TLB);
390 ppc_after_tlbiel_barrier();
391 asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory");
394 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
396 asm volatile("ptesync": : :"memory");
399 * Workaround the fact that the "ric" argument to __tlbie_pid
400 * must be a compile-time constraint to match the "i" constraint
401 * in the asm statement.
405 __tlbie_pid(pid, RIC_FLUSH_TLB);
406 fixup_tlbie_pid(pid);
409 __tlbie_pid(pid, RIC_FLUSH_PWC);
413 __tlbie_pid(pid, RIC_FLUSH_ALL);
414 fixup_tlbie_pid(pid);
416 asm volatile("eieio; tlbsync; ptesync": : :"memory");
419 static inline void _tlbie_pid_lpid(unsigned long pid, unsigned long lpid,
422 asm volatile("ptesync" : : : "memory");
425 * Workaround the fact that the "ric" argument to __tlbie_pid
426 * must be a compile-time contraint to match the "i" constraint
427 * in the asm statement.
431 __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB);
432 fixup_tlbie_pid_lpid(pid, lpid);
435 __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC);
439 __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_ALL);
440 fixup_tlbie_pid_lpid(pid, lpid);
442 asm volatile("eieio; tlbsync; ptesync" : : : "memory");
449 static void do_tlbiel_pid(void *info)
451 struct tlbiel_pid *t = info;
453 if (t->ric == RIC_FLUSH_TLB)
454 _tlbiel_pid(t->pid, RIC_FLUSH_TLB);
455 else if (t->ric == RIC_FLUSH_PWC)
456 _tlbiel_pid(t->pid, RIC_FLUSH_PWC);
458 _tlbiel_pid(t->pid, RIC_FLUSH_ALL);
461 static inline void _tlbiel_pid_multicast(struct mm_struct *mm,
462 unsigned long pid, unsigned long ric)
464 struct cpumask *cpus = mm_cpumask(mm);
465 struct tlbiel_pid t = { .pid = pid, .ric = ric };
467 on_each_cpu_mask(cpus, do_tlbiel_pid, &t, 1);
469 * Always want the CPU translations to be invalidated with tlbiel in
470 * these paths, so while coprocessors must use tlbie, we can not
471 * optimise away the tlbiel component.
473 if (atomic_read(&mm->context.copros) > 0)
474 _tlbie_pid(pid, RIC_FLUSH_ALL);
477 static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric)
479 asm volatile("ptesync": : :"memory");
482 * Workaround the fact that the "ric" argument to __tlbie_pid
483 * must be a compile-time contraint to match the "i" constraint
484 * in the asm statement.
488 __tlbie_lpid(lpid, RIC_FLUSH_TLB);
489 fixup_tlbie_lpid(lpid);
492 __tlbie_lpid(lpid, RIC_FLUSH_PWC);
496 __tlbie_lpid(lpid, RIC_FLUSH_ALL);
497 fixup_tlbie_lpid(lpid);
499 asm volatile("eieio; tlbsync; ptesync": : :"memory");
502 static __always_inline void _tlbie_lpid_guest(unsigned long lpid, unsigned long ric)
505 * Workaround the fact that the "ric" argument to __tlbie_pid
506 * must be a compile-time contraint to match the "i" constraint
507 * in the asm statement.
511 __tlbie_lpid_guest(lpid, RIC_FLUSH_TLB);
514 __tlbie_lpid_guest(lpid, RIC_FLUSH_PWC);
518 __tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
520 fixup_tlbie_lpid(lpid);
521 asm volatile("eieio; tlbsync; ptesync": : :"memory");
524 static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
525 unsigned long pid, unsigned long page_size,
529 unsigned long ap = mmu_get_ap(psize);
531 for (addr = start; addr < end; addr += page_size)
532 __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
535 static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid,
536 unsigned long psize, unsigned long ric)
538 unsigned long ap = mmu_get_ap(psize);
540 asm volatile("ptesync": : :"memory");
541 __tlbiel_va(va, pid, ap, ric);
542 ppc_after_tlbiel_barrier();
545 static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
546 unsigned long pid, unsigned long page_size,
547 unsigned long psize, bool also_pwc)
549 asm volatile("ptesync": : :"memory");
551 __tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
552 __tlbiel_va_range(start, end, pid, page_size, psize);
553 ppc_after_tlbiel_barrier();
556 static inline void __tlbie_va_range(unsigned long start, unsigned long end,
557 unsigned long pid, unsigned long page_size,
561 unsigned long ap = mmu_get_ap(psize);
563 for (addr = start; addr < end; addr += page_size)
564 __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
566 fixup_tlbie_va_range(addr - page_size, pid, ap);
569 static inline void __tlbie_va_range_lpid(unsigned long start, unsigned long end,
570 unsigned long pid, unsigned long lpid,
571 unsigned long page_size,
575 unsigned long ap = mmu_get_ap(psize);
577 for (addr = start; addr < end; addr += page_size)
578 __tlbie_va_lpid(addr, pid, lpid, ap, RIC_FLUSH_TLB);
580 fixup_tlbie_va_range_lpid(addr - page_size, pid, lpid, ap);
583 static __always_inline void _tlbie_va(unsigned long va, unsigned long pid,
584 unsigned long psize, unsigned long ric)
586 unsigned long ap = mmu_get_ap(psize);
588 asm volatile("ptesync": : :"memory");
589 __tlbie_va(va, pid, ap, ric);
590 fixup_tlbie_va(va, pid, ap);
591 asm volatile("eieio; tlbsync; ptesync": : :"memory");
601 static void do_tlbiel_va(void *info)
603 struct tlbiel_va *t = info;
605 if (t->ric == RIC_FLUSH_TLB)
606 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_TLB);
607 else if (t->ric == RIC_FLUSH_PWC)
608 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_PWC);
610 _tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_ALL);
613 static inline void _tlbiel_va_multicast(struct mm_struct *mm,
614 unsigned long va, unsigned long pid,
615 unsigned long psize, unsigned long ric)
617 struct cpumask *cpus = mm_cpumask(mm);
618 struct tlbiel_va t = { .va = va, .pid = pid, .psize = psize, .ric = ric };
619 on_each_cpu_mask(cpus, do_tlbiel_va, &t, 1);
620 if (atomic_read(&mm->context.copros) > 0)
621 _tlbie_va(va, pid, psize, RIC_FLUSH_TLB);
624 struct tlbiel_va_range {
628 unsigned long page_size;
633 static void do_tlbiel_va_range(void *info)
635 struct tlbiel_va_range *t = info;
637 _tlbiel_va_range(t->start, t->end, t->pid, t->page_size,
638 t->psize, t->also_pwc);
641 static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid,
642 unsigned long psize, unsigned long ric)
644 unsigned long ap = mmu_get_ap(psize);
646 asm volatile("ptesync": : :"memory");
647 __tlbie_lpid_va(va, lpid, ap, ric);
648 fixup_tlbie_lpid_va(va, lpid, ap);
649 asm volatile("eieio; tlbsync; ptesync": : :"memory");
652 static inline void _tlbie_va_range(unsigned long start, unsigned long end,
653 unsigned long pid, unsigned long page_size,
654 unsigned long psize, bool also_pwc)
656 asm volatile("ptesync": : :"memory");
658 __tlbie_pid(pid, RIC_FLUSH_PWC);
659 __tlbie_va_range(start, end, pid, page_size, psize);
660 asm volatile("eieio; tlbsync; ptesync": : :"memory");
663 static inline void _tlbie_va_range_lpid(unsigned long start, unsigned long end,
664 unsigned long pid, unsigned long lpid,
665 unsigned long page_size,
666 unsigned long psize, bool also_pwc)
668 asm volatile("ptesync" : : : "memory");
670 __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC);
671 __tlbie_va_range_lpid(start, end, pid, lpid, page_size, psize);
672 asm volatile("eieio; tlbsync; ptesync" : : : "memory");
675 static inline void _tlbiel_va_range_multicast(struct mm_struct *mm,
676 unsigned long start, unsigned long end,
677 unsigned long pid, unsigned long page_size,
678 unsigned long psize, bool also_pwc)
680 struct cpumask *cpus = mm_cpumask(mm);
681 struct tlbiel_va_range t = { .start = start, .end = end,
682 .pid = pid, .page_size = page_size,
683 .psize = psize, .also_pwc = also_pwc };
685 on_each_cpu_mask(cpus, do_tlbiel_va_range, &t, 1);
686 if (atomic_read(&mm->context.copros) > 0)
687 _tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
691 * Base TLB flushing operations:
693 * - flush_tlb_mm(mm) flushes the specified mm context TLB's
694 * - flush_tlb_page(vma, vmaddr) flushes one page
695 * - flush_tlb_range(vma, start, end) flushes a range of pages
696 * - flush_tlb_kernel_range(start, end) flushes kernel pages
698 * - local_* variants of page and mm only apply to the current
701 void radix__local_flush_tlb_mm(struct mm_struct *mm)
706 pid = mm->context.id;
707 if (pid != MMU_NO_CONTEXT)
708 _tlbiel_pid(pid, RIC_FLUSH_TLB);
711 EXPORT_SYMBOL(radix__local_flush_tlb_mm);
714 void radix__local_flush_all_mm(struct mm_struct *mm)
719 pid = mm->context.id;
720 if (pid != MMU_NO_CONTEXT)
721 _tlbiel_pid(pid, RIC_FLUSH_ALL);
724 EXPORT_SYMBOL(radix__local_flush_all_mm);
726 static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
728 radix__local_flush_all_mm(mm);
730 #endif /* CONFIG_SMP */
732 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
738 pid = mm->context.id;
739 if (pid != MMU_NO_CONTEXT)
740 _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
744 void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
746 #ifdef CONFIG_HUGETLB_PAGE
747 /* need the return fix for nohash.c */
748 if (is_vm_hugetlb_page(vma))
749 return radix__local_flush_hugetlb_page(vma, vmaddr);
751 radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);
753 EXPORT_SYMBOL(radix__local_flush_tlb_page);
755 static bool mm_needs_flush_escalation(struct mm_struct *mm)
758 * P9 nest MMU has issues with the page walk cache
759 * caching PTEs and not flushing them properly when
760 * RIC = 0 for a PID/LPID invalidate
762 if (atomic_read(&mm->context.copros) > 0)
768 * If always_flush is true, then flush even if this CPU can't be removed
771 void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush)
773 unsigned long pid = mm->context.id;
774 int cpu = smp_processor_id();
777 * A kthread could have done a mmget_not_zero() after the flushing CPU
778 * checked mm_cpumask, and be in the process of kthread_use_mm when
779 * interrupted here. In that case, current->mm will be set to mm,
780 * because kthread_use_mm() setting ->mm and switching to the mm is
781 * done with interrupts off.
783 if (current->mm == mm)
786 if (current->active_mm == mm) {
787 WARN_ON_ONCE(current->mm != NULL);
788 /* Is a kernel thread and is using mm as the lazy tlb */
790 current->active_mm = &init_mm;
791 switch_mm_irqs_off(mm, &init_mm, current);
796 * This IPI may be initiated from any source including those not
797 * running the mm, so there may be a racing IPI that comes after
798 * this one which finds the cpumask already clear. Check and avoid
799 * underflowing the active_cpus count in that case. The race should
800 * not otherwise be a problem, but the TLB must be flushed because
801 * that's what the caller expects.
803 if (cpumask_test_cpu(cpu, mm_cpumask(mm))) {
804 atomic_dec(&mm->context.active_cpus);
805 cpumask_clear_cpu(cpu, mm_cpumask(mm));
811 _tlbiel_pid(pid, RIC_FLUSH_ALL);
815 static void do_exit_flush_lazy_tlb(void *arg)
817 struct mm_struct *mm = arg;
818 exit_lazy_flush_tlb(mm, true);
821 static void exit_flush_lazy_tlbs(struct mm_struct *mm)
824 * Would be nice if this was async so it could be run in
825 * parallel with our local flush, but generic code does not
826 * give a good API for it. Could extend the generic code or
827 * make a special powerpc IPI for flushing TLBs.
828 * For now it's not too performance critical.
830 smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb,
834 #else /* CONFIG_SMP */
835 static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { }
836 #endif /* CONFIG_SMP */
838 static DEFINE_PER_CPU(unsigned int, mm_cpumask_trim_clock);
841 * Interval between flushes at which we send out IPIs to check whether the
842 * mm_cpumask can be trimmed for the case where it's not a single-threaded
843 * process flushing its own mm. The intent is to reduce the cost of later
844 * flushes. Don't want this to be so low that it adds noticable cost to TLB
845 * flushing, or so high that it doesn't help reduce global TLBIEs.
847 static unsigned long tlb_mm_cpumask_trim_timer = 1073;
849 static bool tick_and_test_trim_clock(void)
851 if (__this_cpu_inc_return(mm_cpumask_trim_clock) ==
852 tlb_mm_cpumask_trim_timer) {
853 __this_cpu_write(mm_cpumask_trim_clock, 0);
859 enum tlb_flush_type {
865 static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
867 int active_cpus = atomic_read(&mm->context.active_cpus);
868 int cpu = smp_processor_id();
870 if (active_cpus == 0)
871 return FLUSH_TYPE_NONE;
872 if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm))) {
873 if (current->mm != mm) {
875 * Asynchronous flush sources may trim down to nothing
876 * if the process is not running, so occasionally try
879 if (tick_and_test_trim_clock()) {
880 exit_lazy_flush_tlb(mm, true);
881 return FLUSH_TYPE_NONE;
884 return FLUSH_TYPE_LOCAL;
887 /* Coprocessors require TLBIE to invalidate nMMU. */
888 if (atomic_read(&mm->context.copros) > 0)
889 return FLUSH_TYPE_GLOBAL;
892 * In the fullmm case there's no point doing the exit_flush_lazy_tlbs
893 * because the mm is being taken down anyway, and a TLBIE tends to
894 * be faster than an IPI+TLBIEL.
897 return FLUSH_TYPE_GLOBAL;
900 * If we are running the only thread of a single-threaded process,
901 * then we should almost always be able to trim off the rest of the
902 * CPU mask (except in the case of use_mm() races), so always try
905 if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) {
906 exit_flush_lazy_tlbs(mm);
908 * use_mm() race could prevent IPIs from being able to clear
909 * the cpumask here, however those users are established
910 * after our first check (and so after the PTEs are removed),
911 * and the TLB still gets flushed by the IPI, so this CPU
912 * will only require a local flush.
914 return FLUSH_TYPE_LOCAL;
918 * Occasionally try to trim down the cpumask. It's possible this can
919 * bring the mask to zero, which results in no flush.
921 if (tick_and_test_trim_clock()) {
922 exit_flush_lazy_tlbs(mm);
923 if (current->mm == mm)
924 return FLUSH_TYPE_LOCAL;
925 if (cpumask_test_cpu(cpu, mm_cpumask(mm)))
926 exit_lazy_flush_tlb(mm, true);
927 return FLUSH_TYPE_NONE;
930 return FLUSH_TYPE_GLOBAL;
934 void radix__flush_tlb_mm(struct mm_struct *mm)
937 enum tlb_flush_type type;
939 pid = mm->context.id;
940 if (unlikely(pid == MMU_NO_CONTEXT))
945 * Order loads of mm_cpumask (in flush_type_needed) vs previous
946 * stores to clear ptes before the invalidate. See barrier in
950 type = flush_type_needed(mm, false);
951 if (type == FLUSH_TYPE_LOCAL) {
952 _tlbiel_pid(pid, RIC_FLUSH_TLB);
953 } else if (type == FLUSH_TYPE_GLOBAL) {
954 if (!mmu_has_feature(MMU_FTR_GTSE)) {
955 unsigned long tgt = H_RPTI_TARGET_CMMU;
957 if (atomic_read(&mm->context.copros) > 0)
958 tgt |= H_RPTI_TARGET_NMMU;
959 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
960 H_RPTI_PAGE_ALL, 0, -1UL);
961 } else if (cputlb_use_tlbie()) {
962 if (mm_needs_flush_escalation(mm))
963 _tlbie_pid(pid, RIC_FLUSH_ALL);
965 _tlbie_pid(pid, RIC_FLUSH_TLB);
967 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
972 EXPORT_SYMBOL(radix__flush_tlb_mm);
974 static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
977 enum tlb_flush_type type;
979 pid = mm->context.id;
980 if (unlikely(pid == MMU_NO_CONTEXT))
984 smp_mb(); /* see radix__flush_tlb_mm */
985 type = flush_type_needed(mm, fullmm);
986 if (type == FLUSH_TYPE_LOCAL) {
987 _tlbiel_pid(pid, RIC_FLUSH_ALL);
988 } else if (type == FLUSH_TYPE_GLOBAL) {
989 if (!mmu_has_feature(MMU_FTR_GTSE)) {
990 unsigned long tgt = H_RPTI_TARGET_CMMU;
991 unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
994 if (atomic_read(&mm->context.copros) > 0)
995 tgt |= H_RPTI_TARGET_NMMU;
996 pseries_rpt_invalidate(pid, tgt, type,
997 H_RPTI_PAGE_ALL, 0, -1UL);
998 } else if (cputlb_use_tlbie())
999 _tlbie_pid(pid, RIC_FLUSH_ALL);
1001 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
1006 void radix__flush_all_mm(struct mm_struct *mm)
1008 __flush_all_mm(mm, false);
1010 EXPORT_SYMBOL(radix__flush_all_mm);
1012 void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
1016 enum tlb_flush_type type;
1018 pid = mm->context.id;
1019 if (unlikely(pid == MMU_NO_CONTEXT))
1023 smp_mb(); /* see radix__flush_tlb_mm */
1024 type = flush_type_needed(mm, false);
1025 if (type == FLUSH_TYPE_LOCAL) {
1026 _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
1027 } else if (type == FLUSH_TYPE_GLOBAL) {
1028 if (!mmu_has_feature(MMU_FTR_GTSE)) {
1029 unsigned long tgt, pg_sizes, size;
1031 tgt = H_RPTI_TARGET_CMMU;
1032 pg_sizes = psize_to_rpti_pgsize(psize);
1033 size = 1UL << mmu_psize_to_shift(psize);
1035 if (atomic_read(&mm->context.copros) > 0)
1036 tgt |= H_RPTI_TARGET_NMMU;
1037 pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
1040 } else if (cputlb_use_tlbie())
1041 _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
1043 _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB);
1048 void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
1050 #ifdef CONFIG_HUGETLB_PAGE
1051 if (is_vm_hugetlb_page(vma))
1052 return radix__flush_hugetlb_page(vma, vmaddr);
1054 radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);
1056 EXPORT_SYMBOL(radix__flush_tlb_page);
1058 #endif /* CONFIG_SMP */
1060 static void do_tlbiel_kernel(void *info)
1062 _tlbiel_pid(0, RIC_FLUSH_ALL);
1065 static inline void _tlbiel_kernel_broadcast(void)
1067 on_each_cpu(do_tlbiel_kernel, NULL, 1);
1068 if (tlbie_capable) {
1070 * Coherent accelerators don't refcount kernel memory mappings,
1071 * so have to always issue a tlbie for them. This is quite a
1074 _tlbie_pid(0, RIC_FLUSH_ALL);
1079 * If kernel TLBIs ever become local rather than global, then
1080 * drivers/misc/ocxl/link.c:ocxl_link_add_pe will need some work, as it
1081 * assumes kernel TLBIs are global.
1083 void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
1085 if (!mmu_has_feature(MMU_FTR_GTSE)) {
1086 unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU;
1087 unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
1090 pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL,
1092 } else if (cputlb_use_tlbie())
1093 _tlbie_pid(0, RIC_FLUSH_ALL);
1095 _tlbiel_kernel_broadcast();
1097 EXPORT_SYMBOL(radix__flush_tlb_kernel_range);
1099 #define TLB_FLUSH_ALL -1UL
1102 * Number of pages above which we invalidate the entire PID rather than
1103 * flush individual pages, for local and global flushes respectively.
1105 * tlbie goes out to the interconnect and individual ops are more costly.
1106 * It also does not iterate over sets like the local tlbiel variant when
1107 * invalidating a full PID, so it has a far lower threshold to change from
1108 * individual page flushes to full-pid flushes.
1110 static u32 tlb_single_page_flush_ceiling __read_mostly = 33;
1111 static u32 tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2;
1113 static inline void __radix__flush_tlb_range(struct mm_struct *mm,
1114 unsigned long start, unsigned long end)
1117 unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift;
1118 unsigned long page_size = 1UL << page_shift;
1119 unsigned long nr_pages = (end - start) >> page_shift;
1120 bool fullmm = (end == TLB_FLUSH_ALL);
1121 bool flush_pid, flush_pwc = false;
1122 enum tlb_flush_type type;
1124 pid = mm->context.id;
1125 if (unlikely(pid == MMU_NO_CONTEXT))
1129 smp_mb(); /* see radix__flush_tlb_mm */
1130 type = flush_type_needed(mm, fullmm);
1131 if (type == FLUSH_TYPE_NONE)
1136 else if (type == FLUSH_TYPE_GLOBAL)
1137 flush_pid = nr_pages > tlb_single_page_flush_ceiling;
1139 flush_pid = nr_pages > tlb_local_single_page_flush_ceiling;
1141 * full pid flush already does the PWC flush. if it is not full pid
1142 * flush check the range is more than PMD and force a pwc flush
1143 * mremap() depends on this behaviour.
1145 if (!flush_pid && (end - start) >= PMD_SIZE)
1148 if (!mmu_has_feature(MMU_FTR_GTSE) && type == FLUSH_TYPE_GLOBAL) {
1149 unsigned long type = H_RPTI_TYPE_TLB;
1150 unsigned long tgt = H_RPTI_TARGET_CMMU;
1151 unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);
1153 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1154 pg_sizes |= psize_to_rpti_pgsize(MMU_PAGE_2M);
1155 if (atomic_read(&mm->context.copros) > 0)
1156 tgt |= H_RPTI_TARGET_NMMU;
1158 type |= H_RPTI_TYPE_PWC;
1159 pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end);
1160 } else if (flush_pid) {
1162 * We are now flushing a range larger than PMD size force a RIC_FLUSH_ALL
1164 if (type == FLUSH_TYPE_LOCAL) {
1165 _tlbiel_pid(pid, RIC_FLUSH_ALL);
1167 if (cputlb_use_tlbie()) {
1168 _tlbie_pid(pid, RIC_FLUSH_ALL);
1170 _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
1174 bool hflush = false;
1175 unsigned long hstart, hend;
1177 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
1178 hstart = (start + PMD_SIZE - 1) & PMD_MASK;
1179 hend = end & PMD_MASK;
1184 if (type == FLUSH_TYPE_LOCAL) {
1185 asm volatile("ptesync": : :"memory");
1187 /* For PWC, only one flush is needed */
1188 __tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
1189 __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize);
1191 __tlbiel_va_range(hstart, hend, pid,
1192 PMD_SIZE, MMU_PAGE_2M);
1193 ppc_after_tlbiel_barrier();
1194 } else if (cputlb_use_tlbie()) {
1195 asm volatile("ptesync": : :"memory");
1197 __tlbie_pid(pid, RIC_FLUSH_PWC);
1198 __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize);
1200 __tlbie_va_range(hstart, hend, pid,
1201 PMD_SIZE, MMU_PAGE_2M);
1202 asm volatile("eieio; tlbsync; ptesync": : :"memory");
1204 _tlbiel_va_range_multicast(mm,
1205 start, end, pid, page_size, mmu_virtual_psize, flush_pwc);
1207 _tlbiel_va_range_multicast(mm,
1208 hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, flush_pwc);
1215 void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
1219 #ifdef CONFIG_HUGETLB_PAGE
1220 if (is_vm_hugetlb_page(vma))
1221 return radix__flush_hugetlb_tlb_range(vma, start, end);
1224 __radix__flush_tlb_range(vma->vm_mm, start, end);
1226 EXPORT_SYMBOL(radix__flush_tlb_range);
1228 static int radix_get_mmu_psize(int page_size)
1232 if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift))
1233 psize = mmu_virtual_psize;
1234 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift))
1235 psize = MMU_PAGE_2M;
1236 else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift))
1237 psize = MMU_PAGE_1G;
1244 * Flush partition scoped LPID address translation for all CPUs.
1246 void radix__flush_tlb_lpid_page(unsigned int lpid,
1248 unsigned long page_size)
1250 int psize = radix_get_mmu_psize(page_size);
1252 _tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB);
1254 EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page);
1257 * Flush partition scoped PWC from LPID for all CPUs.
1259 void radix__flush_pwc_lpid(unsigned int lpid)
1261 _tlbie_lpid(lpid, RIC_FLUSH_PWC);
1263 EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid);
1266 * Flush partition scoped translations from LPID (=LPIDR)
1268 void radix__flush_all_lpid(unsigned int lpid)
1270 _tlbie_lpid(lpid, RIC_FLUSH_ALL);
1272 EXPORT_SYMBOL_GPL(radix__flush_all_lpid);
1275 * Flush process scoped translations from LPID (=LPIDR)
1277 void radix__flush_all_lpid_guest(unsigned int lpid)
1279 _tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
1282 void radix__tlb_flush(struct mmu_gather *tlb)
1285 struct mm_struct *mm = tlb->mm;
1286 int page_size = tlb->page_size;
1287 unsigned long start = tlb->start;
1288 unsigned long end = tlb->end;
1291 * if page size is not something we understand, do a full mm flush
1293 * A "fullmm" flush must always do a flush_all_mm (RIC=2) flush
1294 * that flushes the process table entry cache upon process teardown.
1295 * See the comment for radix in arch_exit_mmap().
1297 if (tlb->fullmm || tlb->need_flush_all) {
1298 __flush_all_mm(mm, true);
1299 } else if ( (psize = radix_get_mmu_psize(page_size)) == -1) {
1300 if (!tlb->freed_tables)
1301 radix__flush_tlb_mm(mm);
1303 radix__flush_all_mm(mm);
1305 if (!tlb->freed_tables)
1306 radix__flush_tlb_range_psize(mm, start, end, psize);
1308 radix__flush_tlb_pwc_range_psize(mm, start, end, psize);
1312 static void __radix__flush_tlb_range_psize(struct mm_struct *mm,
1313 unsigned long start, unsigned long end,
1314 int psize, bool also_pwc)
1317 unsigned int page_shift = mmu_psize_defs[psize].shift;
1318 unsigned long page_size = 1UL << page_shift;
1319 unsigned long nr_pages = (end - start) >> page_shift;
1320 bool fullmm = (end == TLB_FLUSH_ALL);
1322 enum tlb_flush_type type;
1324 pid = mm->context.id;
1325 if (unlikely(pid == MMU_NO_CONTEXT))
1328 fullmm = (end == TLB_FLUSH_ALL);
1331 smp_mb(); /* see radix__flush_tlb_mm */
1332 type = flush_type_needed(mm, fullmm);
1333 if (type == FLUSH_TYPE_NONE)
1338 else if (type == FLUSH_TYPE_GLOBAL)
1339 flush_pid = nr_pages > tlb_single_page_flush_ceiling;
1341 flush_pid = nr_pages > tlb_local_single_page_flush_ceiling;
1343 if (!mmu_has_feature(MMU_FTR_GTSE) && type == FLUSH_TYPE_GLOBAL) {
1344 unsigned long tgt = H_RPTI_TARGET_CMMU;
1345 unsigned long type = H_RPTI_TYPE_TLB;
1346 unsigned long pg_sizes = psize_to_rpti_pgsize(psize);
1349 type |= H_RPTI_TYPE_PWC;
1350 if (atomic_read(&mm->context.copros) > 0)
1351 tgt |= H_RPTI_TARGET_NMMU;
1352 pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end);
1353 } else if (flush_pid) {
1354 if (type == FLUSH_TYPE_LOCAL) {
1355 _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
1357 if (cputlb_use_tlbie()) {
1358 if (mm_needs_flush_escalation(mm))
1362 also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
1364 _tlbiel_pid_multicast(mm, pid,
1365 also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
1370 if (type == FLUSH_TYPE_LOCAL)
1371 _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc);
1372 else if (cputlb_use_tlbie())
1373 _tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
1375 _tlbiel_va_range_multicast(mm,
1376 start, end, pid, page_size, psize, also_pwc);
1382 void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
1383 unsigned long end, int psize)
1385 return __radix__flush_tlb_range_psize(mm, start, end, psize, false);
1388 void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
1389 unsigned long end, int psize)
1391 __radix__flush_tlb_range_psize(mm, start, end, psize, true);
1394 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1395 void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
1397 unsigned long pid, end;
1398 enum tlb_flush_type type;
1400 pid = mm->context.id;
1401 if (unlikely(pid == MMU_NO_CONTEXT))
1404 /* 4k page size, just blow the world */
1405 if (PAGE_SIZE == 0x1000) {
1406 radix__flush_all_mm(mm);
1410 end = addr + HPAGE_PMD_SIZE;
1412 /* Otherwise first do the PWC, then iterate the pages. */
1414 smp_mb(); /* see radix__flush_tlb_mm */
1415 type = flush_type_needed(mm, false);
1416 if (type == FLUSH_TYPE_LOCAL) {
1417 _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1418 } else if (type == FLUSH_TYPE_GLOBAL) {
1419 if (!mmu_has_feature(MMU_FTR_GTSE)) {
1420 unsigned long tgt, type, pg_sizes;
1422 tgt = H_RPTI_TARGET_CMMU;
1423 type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
1425 pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize);
1427 if (atomic_read(&mm->context.copros) > 0)
1428 tgt |= H_RPTI_TARGET_NMMU;
1429 pseries_rpt_invalidate(pid, tgt, type, pg_sizes,
1431 } else if (cputlb_use_tlbie())
1432 _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1434 _tlbiel_va_range_multicast(mm,
1435 addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1440 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1442 void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
1443 unsigned long start, unsigned long end)
1445 radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
1447 EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
1449 void radix__flush_tlb_all(void)
1451 unsigned long rb,prs,r,rs;
1452 unsigned long ric = RIC_FLUSH_ALL;
1454 rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */
1455 prs = 0; /* partition scoped */
1456 r = 1; /* radix format */
1457 rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */
1459 asm volatile("ptesync": : :"memory");
1461 * now flush guest entries by passing PRS = 1 and LPID != 0
1463 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
1464 : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory");
1466 * now flush host entires by passing PRS = 0 and LPID == 0
1468 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
1469 : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");
1470 asm volatile("eieio; tlbsync; ptesync": : :"memory");
1473 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1475 * Performs process-scoped invalidations for a given LPID
1476 * as part of H_RPT_INVALIDATE hcall.
1478 void do_h_rpt_invalidate_prt(unsigned long pid, unsigned long lpid,
1479 unsigned long type, unsigned long pg_sizes,
1480 unsigned long start, unsigned long end)
1482 unsigned long psize, nr_pages;
1483 struct mmu_psize_def *def;
1487 * A H_RPTI_TYPE_ALL request implies RIC=3, hence
1488 * do a single IS=1 based flush.
1490 if ((type & H_RPTI_TYPE_ALL) == H_RPTI_TYPE_ALL) {
1491 _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_ALL);
1495 if (type & H_RPTI_TYPE_PWC)
1496 _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC);
1498 /* Full PID flush */
1499 if (start == 0 && end == -1)
1500 return _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB);
1502 /* Do range invalidation for all the valid page sizes */
1503 for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
1504 def = &mmu_psize_defs[psize];
1505 if (!(pg_sizes & def->h_rpt_pgsize))
1508 nr_pages = (end - start) >> def->shift;
1509 flush_pid = nr_pages > tlb_single_page_flush_ceiling;
1512 * If the number of pages spanning the range is above
1513 * the ceiling, convert the request into a full PID flush.
1514 * And since PID flush takes out all the page sizes, there
1515 * is no need to consider remaining page sizes.
1518 _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB);
1521 _tlbie_va_range_lpid(start, end, pid, lpid,
1522 (1UL << def->shift), psize, false);
1525 EXPORT_SYMBOL_GPL(do_h_rpt_invalidate_prt);
1527 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
1529 static int __init create_tlb_single_page_flush_ceiling(void)
1531 debugfs_create_u32("tlb_single_page_flush_ceiling", 0600,
1532 arch_debugfs_dir, &tlb_single_page_flush_ceiling);
1533 debugfs_create_u32("tlb_local_single_page_flush_ceiling", 0600,
1534 arch_debugfs_dir, &tlb_local_single_page_flush_ceiling);
1537 late_initcall(create_tlb_single_page_flush_ceiling);