arch/riscv/mm/tlbflush.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/mm.h>
   4 #include <linux/smp.h>
   5 #include <linux/sched.h>
   6 #include <linux/hugetlb.h>
   7 #include <asm/sbi.h>
   8 #include <asm/mmu_context.h>
   9
  10 static inline void local_flush_tlb_all_asid(unsigned long asid)
  11 {
  12         if (asid != FLUSH_TLB_NO_ASID)
  13                 __asm__ __volatile__ ("sfence.vma x0, %0"
  14                                 :
  15                                 : "r" (asid)
  16                                 : "memory");
  17         else
  18                 local_flush_tlb_all();
  19 }
  20
  21 static inline void local_flush_tlb_page_asid(unsigned long addr,
  22                 unsigned long asid)
  23 {
  24         if (asid != FLUSH_TLB_NO_ASID)
  25                 __asm__ __volatile__ ("sfence.vma %0, %1"
  26                                 :
  27                                 : "r" (addr), "r" (asid)
  28                                 : "memory");
  29         else
  30                 local_flush_tlb_page(addr);
  31 }
  32
  33 /*
  34  * Flush entire TLB if number of entries to be flushed is greater
  35  * than the threshold below.
  36  */
  37 static unsigned long tlb_flush_all_threshold __read_mostly = 64;
  38
  39 static void local_flush_tlb_range_threshold_asid(unsigned long start,
  40                                                  unsigned long size,
  41                                                  unsigned long stride,
  42                                                  unsigned long asid)
  43 {
  44         unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride);
  45         int i;
  46
  47         if (nr_ptes_in_range > tlb_flush_all_threshold) {
  48                 local_flush_tlb_all_asid(asid);
  49                 return;
  50         }
  51
  52         for (i = 0; i < nr_ptes_in_range; ++i) {
  53                 local_flush_tlb_page_asid(start, asid);
  54                 start += stride;
  55         }
  56 }
  57
  58 static inline void local_flush_tlb_range_asid(unsigned long start,
  59                 unsigned long size, unsigned long stride, unsigned long asid)
  60 {
  61         if (size <= stride)
  62                 local_flush_tlb_page_asid(start, asid);
  63         else if (size == FLUSH_TLB_MAX_SIZE)
  64                 local_flush_tlb_all_asid(asid);
  65         else
  66                 local_flush_tlb_range_threshold_asid(start, size, stride, asid);
  67 }
  68
  69 /* Flush a range of kernel pages without broadcasting */
  70 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
  71 {
  72         local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID);
  73 }
  74
  75 static void __ipi_flush_tlb_all(void *info)
  76 {
  77         local_flush_tlb_all();
  78 }
  79
  80 void flush_tlb_all(void)
  81 {
  82         if (riscv_use_ipi_for_rfence())
  83                 on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
  84         else
  85                 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID);
  86 }
  87
  88 struct flush_tlb_range_data {
  89         unsigned long asid;
  90         unsigned long start;
  91         unsigned long size;
  92         unsigned long stride;
  93 };
  94
  95 static void __ipi_flush_tlb_range_asid(void *info)
  96 {
  97         struct flush_tlb_range_data *d = info;
  98
  99         local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 100 }
 101
 102 static void __flush_tlb_range(const struct cpumask *cmask, unsigned long asid,
 103                               unsigned long start, unsigned long size,
 104                               unsigned long stride)
 105 {
 106         struct flush_tlb_range_data ftd;
 107         bool broadcast;
 108
 109         if (cpumask_empty(cmask))
 110                 return;
 111
 112         if (cmask != cpu_online_mask) {
 113                 unsigned int cpuid;
 114
 115                 cpuid = get_cpu();
 116                 /* check if the tlbflush needs to be sent to other CPUs */
 117                 broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;
 118         } else {
 119                 broadcast = true;
 120         }
 121
 122         if (broadcast) {
 123                 if (riscv_use_ipi_for_rfence()) {
 124                         ftd.asid = asid;
 125                         ftd.start = start;
 126                         ftd.size = size;
 127                         ftd.stride = stride;
 128                         on_each_cpu_mask(cmask,
 129                                          __ipi_flush_tlb_range_asid,
 130                                          &ftd, 1);
 131                 } else
 132                         sbi_remote_sfence_vma_asid(cmask,
 133                                                    start, size, asid);
 134         } else {
 135                 local_flush_tlb_range_asid(start, size, stride, asid);
 136         }
 137
 138         if (cmask != cpu_online_mask)
 139                 put_cpu();
 140 }
 141
 142 static inline unsigned long get_mm_asid(struct mm_struct *mm)
 143 {
 144         return static_branch_unlikely(&use_asid_allocator) ?
 145                         atomic_long_read(&mm->context.id) & asid_mask : FLUSH_TLB_NO_ASID;
 146 }
 147
 148 void flush_tlb_mm(struct mm_struct *mm)
 149 {
 150         __flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm),
 151                           0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
 152 }
 153
 154 void flush_tlb_mm_range(struct mm_struct *mm,
 155                         unsigned long start, unsigned long end,
 156                         unsigned int page_size)
 157 {
 158         __flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm),
 159                           start, end - start, page_size);
 160 }
 161
 162 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 163 {
 164         __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm),
 165                           addr, PAGE_SIZE, PAGE_SIZE);
 166 }
 167
 168 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 169                      unsigned long end)
 170 {
 171         unsigned long stride_size;
 172
 173         if (!is_vm_hugetlb_page(vma)) {
 174                 stride_size = PAGE_SIZE;
 175         } else {
 176                 stride_size = huge_page_size(hstate_vma(vma));
 177
 178                 /*
 179                  * As stated in the privileged specification, every PTE in a
 180                  * NAPOT region must be invalidated, so reset the stride in that
 181                  * case.
 182                  */
 183                 if (has_svnapot()) {
 184                         if (stride_size >= PGDIR_SIZE)
 185                                 stride_size = PGDIR_SIZE;
 186                         else if (stride_size >= P4D_SIZE)
 187                                 stride_size = P4D_SIZE;
 188                         else if (stride_size >= PUD_SIZE)
 189                                 stride_size = PUD_SIZE;
 190                         else if (stride_size >= PMD_SIZE)
 191                                 stride_size = PMD_SIZE;
 192                         else
 193                                 stride_size = PAGE_SIZE;
 194                 }
 195         }
 196
 197         __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm),
 198                           start, end - start, stride_size);
 199 }
 200
 201 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 202 {
 203         __flush_tlb_range(cpu_online_mask, FLUSH_TLB_NO_ASID,
 204                           start, end - start, PAGE_SIZE);
 205 }
 206
 207 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 208 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
 209                         unsigned long end)
 210 {
 211         __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm),
 212                           start, end - start, PMD_SIZE);
 213 }
 214 #endif
 215
 216 bool arch_tlbbatch_should_defer(struct mm_struct *mm)
 217 {
 218         return true;
 219 }
 220
 221 void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
 222                                struct mm_struct *mm,
 223                                unsigned long uaddr)
 224 {
 225         cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
 226 }
 227
 228 void arch_flush_tlb_batched_pending(struct mm_struct *mm)
 229 {
 230         flush_tlb_mm(mm);
 231 }
 232
 233 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
 234 {
 235         __flush_tlb_range(&batch->cpumask, FLUSH_TLB_NO_ASID, 0,
 236                           FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
 237         cpumask_clear(&batch->cpumask);
 238 }