arch/powerpc/mm/book3s64/radix_pgtable.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Page table handling routines for radix page table.
   4  *
   5  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
   6  */
   7
   8 #define pr_fmt(fmt) "radix-mmu: " fmt
   9
  10 #include <linux/io.h>
  11 #include <linux/kernel.h>
  12 #include <linux/sched/mm.h>
  13 #include <linux/memblock.h>
  14 #include <linux/of.h>
  15 #include <linux/of_fdt.h>
  16 #include <linux/mm.h>
  17 #include <linux/hugetlb.h>
  18 #include <linux/string_helpers.h>
  19 #include <linux/memory.h>
  20
  21 #include <asm/pgalloc.h>
  22 #include <asm/mmu_context.h>
  23 #include <asm/dma.h>
  24 #include <asm/machdep.h>
  25 #include <asm/mmu.h>
  26 #include <asm/firmware.h>
  27 #include <asm/powernv.h>
  28 #include <asm/sections.h>
  29 #include <asm/smp.h>
  30 #include <asm/trace.h>
  31 #include <asm/uaccess.h>
  32 #include <asm/ultravisor.h>
  33
  34 #include <trace/events/thp.h>
  35
  36 unsigned int mmu_base_pid;
  37 unsigned long radix_mem_block_size __ro_after_init;
  38
  39 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
  40                         unsigned long region_start, unsigned long region_end)
  41 {
  42         phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
  43         phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
  44         void *ptr;
  45
  46         if (region_start)
  47                 min_addr = region_start;
  48         if (region_end)
  49                 max_addr = region_end;
  50
  51         ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
  52
  53         if (!ptr)
  54                 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
  55                       __func__, size, size, nid, &min_addr, &max_addr);
  56
  57         return ptr;
  58 }
  59
  60 /*
  61  * When allocating pud or pmd pointers, we allocate a complete page
  62  * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
  63  * is to ensure that the page obtained from the memblock allocator
  64  * can be completely used as page table page and can be freed
  65  * correctly when the page table entries are removed.
  66  */
  67 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
  68                           pgprot_t flags,
  69                           unsigned int map_page_size,
  70                           int nid,
  71                           unsigned long region_start, unsigned long region_end)
  72 {
  73         unsigned long pfn = pa >> PAGE_SHIFT;
  74         pgd_t *pgdp;
  75         p4d_t *p4dp;
  76         pud_t *pudp;
  77         pmd_t *pmdp;
  78         pte_t *ptep;
  79
  80         pgdp = pgd_offset_k(ea);
  81         p4dp = p4d_offset(pgdp, ea);
  82         if (p4d_none(*p4dp)) {
  83                 pudp = early_alloc_pgtable(PAGE_SIZE, nid,
  84                                            region_start, region_end);
  85                 p4d_populate(&init_mm, p4dp, pudp);
  86         }
  87         pudp = pud_offset(p4dp, ea);
  88         if (map_page_size == PUD_SIZE) {
  89                 ptep = (pte_t *)pudp;
  90                 goto set_the_pte;
  91         }
  92         if (pud_none(*pudp)) {
  93                 pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
  94                                            region_end);
  95                 pud_populate(&init_mm, pudp, pmdp);
  96         }
  97         pmdp = pmd_offset(pudp, ea);
  98         if (map_page_size == PMD_SIZE) {
  99                 ptep = pmdp_ptep(pmdp);
 100                 goto set_the_pte;
 101         }
 102         if (!pmd_present(*pmdp)) {
 103                 ptep = early_alloc_pgtable(PAGE_SIZE, nid,
 104                                                 region_start, region_end);
 105                 pmd_populate_kernel(&init_mm, pmdp, ptep);
 106         }
 107         ptep = pte_offset_kernel(pmdp, ea);
 108
 109 set_the_pte:
 110         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 111         asm volatile("ptesync": : :"memory");
 112         return 0;
 113 }
 114
 115 /*
 116  * nid, region_start, and region_end are hints to try to place the page
 117  * table memory in the same node or region.
 118  */
 119 static int __map_kernel_page(unsigned long ea, unsigned long pa,
 120                           pgprot_t flags,
 121                           unsigned int map_page_size,
 122                           int nid,
 123                           unsigned long region_start, unsigned long region_end)
 124 {
 125         unsigned long pfn = pa >> PAGE_SHIFT;
 126         pgd_t *pgdp;
 127         p4d_t *p4dp;
 128         pud_t *pudp;
 129         pmd_t *pmdp;
 130         pte_t *ptep;
 131         /*
 132          * Make sure task size is correct as per the max adddr
 133          */
 134         BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
 135
 136 #ifdef CONFIG_PPC_64K_PAGES
 137         BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
 138 #endif
 139
 140         if (unlikely(!slab_is_available()))
 141                 return early_map_kernel_page(ea, pa, flags, map_page_size,
 142                                                 nid, region_start, region_end);
 143
 144         /*
 145          * Should make page table allocation functions be able to take a
 146          * node, so we can place kernel page tables on the right nodes after
 147          * boot.
 148          */
 149         pgdp = pgd_offset_k(ea);
 150         p4dp = p4d_offset(pgdp, ea);
 151         pudp = pud_alloc(&init_mm, p4dp, ea);
 152         if (!pudp)
 153                 return -ENOMEM;
 154         if (map_page_size == PUD_SIZE) {
 155                 ptep = (pte_t *)pudp;
 156                 goto set_the_pte;
 157         }
 158         pmdp = pmd_alloc(&init_mm, pudp, ea);
 159         if (!pmdp)
 160                 return -ENOMEM;
 161         if (map_page_size == PMD_SIZE) {
 162                 ptep = pmdp_ptep(pmdp);
 163                 goto set_the_pte;
 164         }
 165         ptep = pte_alloc_kernel(pmdp, ea);
 166         if (!ptep)
 167                 return -ENOMEM;
 168
 169 set_the_pte:
 170         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 171         asm volatile("ptesync": : :"memory");
 172         return 0;
 173 }
 174
 175 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
 176                           pgprot_t flags,
 177                           unsigned int map_page_size)
 178 {
 179         return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
 180 }
 181
 182 #ifdef CONFIG_STRICT_KERNEL_RWX
 183 static void radix__change_memory_range(unsigned long start, unsigned long end,
 184                                        unsigned long clear)
 185 {
 186         unsigned long idx;
 187         pgd_t *pgdp;
 188         p4d_t *p4dp;
 189         pud_t *pudp;
 190         pmd_t *pmdp;
 191         pte_t *ptep;
 192
 193         start = ALIGN_DOWN(start, PAGE_SIZE);
 194         end = PAGE_ALIGN(end); // aligns up
 195
 196         pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
 197                  start, end, clear);
 198
 199         for (idx = start; idx < end; idx += PAGE_SIZE) {
 200                 pgdp = pgd_offset_k(idx);
 201                 p4dp = p4d_offset(pgdp, idx);
 202                 pudp = pud_alloc(&init_mm, p4dp, idx);
 203                 if (!pudp)
 204                         continue;
 205                 if (pud_is_leaf(*pudp)) {
 206                         ptep = (pte_t *)pudp;
 207                         goto update_the_pte;
 208                 }
 209                 pmdp = pmd_alloc(&init_mm, pudp, idx);
 210                 if (!pmdp)
 211                         continue;
 212                 if (pmd_is_leaf(*pmdp)) {
 213                         ptep = pmdp_ptep(pmdp);
 214                         goto update_the_pte;
 215                 }
 216                 ptep = pte_alloc_kernel(pmdp, idx);
 217                 if (!ptep)
 218                         continue;
 219 update_the_pte:
 220                 radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
 221         }
 222
 223         radix__flush_tlb_kernel_range(start, end);
 224 }
 225
 226 void radix__mark_rodata_ro(void)
 227 {
 228         unsigned long start, end;
 229
 230         start = (unsigned long)_stext;
 231         end = (unsigned long)__init_begin;
 232
 233         radix__change_memory_range(start, end, _PAGE_WRITE);
 234 }
 235
 236 void radix__mark_initmem_nx(void)
 237 {
 238         unsigned long start = (unsigned long)__init_begin;
 239         unsigned long end = (unsigned long)__init_end;
 240
 241         radix__change_memory_range(start, end, _PAGE_EXEC);
 242 }
 243 #endif /* CONFIG_STRICT_KERNEL_RWX */
 244
 245 static inline void __meminit
 246 print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
 247 {
 248         char buf[10];
 249
 250         if (end <= start)
 251                 return;
 252
 253         string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
 254
 255         pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
 256                 exec ? " (exec)" : "");
 257 }
 258
 259 static unsigned long next_boundary(unsigned long addr, unsigned long end)
 260 {
 261 #ifdef CONFIG_STRICT_KERNEL_RWX
 262         if (addr < __pa_symbol(__init_begin))
 263                 return __pa_symbol(__init_begin);
 264 #endif
 265         return end;
 266 }
 267
 268 static int __meminit create_physical_mapping(unsigned long start,
 269                                              unsigned long end,
 270                                              unsigned long max_mapping_size,
 271                                              int nid, pgprot_t _prot)
 272 {
 273         unsigned long vaddr, addr, mapping_size = 0;
 274         bool prev_exec, exec = false;
 275         pgprot_t prot;
 276         int psize;
 277
 278         start = ALIGN(start, PAGE_SIZE);
 279         end   = ALIGN_DOWN(end, PAGE_SIZE);
 280         for (addr = start; addr < end; addr += mapping_size) {
 281                 unsigned long gap, previous_size;
 282                 int rc;
 283
 284                 gap = next_boundary(addr, end) - addr;
 285                 if (gap > max_mapping_size)
 286                         gap = max_mapping_size;
 287                 previous_size = mapping_size;
 288                 prev_exec = exec;
 289
 290                 if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
 291                     mmu_psize_defs[MMU_PAGE_1G].shift) {
 292                         mapping_size = PUD_SIZE;
 293                         psize = MMU_PAGE_1G;
 294                 } else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
 295                            mmu_psize_defs[MMU_PAGE_2M].shift) {
 296                         mapping_size = PMD_SIZE;
 297                         psize = MMU_PAGE_2M;
 298                 } else {
 299                         mapping_size = PAGE_SIZE;
 300                         psize = mmu_virtual_psize;
 301                 }
 302
 303                 vaddr = (unsigned long)__va(addr);
 304
 305                 if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
 306                     overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
 307                         prot = PAGE_KERNEL_X;
 308                         exec = true;
 309                 } else {
 310                         prot = _prot;
 311                         exec = false;
 312                 }
 313
 314                 if (mapping_size != previous_size || exec != prev_exec) {
 315                         print_mapping(start, addr, previous_size, prev_exec);
 316                         start = addr;
 317                 }
 318
 319                 rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
 320                 if (rc)
 321                         return rc;
 322
 323                 update_page_count(psize, 1);
 324         }
 325
 326         print_mapping(start, addr, mapping_size, exec);
 327         return 0;
 328 }
 329
 330 static void __init radix_init_pgtable(void)
 331 {
 332         unsigned long rts_field;
 333         phys_addr_t start, end;
 334         u64 i;
 335
 336         /* We don't support slb for radix */
 337         slb_set_size(0);
 338
 339         /*
 340          * Create the linear mapping
 341          */
 342         for_each_mem_range(i, &start, &end) {
 343                 /*
 344                  * The memblock allocator  is up at this point, so the
 345                  * page tables will be allocated within the range. No
 346                  * need or a node (which we don't have yet).
 347                  */
 348
 349                 if (end >= RADIX_VMALLOC_START) {
 350                         pr_warn("Outside the supported range\n");
 351                         continue;
 352                 }
 353
 354                 WARN_ON(create_physical_mapping(start, end,
 355                                                 radix_mem_block_size,
 356                                                 -1, PAGE_KERNEL));
 357         }
 358
 359         if (!cpu_has_feature(CPU_FTR_HVMODE) &&
 360                         cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
 361                 /*
 362                  * Older versions of KVM on these machines prefer if the
 363                  * guest only uses the low 19 PID bits.
 364                  */
 365                 mmu_pid_bits = 19;
 366         }
 367         mmu_base_pid = 1;
 368
 369         /*
 370          * Allocate Partition table and process table for the
 371          * host.
 372          */
 373         BUG_ON(PRTB_SIZE_SHIFT > 36);
 374         process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
 375         /*
 376          * Fill in the process table.
 377          */
 378         rts_field = radix__get_tree_size();
 379         process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
 380
 381         /*
 382          * The init_mm context is given the first available (non-zero) PID,
 383          * which is the "guard PID" and contains no page table. PIDR should
 384          * never be set to zero because that duplicates the kernel address
 385          * space at the 0x0... offset (quadrant 0)!
 386          *
 387          * An arbitrary PID that may later be allocated by the PID allocator
 388          * for userspace processes must not be used either, because that
 389          * would cause stale user mappings for that PID on CPUs outside of
 390          * the TLB invalidation scheme (because it won't be in mm_cpumask).
 391          *
 392          * So permanently carve out one PID for the purpose of a guard PID.
 393          */
 394         init_mm.context.id = mmu_base_pid;
 395         mmu_base_pid++;
 396 }
 397
 398 static void __init radix_init_partition_table(void)
 399 {
 400         unsigned long rts_field, dw0, dw1;
 401
 402         mmu_partition_table_init();
 403         rts_field = radix__get_tree_size();
 404         dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
 405         dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
 406         mmu_partition_table_set_entry(0, dw0, dw1, false);
 407
 408         pr_info("Initializing Radix MMU\n");
 409 }
 410
 411 static int __init get_idx_from_shift(unsigned int shift)
 412 {
 413         int idx = -1;
 414
 415         switch (shift) {
 416         case 0xc:
 417                 idx = MMU_PAGE_4K;
 418                 break;
 419         case 0x10:
 420                 idx = MMU_PAGE_64K;
 421                 break;
 422         case 0x15:
 423                 idx = MMU_PAGE_2M;
 424                 break;
 425         case 0x1e:
 426                 idx = MMU_PAGE_1G;
 427                 break;
 428         }
 429         return idx;
 430 }
 431
 432 static int __init radix_dt_scan_page_sizes(unsigned long node,
 433                                            const char *uname, int depth,
 434                                            void *data)
 435 {
 436         int size = 0;
 437         int shift, idx;
 438         unsigned int ap;
 439         const __be32 *prop;
 440         const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 441
 442         /* We are scanning "cpu" nodes only */
 443         if (type == NULL || strcmp(type, "cpu") != 0)
 444                 return 0;
 445
 446         /* Grab page size encodings */
 447         prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
 448         if (!prop)
 449                 return 0;
 450
 451         pr_info("Page sizes from device-tree:\n");
 452         for (; size >= 4; size -= 4, ++prop) {
 453
 454                 struct mmu_psize_def *def;
 455
 456                 /* top 3 bit is AP encoding */
 457                 shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
 458                 ap = be32_to_cpu(prop[0]) >> 29;
 459                 pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
 460
 461                 idx = get_idx_from_shift(shift);
 462                 if (idx < 0)
 463                         continue;
 464
 465                 def = &mmu_psize_defs[idx];
 466                 def->shift = shift;
 467                 def->ap  = ap;
 468                 def->h_rpt_pgsize = psize_to_rpti_pgsize(idx);
 469         }
 470
 471         /* needed ? */
 472         cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
 473         return 1;
 474 }
 475
 476 #ifdef CONFIG_MEMORY_HOTPLUG
 477 static int __init probe_memory_block_size(unsigned long node, const char *uname, int
 478                                           depth, void *data)
 479 {
 480         unsigned long *mem_block_size = (unsigned long *)data;
 481         const __be32 *prop;
 482         int len;
 483
 484         if (depth != 1)
 485                 return 0;
 486
 487         if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
 488                 return 0;
 489
 490         prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
 491
 492         if (!prop || len < dt_root_size_cells * sizeof(__be32))
 493                 /*
 494                  * Nothing in the device tree
 495                  */
 496                 *mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 497         else
 498                 *mem_block_size = of_read_number(prop, dt_root_size_cells);
 499         return 1;
 500 }
 501
 502 static unsigned long __init radix_memory_block_size(void)
 503 {
 504         unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 505
 506         /*
 507          * OPAL firmware feature is set by now. Hence we are ok
 508          * to test OPAL feature.
 509          */
 510         if (firmware_has_feature(FW_FEATURE_OPAL))
 511                 mem_block_size = 1UL * 1024 * 1024 * 1024;
 512         else
 513                 of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
 514
 515         return mem_block_size;
 516 }
 517
 518 #else   /* CONFIG_MEMORY_HOTPLUG */
 519
 520 static unsigned long __init radix_memory_block_size(void)
 521 {
 522         return 1UL * 1024 * 1024 * 1024;
 523 }
 524
 525 #endif /* CONFIG_MEMORY_HOTPLUG */
 526
 527
 528 void __init radix__early_init_devtree(void)
 529 {
 530         int rc;
 531
 532         /*
 533          * Try to find the available page sizes in the device-tree
 534          */
 535         rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
 536         if (!rc) {
 537                 /*
 538                  * No page size details found in device tree.
 539                  * Let's assume we have page 4k and 64k support
 540                  */
 541                 mmu_psize_defs[MMU_PAGE_4K].shift = 12;
 542                 mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
 543                 mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize =
 544                         psize_to_rpti_pgsize(MMU_PAGE_4K);
 545
 546                 mmu_psize_defs[MMU_PAGE_64K].shift = 16;
 547                 mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
 548                 mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize =
 549                         psize_to_rpti_pgsize(MMU_PAGE_64K);
 550         }
 551
 552         /*
 553          * Max mapping size used when mapping pages. We don't use
 554          * ppc_md.memory_block_size() here because this get called
 555          * early and we don't have machine probe called yet. Also
 556          * the pseries implementation only check for ibm,lmb-size.
 557          * All hypervisor supporting radix do expose that device
 558          * tree node.
 559          */
 560         radix_mem_block_size = radix_memory_block_size();
 561         return;
 562 }
 563
 564 void __init radix__early_init_mmu(void)
 565 {
 566         unsigned long lpcr;
 567
 568 #ifdef CONFIG_PPC_64S_HASH_MMU
 569 #ifdef CONFIG_PPC_64K_PAGES
 570         /* PAGE_SIZE mappings */
 571         mmu_virtual_psize = MMU_PAGE_64K;
 572 #else
 573         mmu_virtual_psize = MMU_PAGE_4K;
 574 #endif
 575
 576 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 577         /* vmemmap mapping */
 578         if (mmu_psize_defs[MMU_PAGE_2M].shift) {
 579                 /*
 580                  * map vmemmap using 2M if available
 581                  */
 582                 mmu_vmemmap_psize = MMU_PAGE_2M;
 583         } else
 584                 mmu_vmemmap_psize = mmu_virtual_psize;
 585 #endif
 586 #endif
 587         /*
 588          * initialize page table size
 589          */
 590         __pte_index_size = RADIX_PTE_INDEX_SIZE;
 591         __pmd_index_size = RADIX_PMD_INDEX_SIZE;
 592         __pud_index_size = RADIX_PUD_INDEX_SIZE;
 593         __pgd_index_size = RADIX_PGD_INDEX_SIZE;
 594         __pud_cache_index = RADIX_PUD_INDEX_SIZE;
 595         __pte_table_size = RADIX_PTE_TABLE_SIZE;
 596         __pmd_table_size = RADIX_PMD_TABLE_SIZE;
 597         __pud_table_size = RADIX_PUD_TABLE_SIZE;
 598         __pgd_table_size = RADIX_PGD_TABLE_SIZE;
 599
 600         __pmd_val_bits = RADIX_PMD_VAL_BITS;
 601         __pud_val_bits = RADIX_PUD_VAL_BITS;
 602         __pgd_val_bits = RADIX_PGD_VAL_BITS;
 603
 604         __kernel_virt_start = RADIX_KERN_VIRT_START;
 605         __vmalloc_start = RADIX_VMALLOC_START;
 606         __vmalloc_end = RADIX_VMALLOC_END;
 607         __kernel_io_start = RADIX_KERN_IO_START;
 608         __kernel_io_end = RADIX_KERN_IO_END;
 609         vmemmap = (struct page *)RADIX_VMEMMAP_START;
 610         ioremap_bot = IOREMAP_BASE;
 611
 612 #ifdef CONFIG_PCI
 613         pci_io_base = ISA_IO_BASE;
 614 #endif
 615         __pte_frag_nr = RADIX_PTE_FRAG_NR;
 616         __pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
 617         __pmd_frag_nr = RADIX_PMD_FRAG_NR;
 618         __pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
 619
 620         radix_init_pgtable();
 621
 622         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 623                 lpcr = mfspr(SPRN_LPCR);
 624                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 625                 radix_init_partition_table();
 626         } else {
 627                 radix_init_pseries();
 628         }
 629
 630         memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
 631
 632         /* Switch to the guard PID before turning on MMU */
 633         radix__switch_mmu_context(NULL, &init_mm);
 634         tlbiel_all();
 635 }
 636
 637 void radix__early_init_mmu_secondary(void)
 638 {
 639         unsigned long lpcr;
 640         /*
 641          * update partition table control register and UPRT
 642          */
 643         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 644                 lpcr = mfspr(SPRN_LPCR);
 645                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 646
 647                 set_ptcr_when_no_uv(__pa(partition_tb) |
 648                                     (PATB_SIZE_SHIFT - 12));
 649         }
 650
 651         radix__switch_mmu_context(NULL, &init_mm);
 652         tlbiel_all();
 653
 654         /* Make sure userspace can't change the AMR */
 655         mtspr(SPRN_UAMOR, 0);
 656 }
 657
 658 /* Called during kexec sequence with MMU off */
 659 notrace void radix__mmu_cleanup_all(void)
 660 {
 661         unsigned long lpcr;
 662
 663         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 664                 lpcr = mfspr(SPRN_LPCR);
 665                 mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
 666                 set_ptcr_when_no_uv(0);
 667                 powernv_set_nmmu_ptcr(0);
 668                 radix__flush_tlb_all();
 669         }
 670 }
 671
 672 #ifdef CONFIG_MEMORY_HOTPLUG
 673 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
 674 {
 675         pte_t *pte;
 676         int i;
 677
 678         for (i = 0; i < PTRS_PER_PTE; i++) {
 679                 pte = pte_start + i;
 680                 if (!pte_none(*pte))
 681                         return;
 682         }
 683
 684         pte_free_kernel(&init_mm, pte_start);
 685         pmd_clear(pmd);
 686 }
 687
 688 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
 689 {
 690         pmd_t *pmd;
 691         int i;
 692
 693         for (i = 0; i < PTRS_PER_PMD; i++) {
 694                 pmd = pmd_start + i;
 695                 if (!pmd_none(*pmd))
 696                         return;
 697         }
 698
 699         pmd_free(&init_mm, pmd_start);
 700         pud_clear(pud);
 701 }
 702
 703 static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
 704 {
 705         pud_t *pud;
 706         int i;
 707
 708         for (i = 0; i < PTRS_PER_PUD; i++) {
 709                 pud = pud_start + i;
 710                 if (!pud_none(*pud))
 711                         return;
 712         }
 713
 714         pud_free(&init_mm, pud_start);
 715         p4d_clear(p4d);
 716 }
 717
 718 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
 719                              unsigned long end)
 720 {
 721         unsigned long next;
 722         pte_t *pte;
 723
 724         pte = pte_start + pte_index(addr);
 725         for (; addr < end; addr = next, pte++) {
 726                 next = (addr + PAGE_SIZE) & PAGE_MASK;
 727                 if (next > end)
 728                         next = end;
 729
 730                 if (!pte_present(*pte))
 731                         continue;
 732
 733                 if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
 734                         /*
 735                          * The vmemmap_free() and remove_section_mapping()
 736                          * codepaths call us with aligned addresses.
 737                          */
 738                         WARN_ONCE(1, "%s: unaligned range\n", __func__);
 739                         continue;
 740                 }
 741
 742                 pte_clear(&init_mm, addr, pte);
 743         }
 744 }
 745
 746 static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
 747                              unsigned long end)
 748 {
 749         unsigned long next;
 750         pte_t *pte_base;
 751         pmd_t *pmd;
 752
 753         pmd = pmd_start + pmd_index(addr);
 754         for (; addr < end; addr = next, pmd++) {
 755                 next = pmd_addr_end(addr, end);
 756
 757                 if (!pmd_present(*pmd))
 758                         continue;
 759
 760                 if (pmd_is_leaf(*pmd)) {
 761                         if (!IS_ALIGNED(addr, PMD_SIZE) ||
 762                             !IS_ALIGNED(next, PMD_SIZE)) {
 763                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 764                                 continue;
 765                         }
 766                         pte_clear(&init_mm, addr, (pte_t *)pmd);
 767                         continue;
 768                 }
 769
 770                 pte_base = (pte_t *)pmd_page_vaddr(*pmd);
 771                 remove_pte_table(pte_base, addr, next);
 772                 free_pte_table(pte_base, pmd);
 773         }
 774 }
 775
 776 static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
 777                              unsigned long end)
 778 {
 779         unsigned long next;
 780         pmd_t *pmd_base;
 781         pud_t *pud;
 782
 783         pud = pud_start + pud_index(addr);
 784         for (; addr < end; addr = next, pud++) {
 785                 next = pud_addr_end(addr, end);
 786
 787                 if (!pud_present(*pud))
 788                         continue;
 789
 790                 if (pud_is_leaf(*pud)) {
 791                         if (!IS_ALIGNED(addr, PUD_SIZE) ||
 792                             !IS_ALIGNED(next, PUD_SIZE)) {
 793                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 794                                 continue;
 795                         }
 796                         pte_clear(&init_mm, addr, (pte_t *)pud);
 797                         continue;
 798                 }
 799
 800                 pmd_base = pud_pgtable(*pud);
 801                 remove_pmd_table(pmd_base, addr, next);
 802                 free_pmd_table(pmd_base, pud);
 803         }
 804 }
 805
 806 static void __meminit remove_pagetable(unsigned long start, unsigned long end)
 807 {
 808         unsigned long addr, next;
 809         pud_t *pud_base;
 810         pgd_t *pgd;
 811         p4d_t *p4d;
 812
 813         spin_lock(&init_mm.page_table_lock);
 814
 815         for (addr = start; addr < end; addr = next) {
 816                 next = pgd_addr_end(addr, end);
 817
 818                 pgd = pgd_offset_k(addr);
 819                 p4d = p4d_offset(pgd, addr);
 820                 if (!p4d_present(*p4d))
 821                         continue;
 822
 823                 if (p4d_is_leaf(*p4d)) {
 824                         if (!IS_ALIGNED(addr, P4D_SIZE) ||
 825                             !IS_ALIGNED(next, P4D_SIZE)) {
 826                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 827                                 continue;
 828                         }
 829
 830                         pte_clear(&init_mm, addr, (pte_t *)pgd);
 831                         continue;
 832                 }
 833
 834                 pud_base = p4d_pgtable(*p4d);
 835                 remove_pud_table(pud_base, addr, next);
 836                 free_pud_table(pud_base, p4d);
 837         }
 838
 839         spin_unlock(&init_mm.page_table_lock);
 840         radix__flush_tlb_kernel_range(start, end);
 841 }
 842
 843 int __meminit radix__create_section_mapping(unsigned long start,
 844                                             unsigned long end, int nid,
 845                                             pgprot_t prot)
 846 {
 847         if (end >= RADIX_VMALLOC_START) {
 848                 pr_warn("Outside the supported range\n");
 849                 return -1;
 850         }
 851
 852         return create_physical_mapping(__pa(start), __pa(end),
 853                                        radix_mem_block_size, nid, prot);
 854 }
 855
 856 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
 857 {
 858         remove_pagetable(start, end);
 859         return 0;
 860 }
 861 #endif /* CONFIG_MEMORY_HOTPLUG */
 862
 863 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 864 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
 865                                  pgprot_t flags, unsigned int map_page_size,
 866                                  int nid)
 867 {
 868         return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
 869 }
 870
 871 int __meminit radix__vmemmap_create_mapping(unsigned long start,
 872                                       unsigned long page_size,
 873                                       unsigned long phys)
 874 {
 875         /* Create a PTE encoding */
 876         unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
 877         int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
 878         int ret;
 879
 880         if ((start + page_size) >= RADIX_VMEMMAP_END) {
 881                 pr_warn("Outside the supported range\n");
 882                 return -1;
 883         }
 884
 885         ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
 886         BUG_ON(ret);
 887
 888         return 0;
 889 }
 890
 891 #ifdef CONFIG_MEMORY_HOTPLUG
 892 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
 893 {
 894         remove_pagetable(start, start + page_size);
 895 }
 896 #endif
 897 #endif
 898
 899 #ifdef CONFIG_DEBUG_PAGEALLOC
 900 void radix__kernel_map_pages(struct page *page, int numpages, int enable)
 901 {
 902         pr_warn_once("DEBUG_PAGEALLOC not supported in radix mode\n");
 903 }
 904 #endif
 905
 906 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 907
 908 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
 909                                   pmd_t *pmdp, unsigned long clr,
 910                                   unsigned long set)
 911 {
 912         unsigned long old;
 913
 914 #ifdef CONFIG_DEBUG_VM
 915         WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
 916         assert_spin_locked(pmd_lockptr(mm, pmdp));
 917 #endif
 918
 919         old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
 920         trace_hugepage_update(addr, old, clr, set);
 921
 922         return old;
 923 }
 924
 925 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
 926                         pmd_t *pmdp)
 927
 928 {
 929         pmd_t pmd;
 930
 931         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 932         VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
 933         VM_BUG_ON(pmd_devmap(*pmdp));
 934         /*
 935          * khugepaged calls this for normal pmd
 936          */
 937         pmd = *pmdp;
 938         pmd_clear(pmdp);
 939
 940         /*
 941          * pmdp collapse_flush need to ensure that there are no parallel gup
 942          * walk after this call. This is needed so that we can have stable
 943          * page ref count when collapsing a page. We don't allow a collapse page
 944          * if we have gup taken on the page. We can ensure that by sending IPI
 945          * because gup walk happens with IRQ disabled.
 946          */
 947         serialize_against_pte_lookup(vma->vm_mm);
 948
 949         radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
 950
 951         return pmd;
 952 }
 953
 954 /*
 955  * For us pgtable_t is pte_t *. Inorder to save the deposisted
 956  * page table, we consider the allocated page table as a list
 957  * head. On withdraw we need to make sure we zero out the used
 958  * list_head memory area.
 959  */
 960 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 961                                  pgtable_t pgtable)
 962 {
 963         struct list_head *lh = (struct list_head *) pgtable;
 964
 965         assert_spin_locked(pmd_lockptr(mm, pmdp));
 966
 967         /* FIFO */
 968         if (!pmd_huge_pte(mm, pmdp))
 969                 INIT_LIST_HEAD(lh);
 970         else
 971                 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
 972         pmd_huge_pte(mm, pmdp) = pgtable;
 973 }
 974
 975 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 976 {
 977         pte_t *ptep;
 978         pgtable_t pgtable;
 979         struct list_head *lh;
 980
 981         assert_spin_locked(pmd_lockptr(mm, pmdp));
 982
 983         /* FIFO */
 984         pgtable = pmd_huge_pte(mm, pmdp);
 985         lh = (struct list_head *) pgtable;
 986         if (list_empty(lh))
 987                 pmd_huge_pte(mm, pmdp) = NULL;
 988         else {
 989                 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
 990                 list_del(lh);
 991         }
 992         ptep = (pte_t *) pgtable;
 993         *ptep = __pte(0);
 994         ptep++;
 995         *ptep = __pte(0);
 996         return pgtable;
 997 }
 998
 999 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1000                                      unsigned long addr, pmd_t *pmdp)
1001 {
1002         pmd_t old_pmd;
1003         unsigned long old;
1004
1005         old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1006         old_pmd = __pmd(old);
1007         return old_pmd;
1008 }
1009
1010 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1011
1012 void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1013                                   pte_t entry, unsigned long address, int psize)
1014 {
1015         struct mm_struct *mm = vma->vm_mm;
1016         unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
1017                                               _PAGE_RW | _PAGE_EXEC);
1018
1019         unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1020         /*
1021          * To avoid NMMU hang while relaxing access, we need mark
1022          * the pte invalid in between.
1023          */
1024         if ((change & _PAGE_RW) && atomic_read(&mm->context.copros) > 0) {
1025                 unsigned long old_pte, new_pte;
1026
1027                 old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1028                 /*
1029                  * new value of pte
1030                  */
1031                 new_pte = old_pte | set;
1032                 radix__flush_tlb_page_psize(mm, address, psize);
1033                 __radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1034         } else {
1035                 __radix_pte_update(ptep, 0, set);
1036                 /*
1037                  * Book3S does not require a TLB flush when relaxing access
1038                  * restrictions when the address space is not attached to a
1039                  * NMMU, because the core MMU will reload the pte after taking
1040                  * an access fault, which is defined by the architecture.
1041                  */
1042         }
1043         /* See ptesync comment in radix__set_pte_at */
1044 }
1045
1046 void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
1047                                     unsigned long addr, pte_t *ptep,
1048                                     pte_t old_pte, pte_t pte)
1049 {
1050         struct mm_struct *mm = vma->vm_mm;
1051
1052         /*
1053          * To avoid NMMU hang while relaxing access we need to flush the tlb before
1054          * we set the new value. We need to do this only for radix, because hash
1055          * translation does flush when updating the linux pte.
1056          */
1057         if (is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
1058             (atomic_read(&mm->context.copros) > 0))
1059                 radix__flush_tlb_page(vma, addr);
1060
1061         set_pte_at(mm, addr, ptep, pte);
1062 }
1063
1064 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1065 {
1066         pte_t *ptep = (pte_t *)pud;
1067         pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
1068
1069         if (!radix_enabled())
1070                 return 0;
1071
1072         set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
1073
1074         return 1;
1075 }
1076
1077 int pud_clear_huge(pud_t *pud)
1078 {
1079         if (pud_is_leaf(*pud)) {
1080                 pud_clear(pud);
1081                 return 1;
1082         }
1083
1084         return 0;
1085 }
1086
1087 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1088 {
1089         pmd_t *pmd;
1090         int i;
1091
1092         pmd = pud_pgtable(*pud);
1093         pud_clear(pud);
1094
1095         flush_tlb_kernel_range(addr, addr + PUD_SIZE);
1096
1097         for (i = 0; i < PTRS_PER_PMD; i++) {
1098                 if (!pmd_none(pmd[i])) {
1099                         pte_t *pte;
1100                         pte = (pte_t *)pmd_page_vaddr(pmd[i]);
1101
1102                         pte_free_kernel(&init_mm, pte);
1103                 }
1104         }
1105
1106         pmd_free(&init_mm, pmd);
1107
1108         return 1;
1109 }
1110
1111 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1112 {
1113         pte_t *ptep = (pte_t *)pmd;
1114         pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
1115
1116         if (!radix_enabled())
1117                 return 0;
1118
1119         set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
1120
1121         return 1;
1122 }
1123
1124 int pmd_clear_huge(pmd_t *pmd)
1125 {
1126         if (pmd_is_leaf(*pmd)) {
1127                 pmd_clear(pmd);
1128                 return 1;
1129         }
1130
1131         return 0;
1132 }
1133
1134 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1135 {
1136         pte_t *pte;
1137
1138         pte = (pte_t *)pmd_page_vaddr(*pmd);
1139         pmd_clear(pmd);
1140
1141         flush_tlb_kernel_range(addr, addr + PMD_SIZE);
1142
1143         pte_free_kernel(&init_mm, pte);
1144
1145         return 1;
1146 }