powerpc/mm/book3s64/radix_pgtable.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Page table handling routines for radix page table.
   4  *
   5  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
   6  */
   7
   8 #define pr_fmt(fmt) "radix-mmu: " fmt
   9
  10 #include <linux/io.h>
  11 #include <linux/kernel.h>
  12 #include <linux/sched/mm.h>
  13 #include <linux/memblock.h>
  14 #include <linux/of.h>
  15 #include <linux/of_fdt.h>
  16 #include <linux/mm.h>
  17 #include <linux/hugetlb.h>
  18 #include <linux/string_helpers.h>
  19 #include <linux/memory.h>
  20
  21 #include <asm/pgalloc.h>
  22 #include <asm/mmu_context.h>
  23 #include <asm/dma.h>
  24 #include <asm/machdep.h>
  25 #include <asm/mmu.h>
  26 #include <asm/firmware.h>
  27 #include <asm/powernv.h>
  28 #include <asm/sections.h>
  29 #include <asm/smp.h>
  30 #include <asm/trace.h>
  31 #include <asm/uaccess.h>
  32 #include <asm/ultravisor.h>
  33 #include <asm/set_memory.h>
  34
  35 #include <trace/events/thp.h>
  36
  37 #include <mm/mmu_decl.h>
  38
  39 unsigned int mmu_base_pid;
  40 unsigned long radix_mem_block_size __ro_after_init;
  41
  42 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
  43                         unsigned long region_start, unsigned long region_end)
  44 {
  45         phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
  46         phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
  47         void *ptr;
  48
  49         if (region_start)
  50                 min_addr = region_start;
  51         if (region_end)
  52                 max_addr = region_end;
  53
  54         ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
  55
  56         if (!ptr)
  57                 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
  58                       __func__, size, size, nid, &min_addr, &max_addr);
  59
  60         return ptr;
  61 }
  62
  63 /*
  64  * When allocating pud or pmd pointers, we allocate a complete page
  65  * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
  66  * is to ensure that the page obtained from the memblock allocator
  67  * can be completely used as page table page and can be freed
  68  * correctly when the page table entries are removed.
  69  */
  70 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
  71                           pgprot_t flags,
  72                           unsigned int map_page_size,
  73                           int nid,
  74                           unsigned long region_start, unsigned long region_end)
  75 {
  76         unsigned long pfn = pa >> PAGE_SHIFT;
  77         pgd_t *pgdp;
  78         p4d_t *p4dp;
  79         pud_t *pudp;
  80         pmd_t *pmdp;
  81         pte_t *ptep;
  82
  83         pgdp = pgd_offset_k(ea);
  84         p4dp = p4d_offset(pgdp, ea);
  85         if (p4d_none(*p4dp)) {
  86                 pudp = early_alloc_pgtable(PAGE_SIZE, nid,
  87                                            region_start, region_end);
  88                 p4d_populate(&init_mm, p4dp, pudp);
  89         }
  90         pudp = pud_offset(p4dp, ea);
  91         if (map_page_size == PUD_SIZE) {
  92                 ptep = (pte_t *)pudp;
  93                 goto set_the_pte;
  94         }
  95         if (pud_none(*pudp)) {
  96                 pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
  97                                            region_end);
  98                 pud_populate(&init_mm, pudp, pmdp);
  99         }
 100         pmdp = pmd_offset(pudp, ea);
 101         if (map_page_size == PMD_SIZE) {
 102                 ptep = pmdp_ptep(pmdp);
 103                 goto set_the_pte;
 104         }
 105         if (!pmd_present(*pmdp)) {
 106                 ptep = early_alloc_pgtable(PAGE_SIZE, nid,
 107                                                 region_start, region_end);
 108                 pmd_populate_kernel(&init_mm, pmdp, ptep);
 109         }
 110         ptep = pte_offset_kernel(pmdp, ea);
 111
 112 set_the_pte:
 113         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 114         asm volatile("ptesync": : :"memory");
 115         return 0;
 116 }
 117
 118 /*
 119  * nid, region_start, and region_end are hints to try to place the page
 120  * table memory in the same node or region.
 121  */
 122 static int __map_kernel_page(unsigned long ea, unsigned long pa,
 123                           pgprot_t flags,
 124                           unsigned int map_page_size,
 125                           int nid,
 126                           unsigned long region_start, unsigned long region_end)
 127 {
 128         unsigned long pfn = pa >> PAGE_SHIFT;
 129         pgd_t *pgdp;
 130         p4d_t *p4dp;
 131         pud_t *pudp;
 132         pmd_t *pmdp;
 133         pte_t *ptep;
 134         /*
 135          * Make sure task size is correct as per the max adddr
 136          */
 137         BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
 138
 139 #ifdef CONFIG_PPC_64K_PAGES
 140         BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
 141 #endif
 142
 143         if (unlikely(!slab_is_available()))
 144                 return early_map_kernel_page(ea, pa, flags, map_page_size,
 145                                                 nid, region_start, region_end);
 146
 147         /*
 148          * Should make page table allocation functions be able to take a
 149          * node, so we can place kernel page tables on the right nodes after
 150          * boot.
 151          */
 152         pgdp = pgd_offset_k(ea);
 153         p4dp = p4d_offset(pgdp, ea);
 154         pudp = pud_alloc(&init_mm, p4dp, ea);
 155         if (!pudp)
 156                 return -ENOMEM;
 157         if (map_page_size == PUD_SIZE) {
 158                 ptep = (pte_t *)pudp;
 159                 goto set_the_pte;
 160         }
 161         pmdp = pmd_alloc(&init_mm, pudp, ea);
 162         if (!pmdp)
 163                 return -ENOMEM;
 164         if (map_page_size == PMD_SIZE) {
 165                 ptep = pmdp_ptep(pmdp);
 166                 goto set_the_pte;
 167         }
 168         ptep = pte_alloc_kernel(pmdp, ea);
 169         if (!ptep)
 170                 return -ENOMEM;
 171
 172 set_the_pte:
 173         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 174         asm volatile("ptesync": : :"memory");
 175         return 0;
 176 }
 177
 178 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
 179                           pgprot_t flags,
 180                           unsigned int map_page_size)
 181 {
 182         return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
 183 }
 184
 185 #ifdef CONFIG_STRICT_KERNEL_RWX
 186 static void radix__change_memory_range(unsigned long start, unsigned long end,
 187                                        unsigned long clear)
 188 {
 189         unsigned long idx;
 190         pgd_t *pgdp;
 191         p4d_t *p4dp;
 192         pud_t *pudp;
 193         pmd_t *pmdp;
 194         pte_t *ptep;
 195
 196         start = ALIGN_DOWN(start, PAGE_SIZE);
 197         end = PAGE_ALIGN(end); // aligns up
 198
 199         pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
 200                  start, end, clear);
 201
 202         for (idx = start; idx < end; idx += PAGE_SIZE) {
 203                 pgdp = pgd_offset_k(idx);
 204                 p4dp = p4d_offset(pgdp, idx);
 205                 pudp = pud_alloc(&init_mm, p4dp, idx);
 206                 if (!pudp)
 207                         continue;
 208                 if (pud_is_leaf(*pudp)) {
 209                         ptep = (pte_t *)pudp;
 210                         goto update_the_pte;
 211                 }
 212                 pmdp = pmd_alloc(&init_mm, pudp, idx);
 213                 if (!pmdp)
 214                         continue;
 215                 if (pmd_is_leaf(*pmdp)) {
 216                         ptep = pmdp_ptep(pmdp);
 217                         goto update_the_pte;
 218                 }
 219                 ptep = pte_alloc_kernel(pmdp, idx);
 220                 if (!ptep)
 221                         continue;
 222 update_the_pte:
 223                 radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
 224         }
 225
 226         radix__flush_tlb_kernel_range(start, end);
 227 }
 228
 229 void radix__mark_rodata_ro(void)
 230 {
 231         unsigned long start, end;
 232
 233         start = (unsigned long)_stext;
 234         end = (unsigned long)__end_rodata;
 235
 236         radix__change_memory_range(start, end, _PAGE_WRITE);
 237
 238         for (start = PAGE_OFFSET; start < (unsigned long)_stext; start += PAGE_SIZE) {
 239                 end = start + PAGE_SIZE;
 240                 if (overlaps_interrupt_vector_text(start, end))
 241                         radix__change_memory_range(start, end, _PAGE_WRITE);
 242                 else
 243                         break;
 244         }
 245 }
 246
 247 void radix__mark_initmem_nx(void)
 248 {
 249         unsigned long start = (unsigned long)__init_begin;
 250         unsigned long end = (unsigned long)__init_end;
 251
 252         radix__change_memory_range(start, end, _PAGE_EXEC);
 253 }
 254 #endif /* CONFIG_STRICT_KERNEL_RWX */
 255
 256 static inline void __meminit
 257 print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
 258 {
 259         char buf[10];
 260
 261         if (end <= start)
 262                 return;
 263
 264         string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
 265
 266         pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
 267                 exec ? " (exec)" : "");
 268 }
 269
 270 static unsigned long next_boundary(unsigned long addr, unsigned long end)
 271 {
 272 #ifdef CONFIG_STRICT_KERNEL_RWX
 273         unsigned long stext_phys;
 274
 275         stext_phys = __pa_symbol(_stext);
 276
 277         // Relocatable kernel running at non-zero real address
 278         if (stext_phys != 0) {
 279                 // The end of interrupts code at zero is a rodata boundary
 280                 unsigned long end_intr = __pa_symbol(__end_interrupts) - stext_phys;
 281                 if (addr < end_intr)
 282                         return end_intr;
 283
 284                 // Start of relocated kernel text is a rodata boundary
 285                 if (addr < stext_phys)
 286                         return stext_phys;
 287         }
 288
 289         if (addr < __pa_symbol(__srwx_boundary))
 290                 return __pa_symbol(__srwx_boundary);
 291 #endif
 292         return end;
 293 }
 294
 295 static int __meminit create_physical_mapping(unsigned long start,
 296                                              unsigned long end,
 297                                              int nid, pgprot_t _prot)
 298 {
 299         unsigned long vaddr, addr, mapping_size = 0;
 300         bool prev_exec, exec = false;
 301         pgprot_t prot;
 302         int psize;
 303         unsigned long max_mapping_size = radix_mem_block_size;
 304
 305         if (debug_pagealloc_enabled_or_kfence())
 306                 max_mapping_size = PAGE_SIZE;
 307
 308         start = ALIGN(start, PAGE_SIZE);
 309         end   = ALIGN_DOWN(end, PAGE_SIZE);
 310         for (addr = start; addr < end; addr += mapping_size) {
 311                 unsigned long gap, previous_size;
 312                 int rc;
 313
 314                 gap = next_boundary(addr, end) - addr;
 315                 if (gap > max_mapping_size)
 316                         gap = max_mapping_size;
 317                 previous_size = mapping_size;
 318                 prev_exec = exec;
 319
 320                 if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
 321                     mmu_psize_defs[MMU_PAGE_1G].shift) {
 322                         mapping_size = PUD_SIZE;
 323                         psize = MMU_PAGE_1G;
 324                 } else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
 325                            mmu_psize_defs[MMU_PAGE_2M].shift) {
 326                         mapping_size = PMD_SIZE;
 327                         psize = MMU_PAGE_2M;
 328                 } else {
 329                         mapping_size = PAGE_SIZE;
 330                         psize = mmu_virtual_psize;
 331                 }
 332
 333                 vaddr = (unsigned long)__va(addr);
 334
 335                 if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
 336                     overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
 337                         prot = PAGE_KERNEL_X;
 338                         exec = true;
 339                 } else {
 340                         prot = _prot;
 341                         exec = false;
 342                 }
 343
 344                 if (mapping_size != previous_size || exec != prev_exec) {
 345                         print_mapping(start, addr, previous_size, prev_exec);
 346                         start = addr;
 347                 }
 348
 349                 rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
 350                 if (rc)
 351                         return rc;
 352
 353                 update_page_count(psize, 1);
 354         }
 355
 356         print_mapping(start, addr, mapping_size, exec);
 357         return 0;
 358 }
 359
 360 static void __init radix_init_pgtable(void)
 361 {
 362         unsigned long rts_field;
 363         phys_addr_t start, end;
 364         u64 i;
 365
 366         /* We don't support slb for radix */
 367         slb_set_size(0);
 368
 369         /*
 370          * Create the linear mapping
 371          */
 372         for_each_mem_range(i, &start, &end) {
 373                 /*
 374                  * The memblock allocator  is up at this point, so the
 375                  * page tables will be allocated within the range. No
 376                  * need or a node (which we don't have yet).
 377                  */
 378
 379                 if (end >= RADIX_VMALLOC_START) {
 380                         pr_warn("Outside the supported range\n");
 381                         continue;
 382                 }
 383
 384                 WARN_ON(create_physical_mapping(start, end,
 385                                                 -1, PAGE_KERNEL));
 386         }
 387
 388         if (!cpu_has_feature(CPU_FTR_HVMODE) &&
 389                         cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
 390                 /*
 391                  * Older versions of KVM on these machines prefer if the
 392                  * guest only uses the low 19 PID bits.
 393                  */
 394                 mmu_pid_bits = 19;
 395         }
 396         mmu_base_pid = 1;
 397
 398         /*
 399          * Allocate Partition table and process table for the
 400          * host.
 401          */
 402         BUG_ON(PRTB_SIZE_SHIFT > 36);
 403         process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
 404         /*
 405          * Fill in the process table.
 406          */
 407         rts_field = radix__get_tree_size();
 408         process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
 409
 410         /*
 411          * The init_mm context is given the first available (non-zero) PID,
 412          * which is the "guard PID" and contains no page table. PIDR should
 413          * never be set to zero because that duplicates the kernel address
 414          * space at the 0x0... offset (quadrant 0)!
 415          *
 416          * An arbitrary PID that may later be allocated by the PID allocator
 417          * for userspace processes must not be used either, because that
 418          * would cause stale user mappings for that PID on CPUs outside of
 419          * the TLB invalidation scheme (because it won't be in mm_cpumask).
 420          *
 421          * So permanently carve out one PID for the purpose of a guard PID.
 422          */
 423         init_mm.context.id = mmu_base_pid;
 424         mmu_base_pid++;
 425 }
 426
 427 static void __init radix_init_partition_table(void)
 428 {
 429         unsigned long rts_field, dw0, dw1;
 430
 431         mmu_partition_table_init();
 432         rts_field = radix__get_tree_size();
 433         dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
 434         dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
 435         mmu_partition_table_set_entry(0, dw0, dw1, false);
 436
 437         pr_info("Initializing Radix MMU\n");
 438 }
 439
 440 static int __init get_idx_from_shift(unsigned int shift)
 441 {
 442         int idx = -1;
 443
 444         switch (shift) {
 445         case 0xc:
 446                 idx = MMU_PAGE_4K;
 447                 break;
 448         case 0x10:
 449                 idx = MMU_PAGE_64K;
 450                 break;
 451         case 0x15:
 452                 idx = MMU_PAGE_2M;
 453                 break;
 454         case 0x1e:
 455                 idx = MMU_PAGE_1G;
 456                 break;
 457         }
 458         return idx;
 459 }
 460
 461 static int __init radix_dt_scan_page_sizes(unsigned long node,
 462                                            const char *uname, int depth,
 463                                            void *data)
 464 {
 465         int size = 0;
 466         int shift, idx;
 467         unsigned int ap;
 468         const __be32 *prop;
 469         const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 470
 471         /* We are scanning "cpu" nodes only */
 472         if (type == NULL || strcmp(type, "cpu") != 0)
 473                 return 0;
 474
 475         /* Grab page size encodings */
 476         prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
 477         if (!prop)
 478                 return 0;
 479
 480         pr_info("Page sizes from device-tree:\n");
 481         for (; size >= 4; size -= 4, ++prop) {
 482
 483                 struct mmu_psize_def *def;
 484
 485                 /* top 3 bit is AP encoding */
 486                 shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
 487                 ap = be32_to_cpu(prop[0]) >> 29;
 488                 pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
 489
 490                 idx = get_idx_from_shift(shift);
 491                 if (idx < 0)
 492                         continue;
 493
 494                 def = &mmu_psize_defs[idx];
 495                 def->shift = shift;
 496                 def->ap  = ap;
 497                 def->h_rpt_pgsize = psize_to_rpti_pgsize(idx);
 498         }
 499
 500         /* needed ? */
 501         cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
 502         return 1;
 503 }
 504
 505 #ifdef CONFIG_MEMORY_HOTPLUG
 506 static int __init probe_memory_block_size(unsigned long node, const char *uname, int
 507                                           depth, void *data)
 508 {
 509         unsigned long *mem_block_size = (unsigned long *)data;
 510         const __be32 *prop;
 511         int len;
 512
 513         if (depth != 1)
 514                 return 0;
 515
 516         if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
 517                 return 0;
 518
 519         prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
 520
 521         if (!prop || len < dt_root_size_cells * sizeof(__be32))
 522                 /*
 523                  * Nothing in the device tree
 524                  */
 525                 *mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 526         else
 527                 *mem_block_size = of_read_number(prop, dt_root_size_cells);
 528         return 1;
 529 }
 530
 531 static unsigned long __init radix_memory_block_size(void)
 532 {
 533         unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 534
 535         /*
 536          * OPAL firmware feature is set by now. Hence we are ok
 537          * to test OPAL feature.
 538          */
 539         if (firmware_has_feature(FW_FEATURE_OPAL))
 540                 mem_block_size = 1UL * 1024 * 1024 * 1024;
 541         else
 542                 of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
 543
 544         return mem_block_size;
 545 }
 546
 547 #else   /* CONFIG_MEMORY_HOTPLUG */
 548
 549 static unsigned long __init radix_memory_block_size(void)
 550 {
 551         return 1UL * 1024 * 1024 * 1024;
 552 }
 553
 554 #endif /* CONFIG_MEMORY_HOTPLUG */
 555
 556
 557 void __init radix__early_init_devtree(void)
 558 {
 559         int rc;
 560
 561         /*
 562          * Try to find the available page sizes in the device-tree
 563          */
 564         rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
 565         if (!rc) {
 566                 /*
 567                  * No page size details found in device tree.
 568                  * Let's assume we have page 4k and 64k support
 569                  */
 570                 mmu_psize_defs[MMU_PAGE_4K].shift = 12;
 571                 mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
 572                 mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize =
 573                         psize_to_rpti_pgsize(MMU_PAGE_4K);
 574
 575                 mmu_psize_defs[MMU_PAGE_64K].shift = 16;
 576                 mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
 577                 mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize =
 578                         psize_to_rpti_pgsize(MMU_PAGE_64K);
 579         }
 580
 581         /*
 582          * Max mapping size used when mapping pages. We don't use
 583          * ppc_md.memory_block_size() here because this get called
 584          * early and we don't have machine probe called yet. Also
 585          * the pseries implementation only check for ibm,lmb-size.
 586          * All hypervisor supporting radix do expose that device
 587          * tree node.
 588          */
 589         radix_mem_block_size = radix_memory_block_size();
 590         return;
 591 }
 592
 593 void __init radix__early_init_mmu(void)
 594 {
 595         unsigned long lpcr;
 596
 597 #ifdef CONFIG_PPC_64S_HASH_MMU
 598 #ifdef CONFIG_PPC_64K_PAGES
 599         /* PAGE_SIZE mappings */
 600         mmu_virtual_psize = MMU_PAGE_64K;
 601 #else
 602         mmu_virtual_psize = MMU_PAGE_4K;
 603 #endif
 604
 605 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 606         /* vmemmap mapping */
 607         if (mmu_psize_defs[MMU_PAGE_2M].shift) {
 608                 /*
 609                  * map vmemmap using 2M if available
 610                  */
 611                 mmu_vmemmap_psize = MMU_PAGE_2M;
 612         } else
 613                 mmu_vmemmap_psize = mmu_virtual_psize;
 614 #endif
 615 #endif
 616         /*
 617          * initialize page table size
 618          */
 619         __pte_index_size = RADIX_PTE_INDEX_SIZE;
 620         __pmd_index_size = RADIX_PMD_INDEX_SIZE;
 621         __pud_index_size = RADIX_PUD_INDEX_SIZE;
 622         __pgd_index_size = RADIX_PGD_INDEX_SIZE;
 623         __pud_cache_index = RADIX_PUD_INDEX_SIZE;
 624         __pte_table_size = RADIX_PTE_TABLE_SIZE;
 625         __pmd_table_size = RADIX_PMD_TABLE_SIZE;
 626         __pud_table_size = RADIX_PUD_TABLE_SIZE;
 627         __pgd_table_size = RADIX_PGD_TABLE_SIZE;
 628
 629         __pmd_val_bits = RADIX_PMD_VAL_BITS;
 630         __pud_val_bits = RADIX_PUD_VAL_BITS;
 631         __pgd_val_bits = RADIX_PGD_VAL_BITS;
 632
 633         __kernel_virt_start = RADIX_KERN_VIRT_START;
 634         __vmalloc_start = RADIX_VMALLOC_START;
 635         __vmalloc_end = RADIX_VMALLOC_END;
 636         __kernel_io_start = RADIX_KERN_IO_START;
 637         __kernel_io_end = RADIX_KERN_IO_END;
 638         vmemmap = (struct page *)RADIX_VMEMMAP_START;
 639         ioremap_bot = IOREMAP_BASE;
 640
 641 #ifdef CONFIG_PCI
 642         pci_io_base = ISA_IO_BASE;
 643 #endif
 644         __pte_frag_nr = RADIX_PTE_FRAG_NR;
 645         __pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
 646         __pmd_frag_nr = RADIX_PMD_FRAG_NR;
 647         __pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
 648
 649         radix_init_pgtable();
 650
 651         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 652                 lpcr = mfspr(SPRN_LPCR);
 653                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 654                 radix_init_partition_table();
 655         } else {
 656                 radix_init_pseries();
 657         }
 658
 659         memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
 660
 661         /* Switch to the guard PID before turning on MMU */
 662         radix__switch_mmu_context(NULL, &init_mm);
 663         tlbiel_all();
 664 }
 665
 666 void radix__early_init_mmu_secondary(void)
 667 {
 668         unsigned long lpcr;
 669         /*
 670          * update partition table control register and UPRT
 671          */
 672         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 673                 lpcr = mfspr(SPRN_LPCR);
 674                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 675
 676                 set_ptcr_when_no_uv(__pa(partition_tb) |
 677                                     (PATB_SIZE_SHIFT - 12));
 678         }
 679
 680         radix__switch_mmu_context(NULL, &init_mm);
 681         tlbiel_all();
 682
 683         /* Make sure userspace can't change the AMR */
 684         mtspr(SPRN_UAMOR, 0);
 685 }
 686
 687 /* Called during kexec sequence with MMU off */
 688 notrace void radix__mmu_cleanup_all(void)
 689 {
 690         unsigned long lpcr;
 691
 692         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 693                 lpcr = mfspr(SPRN_LPCR);
 694                 mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
 695                 set_ptcr_when_no_uv(0);
 696                 powernv_set_nmmu_ptcr(0);
 697                 radix__flush_tlb_all();
 698         }
 699 }
 700
 701 #ifdef CONFIG_MEMORY_HOTPLUG
 702 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
 703 {
 704         pte_t *pte;
 705         int i;
 706
 707         for (i = 0; i < PTRS_PER_PTE; i++) {
 708                 pte = pte_start + i;
 709                 if (!pte_none(*pte))
 710                         return;
 711         }
 712
 713         pte_free_kernel(&init_mm, pte_start);
 714         pmd_clear(pmd);
 715 }
 716
 717 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
 718 {
 719         pmd_t *pmd;
 720         int i;
 721
 722         for (i = 0; i < PTRS_PER_PMD; i++) {
 723                 pmd = pmd_start + i;
 724                 if (!pmd_none(*pmd))
 725                         return;
 726         }
 727
 728         pmd_free(&init_mm, pmd_start);
 729         pud_clear(pud);
 730 }
 731
 732 static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
 733 {
 734         pud_t *pud;
 735         int i;
 736
 737         for (i = 0; i < PTRS_PER_PUD; i++) {
 738                 pud = pud_start + i;
 739                 if (!pud_none(*pud))
 740                         return;
 741         }
 742
 743         pud_free(&init_mm, pud_start);
 744         p4d_clear(p4d);
 745 }
 746
 747 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
 748                              unsigned long end, bool direct)
 749 {
 750         unsigned long next, pages = 0;
 751         pte_t *pte;
 752
 753         pte = pte_start + pte_index(addr);
 754         for (; addr < end; addr = next, pte++) {
 755                 next = (addr + PAGE_SIZE) & PAGE_MASK;
 756                 if (next > end)
 757                         next = end;
 758
 759                 if (!pte_present(*pte))
 760                         continue;
 761
 762                 if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
 763                         /*
 764                          * The vmemmap_free() and remove_section_mapping()
 765                          * codepaths call us with aligned addresses.
 766                          */
 767                         WARN_ONCE(1, "%s: unaligned range\n", __func__);
 768                         continue;
 769                 }
 770
 771                 pte_clear(&init_mm, addr, pte);
 772                 pages++;
 773         }
 774         if (direct)
 775                 update_page_count(mmu_virtual_psize, -pages);
 776 }
 777
 778 static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
 779                                        unsigned long end, bool direct)
 780 {
 781         unsigned long next, pages = 0;
 782         pte_t *pte_base;
 783         pmd_t *pmd;
 784
 785         pmd = pmd_start + pmd_index(addr);
 786         for (; addr < end; addr = next, pmd++) {
 787                 next = pmd_addr_end(addr, end);
 788
 789                 if (!pmd_present(*pmd))
 790                         continue;
 791
 792                 if (pmd_is_leaf(*pmd)) {
 793                         if (!IS_ALIGNED(addr, PMD_SIZE) ||
 794                             !IS_ALIGNED(next, PMD_SIZE)) {
 795                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 796                                 continue;
 797                         }
 798                         pte_clear(&init_mm, addr, (pte_t *)pmd);
 799                         pages++;
 800                         continue;
 801                 }
 802
 803                 pte_base = (pte_t *)pmd_page_vaddr(*pmd);
 804                 remove_pte_table(pte_base, addr, next, direct);
 805                 free_pte_table(pte_base, pmd);
 806         }
 807         if (direct)
 808                 update_page_count(MMU_PAGE_2M, -pages);
 809 }
 810
 811 static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
 812                                        unsigned long end, bool direct)
 813 {
 814         unsigned long next, pages = 0;
 815         pmd_t *pmd_base;
 816         pud_t *pud;
 817
 818         pud = pud_start + pud_index(addr);
 819         for (; addr < end; addr = next, pud++) {
 820                 next = pud_addr_end(addr, end);
 821
 822                 if (!pud_present(*pud))
 823                         continue;
 824
 825                 if (pud_is_leaf(*pud)) {
 826                         if (!IS_ALIGNED(addr, PUD_SIZE) ||
 827                             !IS_ALIGNED(next, PUD_SIZE)) {
 828                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 829                                 continue;
 830                         }
 831                         pte_clear(&init_mm, addr, (pte_t *)pud);
 832                         pages++;
 833                         continue;
 834                 }
 835
 836                 pmd_base = pud_pgtable(*pud);
 837                 remove_pmd_table(pmd_base, addr, next, direct);
 838                 free_pmd_table(pmd_base, pud);
 839         }
 840         if (direct)
 841                 update_page_count(MMU_PAGE_1G, -pages);
 842 }
 843
 844 static void __meminit remove_pagetable(unsigned long start, unsigned long end,
 845                                        bool direct)
 846 {
 847         unsigned long addr, next;
 848         pud_t *pud_base;
 849         pgd_t *pgd;
 850         p4d_t *p4d;
 851
 852         spin_lock(&init_mm.page_table_lock);
 853
 854         for (addr = start; addr < end; addr = next) {
 855                 next = pgd_addr_end(addr, end);
 856
 857                 pgd = pgd_offset_k(addr);
 858                 p4d = p4d_offset(pgd, addr);
 859                 if (!p4d_present(*p4d))
 860                         continue;
 861
 862                 if (p4d_is_leaf(*p4d)) {
 863                         if (!IS_ALIGNED(addr, P4D_SIZE) ||
 864                             !IS_ALIGNED(next, P4D_SIZE)) {
 865                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 866                                 continue;
 867                         }
 868
 869                         pte_clear(&init_mm, addr, (pte_t *)pgd);
 870                         continue;
 871                 }
 872
 873                 pud_base = p4d_pgtable(*p4d);
 874                 remove_pud_table(pud_base, addr, next, direct);
 875                 free_pud_table(pud_base, p4d);
 876         }
 877
 878         spin_unlock(&init_mm.page_table_lock);
 879         radix__flush_tlb_kernel_range(start, end);
 880 }
 881
 882 int __meminit radix__create_section_mapping(unsigned long start,
 883                                             unsigned long end, int nid,
 884                                             pgprot_t prot)
 885 {
 886         if (end >= RADIX_VMALLOC_START) {
 887                 pr_warn("Outside the supported range\n");
 888                 return -1;
 889         }
 890
 891         return create_physical_mapping(__pa(start), __pa(end),
 892                                        nid, prot);
 893 }
 894
 895 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
 896 {
 897         remove_pagetable(start, end, true);
 898         return 0;
 899 }
 900 #endif /* CONFIG_MEMORY_HOTPLUG */
 901
 902 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 903 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
 904                                  pgprot_t flags, unsigned int map_page_size,
 905                                  int nid)
 906 {
 907         return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
 908 }
 909
 910 int __meminit radix__vmemmap_create_mapping(unsigned long start,
 911                                       unsigned long page_size,
 912                                       unsigned long phys)
 913 {
 914         /* Create a PTE encoding */
 915         unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
 916         int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
 917         int ret;
 918
 919         if ((start + page_size) >= RADIX_VMEMMAP_END) {
 920                 pr_warn("Outside the supported range\n");
 921                 return -1;
 922         }
 923
 924         ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
 925         BUG_ON(ret);
 926
 927         return 0;
 928 }
 929
 930 #ifdef CONFIG_MEMORY_HOTPLUG
 931 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
 932 {
 933         remove_pagetable(start, start + page_size, false);
 934 }
 935 #endif
 936 #endif
 937
 938 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
 939 void radix__kernel_map_pages(struct page *page, int numpages, int enable)
 940 {
 941         unsigned long addr;
 942
 943         addr = (unsigned long)page_address(page);
 944
 945         if (enable)
 946                 set_memory_p(addr, numpages);
 947         else
 948                 set_memory_np(addr, numpages);
 949 }
 950 #endif
 951
 952 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 953
 954 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
 955                                   pmd_t *pmdp, unsigned long clr,
 956                                   unsigned long set)
 957 {
 958         unsigned long old;
 959
 960 #ifdef CONFIG_DEBUG_VM
 961         WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
 962         assert_spin_locked(pmd_lockptr(mm, pmdp));
 963 #endif
 964
 965         old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
 966         trace_hugepage_update(addr, old, clr, set);
 967
 968         return old;
 969 }
 970
 971 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
 972                         pmd_t *pmdp)
 973
 974 {
 975         pmd_t pmd;
 976
 977         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 978         VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
 979         VM_BUG_ON(pmd_devmap(*pmdp));
 980         /*
 981          * khugepaged calls this for normal pmd
 982          */
 983         pmd = *pmdp;
 984         pmd_clear(pmdp);
 985
 986         radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
 987
 988         return pmd;
 989 }
 990
 991 /*
 992  * For us pgtable_t is pte_t *. Inorder to save the deposisted
 993  * page table, we consider the allocated page table as a list
 994  * head. On withdraw we need to make sure we zero out the used
 995  * list_head memory area.
 996  */
 997 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 998                                  pgtable_t pgtable)
 999 {
1000         struct list_head *lh = (struct list_head *) pgtable;
1001
1002         assert_spin_locked(pmd_lockptr(mm, pmdp));
1003
1004         /* FIFO */
1005         if (!pmd_huge_pte(mm, pmdp))
1006                 INIT_LIST_HEAD(lh);
1007         else
1008                 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
1009         pmd_huge_pte(mm, pmdp) = pgtable;
1010 }
1011
1012 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
1013 {
1014         pte_t *ptep;
1015         pgtable_t pgtable;
1016         struct list_head *lh;
1017
1018         assert_spin_locked(pmd_lockptr(mm, pmdp));
1019
1020         /* FIFO */
1021         pgtable = pmd_huge_pte(mm, pmdp);
1022         lh = (struct list_head *) pgtable;
1023         if (list_empty(lh))
1024                 pmd_huge_pte(mm, pmdp) = NULL;
1025         else {
1026                 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
1027                 list_del(lh);
1028         }
1029         ptep = (pte_t *) pgtable;
1030         *ptep = __pte(0);
1031         ptep++;
1032         *ptep = __pte(0);
1033         return pgtable;
1034 }
1035
1036 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1037                                      unsigned long addr, pmd_t *pmdp)
1038 {
1039         pmd_t old_pmd;
1040         unsigned long old;
1041
1042         old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1043         old_pmd = __pmd(old);
1044         return old_pmd;
1045 }
1046
1047 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1048
1049 void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1050                                   pte_t entry, unsigned long address, int psize)
1051 {
1052         struct mm_struct *mm = vma->vm_mm;
1053         unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_SOFT_DIRTY |
1054                                               _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
1055
1056         unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1057         /*
1058          * On POWER9, the NMMU is not able to relax PTE access permissions
1059          * for a translation with a TLB. The PTE must be invalidated, TLB
1060          * flushed before the new PTE is installed.
1061          *
1062          * This only needs to be done for radix, because hash translation does
1063          * flush when updating the linux pte (and we don't support NMMU
1064          * accelerators on HPT on POWER9 anyway XXX: do we?).
1065          *
1066          * POWER10 (and P9P) NMMU does behave as per ISA.
1067          */
1068         if (!cpu_has_feature(CPU_FTR_ARCH_31) && (change & _PAGE_RW) &&
1069             atomic_read(&mm->context.copros) > 0) {
1070                 unsigned long old_pte, new_pte;
1071
1072                 old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1073                 new_pte = old_pte | set;
1074                 radix__flush_tlb_page_psize(mm, address, psize);
1075                 __radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1076         } else {
1077                 __radix_pte_update(ptep, 0, set);
1078                 /*
1079                  * Book3S does not require a TLB flush when relaxing access
1080                  * restrictions when the address space (modulo the POWER9 nest
1081                  * MMU issue above) because the MMU will reload the PTE after
1082                  * taking an access fault, as defined by the architecture. See
1083                  * "Setting a Reference or Change Bit or Upgrading Access
1084                  *  Authority (PTE Subject to Atomic Hardware Updates)" in
1085                  *  Power ISA Version 3.1B.
1086                  */
1087         }
1088         /* See ptesync comment in radix__set_pte_at */
1089 }
1090
1091 void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
1092                                     unsigned long addr, pte_t *ptep,
1093                                     pte_t old_pte, pte_t pte)
1094 {
1095         struct mm_struct *mm = vma->vm_mm;
1096
1097         /*
1098          * POWER9 NMMU must flush the TLB after clearing the PTE before
1099          * installing a PTE with more relaxed access permissions, see
1100          * radix__ptep_set_access_flags.
1101          */
1102         if (!cpu_has_feature(CPU_FTR_ARCH_31) &&
1103             is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
1104             (atomic_read(&mm->context.copros) > 0))
1105                 radix__flush_tlb_page(vma, addr);
1106
1107         set_pte_at(mm, addr, ptep, pte);
1108 }
1109
1110 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1111 {
1112         pte_t *ptep = (pte_t *)pud;
1113         pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
1114
1115         if (!radix_enabled())
1116                 return 0;
1117
1118         set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
1119
1120         return 1;
1121 }
1122
1123 int pud_clear_huge(pud_t *pud)
1124 {
1125         if (pud_is_leaf(*pud)) {
1126                 pud_clear(pud);
1127                 return 1;
1128         }
1129
1130         return 0;
1131 }
1132
1133 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1134 {
1135         pmd_t *pmd;
1136         int i;
1137
1138         pmd = pud_pgtable(*pud);
1139         pud_clear(pud);
1140
1141         flush_tlb_kernel_range(addr, addr + PUD_SIZE);
1142
1143         for (i = 0; i < PTRS_PER_PMD; i++) {
1144                 if (!pmd_none(pmd[i])) {
1145                         pte_t *pte;
1146                         pte = (pte_t *)pmd_page_vaddr(pmd[i]);
1147
1148                         pte_free_kernel(&init_mm, pte);
1149                 }
1150         }
1151
1152         pmd_free(&init_mm, pmd);
1153
1154         return 1;
1155 }
1156
1157 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1158 {
1159         pte_t *ptep = (pte_t *)pmd;
1160         pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
1161
1162         if (!radix_enabled())
1163                 return 0;
1164
1165         set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
1166
1167         return 1;
1168 }
1169
1170 int pmd_clear_huge(pmd_t *pmd)
1171 {
1172         if (pmd_is_leaf(*pmd)) {
1173                 pmd_clear(pmd);
1174                 return 1;
1175         }
1176
1177         return 0;
1178 }
1179
1180 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1181 {
1182         pte_t *pte;
1183
1184         pte = (pte_t *)pmd_page_vaddr(*pmd);
1185         pmd_clear(pmd);
1186
1187         flush_tlb_kernel_range(addr, addr + PMD_SIZE);
1188
1189         pte_free_kernel(&init_mm, pte);
1190
1191         return 1;
1192 }