3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
5 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
6 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
7 * Copyright (C) 1996 Paul Mackerras
9 * Derived from "arch/i386/mm/init.c"
10 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
12 * Dave Engebretsen <engebret@us.ibm.com>
13 * Rework for PPC64 port.
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
24 #include <linux/signal.h>
25 #include <linux/sched.h>
26 #include <linux/kernel.h>
27 #include <linux/errno.h>
28 #include <linux/string.h>
29 #include <linux/types.h>
30 #include <linux/mman.h>
32 #include <linux/swap.h>
33 #include <linux/stddef.h>
34 #include <linux/vmalloc.h>
35 #include <linux/init.h>
36 #include <linux/delay.h>
37 #include <linux/highmem.h>
38 #include <linux/idr.h>
39 #include <linux/nodemask.h>
40 #include <linux/module.h>
41 #include <linux/poison.h>
42 #include <linux/memblock.h>
43 #include <linux/hugetlb.h>
44 #include <linux/slab.h>
45 #include <linux/of_fdt.h>
46 #include <linux/libfdt.h>
47 #include <linux/memremap.h>
49 #include <asm/pgalloc.h>
54 #include <asm/mmu_context.h>
55 #include <asm/pgtable.h>
57 #include <linux/uaccess.h>
59 #include <asm/machdep.h>
62 #include <asm/processor.h>
63 #include <asm/mmzone.h>
64 #include <asm/cputable.h>
65 #include <asm/sections.h>
66 #include <asm/iommu.h>
71 #ifdef CONFIG_PPC_STD_MMU_64
72 #if H_PGTABLE_RANGE > USER_VSID_RANGE
73 #warning Limited user VSID range means pagetable space is wasted
75 #endif /* CONFIG_PPC_STD_MMU_64 */
77 phys_addr_t memstart_addr = ~0;
78 EXPORT_SYMBOL_GPL(memstart_addr);
79 phys_addr_t kernstart_addr;
80 EXPORT_SYMBOL_GPL(kernstart_addr);
82 #ifdef CONFIG_SPARSEMEM_VMEMMAP
84 * Given an address within the vmemmap, determine the pfn of the page that
85 * represents the start of the section it is within. Note that we have to
86 * do this by hand as the proffered address may not be correctly aligned.
87 * Subtraction of non-aligned pointers produces undefined results.
89 static unsigned long __meminit vmemmap_section_start(unsigned long page)
91 unsigned long offset = page - ((unsigned long)(vmemmap));
93 /* Return the pfn of the start of the section. */
94 return (offset / sizeof(struct page)) & PAGE_SECTION_MASK;
98 * Check if this vmemmap page is already initialised. If any section
99 * which overlaps this vmemmap page is initialised then this page is
100 * initialised already.
102 static int __meminit vmemmap_populated(unsigned long start, int page_size)
104 unsigned long end = start + page_size;
105 start = (unsigned long)(pfn_to_page(vmemmap_section_start(start)));
107 for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page)))
108 if (pfn_valid(page_to_pfn((struct page *)start)))
115 * vmemmap virtual address space management does not have a traditonal page
116 * table to track which virtual struct pages are backed by physical mapping.
117 * The virtual to physical mappings are tracked in a simple linked list
118 * format. 'vmemmap_list' maintains the entire vmemmap physical mapping at
119 * all times where as the 'next' list maintains the available
120 * vmemmap_backing structures which have been deleted from the
121 * 'vmemmap_global' list during system runtime (memory hotplug remove
122 * operation). The freed 'vmemmap_backing' structures are reused later when
123 * new requests come in without allocating fresh memory. This pointer also
124 * tracks the allocated 'vmemmap_backing' structures as we allocate one
125 * full page memory at a time when we dont have any.
127 struct vmemmap_backing *vmemmap_list;
128 static struct vmemmap_backing *next;
131 * The same pointer 'next' tracks individual chunks inside the allocated
132 * full page during the boot time and again tracks the freeed nodes during
133 * runtime. It is racy but it does not happen as they are separated by the
134 * boot process. Will create problem if some how we have memory hotplug
135 * operation during boot !!
138 static int num_freed;
140 static __meminit struct vmemmap_backing * vmemmap_list_alloc(int node)
142 struct vmemmap_backing *vmem_back;
143 /* get from freed entries first */
152 /* allocate a page when required and hand out chunks */
154 next = vmemmap_alloc_block(PAGE_SIZE, node);
155 if (unlikely(!next)) {
159 num_left = PAGE_SIZE / sizeof(struct vmemmap_backing);
167 static __meminit void vmemmap_list_populate(unsigned long phys,
171 struct vmemmap_backing *vmem_back;
173 vmem_back = vmemmap_list_alloc(node);
174 if (unlikely(!vmem_back)) {
179 vmem_back->phys = phys;
180 vmem_back->virt_addr = start;
181 vmem_back->list = vmemmap_list;
183 vmemmap_list = vmem_back;
186 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
188 unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
190 /* Align to the page size of the linear mapping. */
191 start = _ALIGN_DOWN(start, page_size);
193 pr_debug("vmemmap_populate %lx..%lx, node %d\n", start, end, node);
195 for (; start < end; start += page_size) {
196 struct vmem_altmap *altmap;
200 if (vmemmap_populated(start, page_size))
203 /* altmap lookups only work at section boundaries */
204 altmap = to_vmem_altmap(SECTION_ALIGN_DOWN(start));
206 p = __vmemmap_alloc_block_buf(page_size, node, altmap);
210 vmemmap_list_populate(__pa(p), start, node);
212 pr_debug(" * %016lx..%016lx allocated at %p\n",
213 start, start + page_size, p);
215 rc = vmemmap_create_mapping(start, page_size, __pa(p));
218 "vmemmap_populate: Unable to create vmemmap mapping: %d\n",
227 #ifdef CONFIG_MEMORY_HOTPLUG
228 static unsigned long vmemmap_list_free(unsigned long start)
230 struct vmemmap_backing *vmem_back, *vmem_back_prev;
232 vmem_back_prev = vmem_back = vmemmap_list;
234 /* look for it with prev pointer recorded */
235 for (; vmem_back; vmem_back = vmem_back->list) {
236 if (vmem_back->virt_addr == start)
238 vmem_back_prev = vmem_back;
241 if (unlikely(!vmem_back)) {
246 /* remove it from vmemmap_list */
247 if (vmem_back == vmemmap_list) /* remove head */
248 vmemmap_list = vmem_back->list;
250 vmem_back_prev->list = vmem_back->list;
252 /* next point to this freed entry */
253 vmem_back->list = next;
257 return vmem_back->phys;
260 void __ref vmemmap_free(unsigned long start, unsigned long end)
262 unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
263 unsigned long page_order = get_order(page_size);
265 start = _ALIGN_DOWN(start, page_size);
267 pr_debug("vmemmap_free %lx...%lx\n", start, end);
269 for (; start < end; start += page_size) {
270 unsigned long nr_pages, addr;
271 struct vmem_altmap *altmap;
272 struct page *section_base;
276 * the section has already be marked as invalid, so
277 * vmemmap_populated() true means some other sections still
278 * in this page, so skip it.
280 if (vmemmap_populated(start, page_size))
283 addr = vmemmap_list_free(start);
287 page = pfn_to_page(addr >> PAGE_SHIFT);
288 section_base = pfn_to_page(vmemmap_section_start(start));
289 nr_pages = 1 << page_order;
291 altmap = to_vmem_altmap((unsigned long) section_base);
293 vmem_altmap_free(altmap, nr_pages);
294 } else if (PageReserved(page)) {
295 /* allocated from bootmem */
296 if (page_size < PAGE_SIZE) {
298 * this shouldn't happen, but if it is
299 * the case, leave the memory there
304 free_reserved_page(page++);
307 free_pages((unsigned long)(__va(addr)), page_order);
310 vmemmap_remove_mapping(start, page_size);
314 void register_page_bootmem_memmap(unsigned long section_nr,
315 struct page *start_page, unsigned long size)
320 * We do not have access to the sparsemem vmemmap, so we fallback to
321 * walking the list of sparsemem blocks which we already maintain for
322 * the sake of crashdump. In the long run, we might want to maintain
323 * a tree if performance of that linear walk becomes a problem.
325 * realmode_pfn_to_page functions can fail due to:
326 * 1) As real sparsemem blocks do not lay in RAM continously (they
327 * are in virtual address space which is not available in the real mode),
328 * the requested page struct can be split between blocks so get_page/put_page
330 * 2) When huge pages are used, the get_page/put_page API will fail
331 * in real mode as the linked addresses in the page struct are virtual
334 struct page *realmode_pfn_to_page(unsigned long pfn)
336 struct vmemmap_backing *vmem_back;
338 unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
339 unsigned long pg_va = (unsigned long) pfn_to_page(pfn);
341 for (vmem_back = vmemmap_list; vmem_back; vmem_back = vmem_back->list) {
342 if (pg_va < vmem_back->virt_addr)
345 /* After vmemmap_list entry free is possible, need check all */
346 if ((pg_va + sizeof(struct page)) <=
347 (vmem_back->virt_addr + page_size)) {
348 page = (struct page *) (vmem_back->phys + pg_va -
349 vmem_back->virt_addr);
354 /* Probably that page struct is split between real pages */
357 EXPORT_SYMBOL_GPL(realmode_pfn_to_page);
361 struct page *realmode_pfn_to_page(unsigned long pfn)
363 struct page *page = pfn_to_page(pfn);
366 EXPORT_SYMBOL_GPL(realmode_pfn_to_page);
368 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
370 #ifdef CONFIG_PPC_STD_MMU_64
371 static bool disable_radix;
372 static int __init parse_disable_radix(char *p)
374 disable_radix = true;
377 early_param("disable_radix", parse_disable_radix);
380 * If we're running under a hypervisor, we need to check the contents of
381 * /chosen/ibm,architecture-vec-5 to see if the hypervisor is willing to do
382 * radix. If not, we clear the radix feature bit so we fall back to hash.
384 static void __init early_check_vec5(void)
386 unsigned long root, chosen;
391 root = of_get_flat_dt_root();
392 chosen = of_get_flat_dt_subnode_by_name(root, "chosen");
393 if (chosen == -FDT_ERR_NOTFOUND) {
394 cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
397 vec5 = of_get_flat_dt_prop(chosen, "ibm,architecture-vec-5", &size);
399 cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
402 if (size <= OV5_INDX(OV5_MMU_SUPPORT)) {
403 cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
407 /* Check for supported configuration */
408 mmu_supported = vec5[OV5_INDX(OV5_MMU_SUPPORT)] &
409 OV5_FEAT(OV5_MMU_SUPPORT);
410 if (mmu_supported == OV5_FEAT(OV5_MMU_RADIX)) {
411 /* Hypervisor only supports radix - check enabled && GTSE */
412 if (!early_radix_enabled()) {
413 pr_warn("WARNING: Ignoring cmdline option disable_radix\n");
415 if (!(vec5[OV5_INDX(OV5_RADIX_GTSE)] &
416 OV5_FEAT(OV5_RADIX_GTSE))) {
417 pr_warn("WARNING: Hypervisor doesn't support RADIX with GTSE\n");
419 /* Do radix anyway - the hypervisor said we had to */
420 cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX;
421 } else if (mmu_supported == OV5_FEAT(OV5_MMU_HASH)) {
422 /* Hypervisor only supports hash - disable radix */
423 cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
427 void __init mmu_early_init_devtree(void)
429 /* Disable radix mode based on kernel command line. */
431 cur_cpu_spec->mmu_features &= ~MMU_FTR_TYPE_RADIX;
434 * Check /chosen/ibm,architecture-vec-5 if running as a guest.
435 * When running bare-metal, we can use radix if we like
436 * even though the ibm,architecture-vec-5 property created by
437 * skiboot doesn't have the necessary bits set.
439 if (!(mfmsr() & MSR_HV))
442 if (early_radix_enabled())
443 radix__early_init_devtree();
445 hash__early_init_devtree();
447 #endif /* CONFIG_PPC_STD_MMU_64 */