1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/spinlock.h>
4 #include <linux/percpu.h>
5 #include <linux/kallsyms.h>
6 #include <linux/kcore.h>
8 #include <asm/cpu_entry_area.h>
9 #include <asm/pgtable.h>
10 #include <asm/fixmap.h>
13 static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage);
16 static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
17 [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
18 static DEFINE_PER_CPU(struct kcore_list, kcore_entry_trampoline);
21 struct cpu_entry_area *get_cpu_entry_area(int cpu)
23 unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE;
24 BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0);
26 return (struct cpu_entry_area *) va;
28 EXPORT_SYMBOL(get_cpu_entry_area);
30 void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags)
32 unsigned long va = (unsigned long) cea_vaddr;
33 pte_t pte = pfn_pte(pa >> PAGE_SHIFT, flags);
36 * The cpu_entry_area is shared between the user and kernel
37 * page tables. All of its ptes can safely be global.
38 * _PAGE_GLOBAL gets reused to help indicate PROT_NONE for
39 * non-present PTEs, so be careful not to set it in that
40 * case to avoid confusion.
42 if (boot_cpu_has(X86_FEATURE_PGE) &&
43 (pgprot_val(flags) & _PAGE_PRESENT))
44 pte = pte_set_flags(pte, _PAGE_GLOBAL);
46 set_pte_vaddr(va, pte);
50 cea_map_percpu_pages(void *cea_vaddr, void *ptr, int pages, pgprot_t prot)
52 for ( ; pages; pages--, cea_vaddr+= PAGE_SIZE, ptr += PAGE_SIZE)
53 cea_set_pte(cea_vaddr, per_cpu_ptr_to_phys(ptr), prot);
56 static void percpu_setup_debug_store(int cpu)
58 #ifdef CONFIG_CPU_SUP_INTEL
62 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
65 cea = &get_cpu_entry_area(cpu)->cpu_debug_store;
66 npages = sizeof(struct debug_store) / PAGE_SIZE;
67 BUILD_BUG_ON(sizeof(struct debug_store) % PAGE_SIZE != 0);
68 cea_map_percpu_pages(cea, &per_cpu(cpu_debug_store, cpu), npages,
71 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers;
73 * Force the population of PMDs for not yet allocated per cpu
74 * memory like debug store buffers.
76 npages = sizeof(struct debug_store_buffers) / PAGE_SIZE;
77 for (; npages; npages--, cea += PAGE_SIZE)
78 cea_set_pte(cea, 0, PAGE_NONE);
82 /* Setup the fixmap mappings only once per-processor */
83 static void __init setup_cpu_entry_area(int cpu)
86 extern char _entry_trampoline[];
88 /* On 64-bit systems, we use a read-only fixmap GDT and TSS. */
89 pgprot_t gdt_prot = PAGE_KERNEL_RO;
90 pgprot_t tss_prot = PAGE_KERNEL_RO;
93 * On native 32-bit systems, the GDT cannot be read-only because
94 * our double fault handler uses a task gate, and entering through
95 * a task gate needs to change an available TSS to busy. If the
96 * GDT is read-only, that will triple fault. The TSS cannot be
97 * read-only because the CPU writes to it on task switches.
99 * On Xen PV, the GDT must be read-only because the hypervisor
102 pgprot_t gdt_prot = boot_cpu_has(X86_FEATURE_XENPV) ?
103 PAGE_KERNEL_RO : PAGE_KERNEL;
104 pgprot_t tss_prot = PAGE_KERNEL;
107 cea_set_pte(&get_cpu_entry_area(cpu)->gdt, get_cpu_gdt_paddr(cpu),
110 cea_map_percpu_pages(&get_cpu_entry_area(cpu)->entry_stack_page,
111 per_cpu_ptr(&entry_stack_storage, cpu), 1,
115 * The Intel SDM says (Volume 3, 7.2.1):
117 * Avoid placing a page boundary in the part of the TSS that the
118 * processor reads during a task switch (the first 104 bytes). The
119 * processor may not correctly perform address translations if a
120 * boundary occurs in this area. During a task switch, the processor
121 * reads and writes into the first 104 bytes of each TSS (using
122 * contiguous physical addresses beginning with the physical address
123 * of the first byte of the TSS). So, after TSS access begins, if
124 * part of the 104 bytes is not physically contiguous, the processor
125 * will access incorrect information without generating a page-fault
128 * There are also a lot of errata involving the TSS spanning a page
129 * boundary. Assert that we're not doing that.
131 BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^
132 offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK);
133 BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0);
134 cea_map_percpu_pages(&get_cpu_entry_area(cpu)->tss,
135 &per_cpu(cpu_tss_rw, cpu),
136 sizeof(struct tss_struct) / PAGE_SIZE, tss_prot);
139 per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu);
143 BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
144 BUILD_BUG_ON(sizeof(exception_stacks) !=
145 sizeof(((struct cpu_entry_area *)0)->exception_stacks));
146 cea_map_percpu_pages(&get_cpu_entry_area(cpu)->exception_stacks,
147 &per_cpu(exception_stacks, cpu),
148 sizeof(exception_stacks) / PAGE_SIZE, PAGE_KERNEL);
150 cea_set_pte(&get_cpu_entry_area(cpu)->entry_trampoline,
151 __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX);
153 * The cpu_entry_area alias addresses are not in the kernel binary
154 * so they do not show up in /proc/kcore normally. This adds entries
157 kclist_add_remap(&per_cpu(kcore_entry_trampoline, cpu),
159 &get_cpu_entry_area(cpu)->entry_trampoline, PAGE_SIZE);
161 percpu_setup_debug_store(cpu);
165 int arch_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
168 unsigned int cpu, ncpu = 0;
170 if (symnum >= num_possible_cpus())
173 for_each_possible_cpu(cpu) {
174 if (ncpu++ >= symnum)
178 *value = (unsigned long)&get_cpu_entry_area(cpu)->entry_trampoline;
180 strlcpy(name, "__entry_SYSCALL_64_trampoline", KSYM_NAME_LEN);
186 static __init void setup_cpu_entry_area_ptes(void)
189 unsigned long start, end;
191 BUILD_BUG_ON(CPU_ENTRY_AREA_PAGES * PAGE_SIZE < CPU_ENTRY_AREA_MAP_SIZE);
192 BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK);
194 start = CPU_ENTRY_AREA_BASE;
195 end = start + CPU_ENTRY_AREA_MAP_SIZE;
197 /* Careful here: start + PMD_SIZE might wrap around */
198 for (; start < end && start >= CPU_ENTRY_AREA_BASE; start += PMD_SIZE)
199 populate_extra_pte(start);
203 void __init setup_cpu_entry_areas(void)
207 setup_cpu_entry_area_ptes();
209 for_each_possible_cpu(cpu)
210 setup_cpu_entry_area(cpu);
213 * This is the last essential update to swapper_pgdir which needs
214 * to be synchronized to initial_page_table on 32bit.
216 sync_initial_page_table();