1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/sched.h>
4 #include <linux/sched/debug.h>
5 #include <linux/init_task.h>
8 #include <linux/uaccess.h>
9 #include <asm/processor.h>
11 #include <asm/traps.h>
13 #define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
15 #define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x)
17 static void set_df_gdt_entry(unsigned int cpu);
20 * Called by double_fault with CR0.TS and EFLAGS.NT cleared. The CPU thinks
21 * we're running the doublefault task. Cannot return.
23 asmlinkage noinstr void __noreturn doublefault_shim(void)
28 BUILD_BUG_ON(sizeof(struct doublefault_stack) != PAGE_SIZE);
30 cr2 = native_read_cr2();
32 /* Reset back to the normal kernel task. */
34 set_df_gdt_entry(smp_processor_id());
39 * Fill in pt_regs. A downside of doing this in C is that the unwinder
40 * won't see it (no ENCODE_FRAME_POINTER), so a nested stack dump
41 * won't successfully unwind to the source of the double fault.
42 * The main dump from exc_double_fault() is fine, though, since it
43 * uses these regs directly.
45 * If anyone ever cares, this could be moved to asm.
50 regs.flags = TSS(flags);
52 /* We won't go through the entry asm, so we can leave __csh as 0. */
72 exc_double_fault(®s, 0, cr2);
75 * x86_32 does not save the original CR3 anywhere on a task switch.
76 * This means that, even if we wanted to return, we would need to find
77 * some way to reconstruct CR3. We could make a credible guess based
78 * on cpu_tlbstate, but that would be racy and would not account for
81 panic("cannot return from double fault\n");
84 DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = {
87 * No sp0 or ss0 -- we never run CPL != 0 with this TSS
88 * active. sp is filled in later.
91 .io_bitmap_base = IO_BITMAP_OFFSET_INVALID,
93 .ip = (unsigned long) asm_exc_double_fault,
94 .flags = X86_EFLAGS_FIXED,
99 .fs = __KERNEL_PERCPU,
102 .__cr3 = __pa_nodebug(swapper_pg_dir),
106 static void set_df_gdt_entry(unsigned int cpu)
108 /* Set up doublefault TSS pointer in the GDT */
109 __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS,
110 &get_cpu_entry_area(cpu)->doublefault_stack.tss);
114 void doublefault_init_cpu_tss(void)
116 unsigned int cpu = smp_processor_id();
117 struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
120 * The linker isn't smart enough to initialize percpu variables that
121 * point to other places in percpu space.
123 this_cpu_write(doublefault_stack.tss.sp,
124 (unsigned long)&cea->doublefault_stack.stack +
125 sizeof(doublefault_stack.stack));
127 set_df_gdt_entry(cpu);