1 // SPDX-License-Identifier: GPL-2.0-only
3 * Hibernation support for RISCV
5 * Copyright (C) 2023 StarFive Technology Co., Ltd.
7 * Author: Jee Heng Sia <jeeheng.sia@starfivetech.com>
10 #include <asm/barrier.h>
11 #include <asm/cacheflush.h>
12 #include <asm/mmu_context.h>
14 #include <asm/pgalloc.h>
15 #include <asm/pgtable.h>
16 #include <asm/sections.h>
17 #include <asm/set_memory.h>
19 #include <asm/suspend.h>
21 #include <linux/cpu.h>
22 #include <linux/memblock.h>
24 #include <linux/sched.h>
25 #include <linux/suspend.h>
26 #include <linux/utsname.h>
28 /* The logical cpu number we should resume on, initialised to a non-cpu number. */
29 static int sleep_cpu = -EINVAL;
31 /* Pointer to the temporary resume page table. */
32 static pgd_t *resume_pg_dir;
34 /* CPU context to be saved. */
35 struct suspend_context *hibernate_cpu_context;
36 EXPORT_SYMBOL_GPL(hibernate_cpu_context);
38 unsigned long relocated_restore_code;
39 EXPORT_SYMBOL_GPL(relocated_restore_code);
42 * struct arch_hibernate_hdr_invariants - container to store kernel build version.
43 * @uts_version: to save the build number and date so that we do not resume with
46 struct arch_hibernate_hdr_invariants {
47 char uts_version[__NEW_UTS_LEN + 1];
51 * struct arch_hibernate_hdr - helper parameters that help us to restore the image.
52 * @invariants: container to store kernel build version.
53 * @hartid: to make sure same boot_cpu executes the hibernate/restore code.
54 * @saved_satp: original page table used by the hibernated image.
55 * @restore_cpu_addr: the kernel's image address to restore the CPU context.
57 static struct arch_hibernate_hdr {
58 struct arch_hibernate_hdr_invariants invariants;
60 unsigned long saved_satp;
61 unsigned long restore_cpu_addr;
64 static void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i)
66 memset(i, 0, sizeof(*i));
67 memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version));
71 * Check if the given pfn is in the 'nosave' section.
73 int pfn_is_nosave(unsigned long pfn)
75 unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin);
76 unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1);
78 return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn));
81 void notrace save_processor_state(void)
85 void notrace restore_processor_state(void)
90 * Helper parameters need to be saved to the hibernation image header.
92 int arch_hibernation_header_save(void *addr, unsigned int max_size)
94 struct arch_hibernate_hdr *hdr = addr;
96 if (max_size < sizeof(*hdr))
99 arch_hdr_invariants(&hdr->invariants);
101 hdr->hartid = cpuid_to_hartid_map(sleep_cpu);
102 hdr->saved_satp = csr_read(CSR_SATP);
103 hdr->restore_cpu_addr = (unsigned long)__hibernate_cpu_resume;
107 EXPORT_SYMBOL_GPL(arch_hibernation_header_save);
110 * Retrieve the helper parameters from the hibernation image header.
112 int arch_hibernation_header_restore(void *addr)
114 struct arch_hibernate_hdr_invariants invariants;
115 struct arch_hibernate_hdr *hdr = addr;
118 arch_hdr_invariants(&invariants);
120 if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) {
121 pr_crit("Hibernate image not generated by this kernel!\n");
125 sleep_cpu = riscv_hartid_to_cpuid(hdr->hartid);
127 pr_crit("Hibernated on a CPU not known to this kernel!\n");
133 ret = bringup_hibernate_cpu(sleep_cpu);
143 EXPORT_SYMBOL_GPL(arch_hibernation_header_restore);
145 int swsusp_arch_suspend(void)
149 if (__cpu_suspend_enter(hibernate_cpu_context)) {
150 sleep_cpu = smp_processor_id();
151 suspend_save_csrs(hibernate_cpu_context);
154 suspend_restore_csrs(hibernate_cpu_context);
159 * Tell the hibernation core that we've just restored the memory.
168 static int temp_pgtable_map_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start,
169 unsigned long end, pgprot_t prot)
174 if (pmd_none(READ_ONCE(*dst_pmdp))) {
175 dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC);
179 pmd_populate_kernel(NULL, dst_pmdp, dst_ptep);
182 dst_ptep = pte_offset_kernel(dst_pmdp, start);
183 src_ptep = pte_offset_kernel(src_pmdp, start);
186 pte_t pte = READ_ONCE(*src_ptep);
188 if (pte_present(pte))
189 set_pte(dst_ptep, __pte(pte_val(pte) | pgprot_val(prot)));
190 } while (dst_ptep++, src_ptep++, start += PAGE_SIZE, start < end);
195 static int temp_pgtable_map_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start,
196 unsigned long end, pgprot_t prot)
203 if (pud_none(READ_ONCE(*dst_pudp))) {
204 dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC);
208 pud_populate(NULL, dst_pudp, dst_pmdp);
211 dst_pmdp = pmd_offset(dst_pudp, start);
212 src_pmdp = pmd_offset(src_pudp, start);
215 pmd_t pmd = READ_ONCE(*src_pmdp);
217 next = pmd_addr_end(start, end);
223 set_pmd(dst_pmdp, __pmd(pmd_val(pmd) | pgprot_val(prot)));
225 ret = temp_pgtable_map_pte(dst_pmdp, src_pmdp, start, next, prot);
229 } while (dst_pmdp++, src_pmdp++, start = next, start != end);
234 static int temp_pgtable_map_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long start,
235 unsigned long end, pgprot_t prot)
242 if (p4d_none(READ_ONCE(*dst_p4dp))) {
243 dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC);
247 p4d_populate(NULL, dst_p4dp, dst_pudp);
250 dst_pudp = pud_offset(dst_p4dp, start);
251 src_pudp = pud_offset(src_p4dp, start);
254 pud_t pud = READ_ONCE(*src_pudp);
256 next = pud_addr_end(start, end);
262 set_pud(dst_pudp, __pud(pud_val(pud) | pgprot_val(prot)));
264 ret = temp_pgtable_map_pmd(dst_pudp, src_pudp, start, next, prot);
268 } while (dst_pudp++, src_pudp++, start = next, start != end);
273 static int temp_pgtable_map_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
274 unsigned long end, pgprot_t prot)
281 if (pgd_none(READ_ONCE(*dst_pgdp))) {
282 dst_p4dp = (p4d_t *)get_safe_page(GFP_ATOMIC);
286 pgd_populate(NULL, dst_pgdp, dst_p4dp);
289 dst_p4dp = p4d_offset(dst_pgdp, start);
290 src_p4dp = p4d_offset(src_pgdp, start);
293 p4d_t p4d = READ_ONCE(*src_p4dp);
295 next = p4d_addr_end(start, end);
301 set_p4d(dst_p4dp, __p4d(p4d_val(p4d) | pgprot_val(prot)));
303 ret = temp_pgtable_map_pud(dst_p4dp, src_p4dp, start, next, prot);
307 } while (dst_p4dp++, src_p4dp++, start = next, start != end);
312 static int temp_pgtable_mapping(pgd_t *pgdp, unsigned long start, unsigned long end, pgprot_t prot)
314 pgd_t *dst_pgdp = pgd_offset_pgd(pgdp, start);
315 pgd_t *src_pgdp = pgd_offset_k(start);
320 pgd_t pgd = READ_ONCE(*src_pgdp);
322 next = pgd_addr_end(start, end);
328 set_pgd(dst_pgdp, __pgd(pgd_val(pgd) | pgprot_val(prot)));
330 ret = temp_pgtable_map_p4d(dst_pgdp, src_pgdp, start, next, prot);
334 } while (dst_pgdp++, src_pgdp++, start = next, start != end);
339 static unsigned long relocate_restore_code(void)
341 void *page = (void *)get_safe_page(GFP_ATOMIC);
346 copy_page(page, hibernate_core_restore_code);
348 /* Make the page containing the relocated code executable. */
349 set_memory_x((unsigned long)page, 1);
351 return (unsigned long)page;
354 int swsusp_arch_resume(void)
356 unsigned long end = (unsigned long)pfn_to_virt(max_low_pfn);
357 unsigned long start = PAGE_OFFSET;
361 * Memory allocated by get_safe_page() will be dealt with by the hibernation core,
362 * we don't need to free it here.
364 resume_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC);
369 * Create a temporary page table and map the whole linear region as executable and
372 ret = temp_pgtable_mapping(resume_pg_dir, start, end, __pgprot(_PAGE_WRITE | _PAGE_EXEC));
376 /* Move the restore code to a new page so that it doesn't get overwritten by itself. */
377 relocated_restore_code = relocate_restore_code();
378 if (relocated_restore_code == -ENOMEM)
382 * Map the __hibernate_cpu_resume() address to the temporary page table so that the
383 * restore code can jumps to it after finished restore the image. The next execution
384 * code doesn't find itself in a different address space after switching over to the
385 * original page table used by the hibernated image.
386 * The __hibernate_cpu_resume() mapping is unnecessary for RV32 since the kernel and
387 * linear addresses are identical, but different for RV64. To ensure consistency, we
388 * map it for both RV32 and RV64 kernels.
389 * Additionally, we should ensure that the page is writable before restoring the image.
391 start = (unsigned long)resume_hdr.restore_cpu_addr;
392 end = start + PAGE_SIZE;
394 ret = temp_pgtable_mapping(resume_pg_dir, start, end, __pgprot(_PAGE_WRITE));
398 hibernate_restore_image(resume_hdr.saved_satp, (PFN_DOWN(__pa(resume_pg_dir)) | satp_mode),
399 resume_hdr.restore_cpu_addr);
404 #ifdef CONFIG_PM_SLEEP_SMP
405 int hibernate_resume_nonboot_cpu_disable(void)
408 pr_err("Failing to resume from hibernate on an unknown CPU\n");
412 return freeze_secondary_cpus(sleep_cpu);
416 static int __init riscv_hibernate_init(void)
418 hibernate_cpu_context = kzalloc(sizeof(*hibernate_cpu_context), GFP_KERNEL);
420 if (WARN_ON(!hibernate_cpu_context))
426 early_initcall(riscv_hibernate_init);