arch/riscv/kernel/hibernate.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Hibernation support for RISCV
   4  *
   5  * Copyright (C) 2023 StarFive Technology Co., Ltd.
   6  *
   7  * Author: Jee Heng Sia <jeeheng.sia@starfivetech.com>
   8  */
   9
  10 #include <asm/barrier.h>
  11 #include <asm/cacheflush.h>
  12 #include <asm/mmu_context.h>
  13 #include <asm/page.h>
  14 #include <asm/pgalloc.h>
  15 #include <asm/pgtable.h>
  16 #include <asm/sections.h>
  17 #include <asm/set_memory.h>
  18 #include <asm/smp.h>
  19 #include <asm/suspend.h>
  20
  21 #include <linux/cpu.h>
  22 #include <linux/memblock.h>
  23 #include <linux/pm.h>
  24 #include <linux/sched.h>
  25 #include <linux/suspend.h>
  26 #include <linux/utsname.h>
  27
  28 /* The logical cpu number we should resume on, initialised to a non-cpu number. */
  29 static int sleep_cpu = -EINVAL;
  30
  31 /* Pointer to the temporary resume page table. */
  32 static pgd_t *resume_pg_dir;
  33
  34 /* CPU context to be saved. */
  35 struct suspend_context *hibernate_cpu_context;
  36 EXPORT_SYMBOL_GPL(hibernate_cpu_context);
  37
  38 unsigned long relocated_restore_code;
  39 EXPORT_SYMBOL_GPL(relocated_restore_code);
  40
  41 /**
  42  * struct arch_hibernate_hdr_invariants - container to store kernel build version.
  43  * @uts_version: to save the build number and date so that we do not resume with
  44  *              a different kernel.
  45  */
  46 struct arch_hibernate_hdr_invariants {
  47         char            uts_version[__NEW_UTS_LEN + 1];
  48 };
  49
  50 /**
  51  * struct arch_hibernate_hdr - helper parameters that help us to restore the image.
  52  * @invariants: container to store kernel build version.
  53  * @hartid: to make sure same boot_cpu executes the hibernate/restore code.
  54  * @saved_satp: original page table used by the hibernated image.
  55  * @restore_cpu_addr: the kernel's image address to restore the CPU context.
  56  */
  57 static struct arch_hibernate_hdr {
  58         struct arch_hibernate_hdr_invariants invariants;
  59         unsigned long   hartid;
  60         unsigned long   saved_satp;
  61         unsigned long   restore_cpu_addr;
  62 } resume_hdr;
  63
  64 static void arch_hdr_invariants(struct arch_hibernate_hdr_invariants *i)
  65 {
  66         memset(i, 0, sizeof(*i));
  67         memcpy(i->uts_version, init_utsname()->version, sizeof(i->uts_version));
  68 }
  69
  70 /*
  71  * Check if the given pfn is in the 'nosave' section.
  72  */
  73 int pfn_is_nosave(unsigned long pfn)
  74 {
  75         unsigned long nosave_begin_pfn = sym_to_pfn(&__nosave_begin);
  76         unsigned long nosave_end_pfn = sym_to_pfn(&__nosave_end - 1);
  77
  78         return ((pfn >= nosave_begin_pfn) && (pfn <= nosave_end_pfn));
  79 }
  80
  81 void notrace save_processor_state(void)
  82 {
  83 }
  84
  85 void notrace restore_processor_state(void)
  86 {
  87 }
  88
  89 /*
  90  * Helper parameters need to be saved to the hibernation image header.
  91  */
  92 int arch_hibernation_header_save(void *addr, unsigned int max_size)
  93 {
  94         struct arch_hibernate_hdr *hdr = addr;
  95
  96         if (max_size < sizeof(*hdr))
  97                 return -EOVERFLOW;
  98
  99         arch_hdr_invariants(&hdr->invariants);
 100
 101         hdr->hartid = cpuid_to_hartid_map(sleep_cpu);
 102         hdr->saved_satp = csr_read(CSR_SATP);
 103         hdr->restore_cpu_addr = (unsigned long)__hibernate_cpu_resume;
 104
 105         return 0;
 106 }
 107 EXPORT_SYMBOL_GPL(arch_hibernation_header_save);
 108
 109 /*
 110  * Retrieve the helper parameters from the hibernation image header.
 111  */
 112 int arch_hibernation_header_restore(void *addr)
 113 {
 114         struct arch_hibernate_hdr_invariants invariants;
 115         struct arch_hibernate_hdr *hdr = addr;
 116         int ret = 0;
 117
 118         arch_hdr_invariants(&invariants);
 119
 120         if (memcmp(&hdr->invariants, &invariants, sizeof(invariants))) {
 121                 pr_crit("Hibernate image not generated by this kernel!\n");
 122                 return -EINVAL;
 123         }
 124
 125         sleep_cpu = riscv_hartid_to_cpuid(hdr->hartid);
 126         if (sleep_cpu < 0) {
 127                 pr_crit("Hibernated on a CPU not known to this kernel!\n");
 128                 sleep_cpu = -EINVAL;
 129                 return -EINVAL;
 130         }
 131
 132 #ifdef CONFIG_SMP
 133         ret = bringup_hibernate_cpu(sleep_cpu);
 134         if (ret) {
 135                 sleep_cpu = -EINVAL;
 136                 return ret;
 137         }
 138 #endif
 139         resume_hdr = *hdr;
 140
 141         return ret;
 142 }
 143 EXPORT_SYMBOL_GPL(arch_hibernation_header_restore);
 144
 145 int swsusp_arch_suspend(void)
 146 {
 147         int ret = 0;
 148
 149         if (__cpu_suspend_enter(hibernate_cpu_context)) {
 150                 sleep_cpu = smp_processor_id();
 151                 suspend_save_csrs(hibernate_cpu_context);
 152                 ret = swsusp_save();
 153         } else {
 154                 suspend_restore_csrs(hibernate_cpu_context);
 155                 flush_tlb_all();
 156                 flush_icache_all();
 157
 158                 /*
 159                  * Tell the hibernation core that we've just restored the memory.
 160                  */
 161                 in_suspend = 0;
 162                 sleep_cpu = -EINVAL;
 163         }
 164
 165         return ret;
 166 }
 167
 168 static int temp_pgtable_map_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start,
 169                                 unsigned long end, pgprot_t prot)
 170 {
 171         pte_t *src_ptep;
 172         pte_t *dst_ptep;
 173
 174         if (pmd_none(READ_ONCE(*dst_pmdp))) {
 175                 dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC);
 176                 if (!dst_ptep)
 177                         return -ENOMEM;
 178
 179                 pmd_populate_kernel(NULL, dst_pmdp, dst_ptep);
 180         }
 181
 182         dst_ptep = pte_offset_kernel(dst_pmdp, start);
 183         src_ptep = pte_offset_kernel(src_pmdp, start);
 184
 185         do {
 186                 pte_t pte = READ_ONCE(*src_ptep);
 187
 188                 if (pte_present(pte))
 189                         set_pte(dst_ptep, __pte(pte_val(pte) | pgprot_val(prot)));
 190         } while (dst_ptep++, src_ptep++, start += PAGE_SIZE, start < end);
 191
 192         return 0;
 193 }
 194
 195 static int temp_pgtable_map_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start,
 196                                 unsigned long end, pgprot_t prot)
 197 {
 198         unsigned long next;
 199         unsigned long ret;
 200         pmd_t *src_pmdp;
 201         pmd_t *dst_pmdp;
 202
 203         if (pud_none(READ_ONCE(*dst_pudp))) {
 204                 dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC);
 205                 if (!dst_pmdp)
 206                         return -ENOMEM;
 207
 208                 pud_populate(NULL, dst_pudp, dst_pmdp);
 209         }
 210
 211         dst_pmdp = pmd_offset(dst_pudp, start);
 212         src_pmdp = pmd_offset(src_pudp, start);
 213
 214         do {
 215                 pmd_t pmd = READ_ONCE(*src_pmdp);
 216
 217                 next = pmd_addr_end(start, end);
 218
 219                 if (pmd_none(pmd))
 220                         continue;
 221
 222                 if (pmd_leaf(pmd)) {
 223                         set_pmd(dst_pmdp, __pmd(pmd_val(pmd) | pgprot_val(prot)));
 224                 } else {
 225                         ret = temp_pgtable_map_pte(dst_pmdp, src_pmdp, start, next, prot);
 226                         if (ret)
 227                                 return -ENOMEM;
 228                 }
 229         } while (dst_pmdp++, src_pmdp++, start = next, start != end);
 230
 231         return 0;
 232 }
 233
 234 static int temp_pgtable_map_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long start,
 235                                 unsigned long end, pgprot_t prot)
 236 {
 237         unsigned long next;
 238         unsigned long ret;
 239         pud_t *dst_pudp;
 240         pud_t *src_pudp;
 241
 242         if (p4d_none(READ_ONCE(*dst_p4dp))) {
 243                 dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC);
 244                 if (!dst_pudp)
 245                         return -ENOMEM;
 246
 247                 p4d_populate(NULL, dst_p4dp, dst_pudp);
 248         }
 249
 250         dst_pudp = pud_offset(dst_p4dp, start);
 251         src_pudp = pud_offset(src_p4dp, start);
 252
 253         do {
 254                 pud_t pud = READ_ONCE(*src_pudp);
 255
 256                 next = pud_addr_end(start, end);
 257
 258                 if (pud_none(pud))
 259                         continue;
 260
 261                 if (pud_leaf(pud)) {
 262                         set_pud(dst_pudp, __pud(pud_val(pud) | pgprot_val(prot)));
 263                 } else {
 264                         ret = temp_pgtable_map_pmd(dst_pudp, src_pudp, start, next, prot);
 265                         if (ret)
 266                                 return -ENOMEM;
 267                 }
 268         } while (dst_pudp++, src_pudp++, start = next, start != end);
 269
 270         return 0;
 271 }
 272
 273 static int temp_pgtable_map_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
 274                                 unsigned long end, pgprot_t prot)
 275 {
 276         unsigned long next;
 277         unsigned long ret;
 278         p4d_t *dst_p4dp;
 279         p4d_t *src_p4dp;
 280
 281         if (pgd_none(READ_ONCE(*dst_pgdp))) {
 282                 dst_p4dp = (p4d_t *)get_safe_page(GFP_ATOMIC);
 283                 if (!dst_p4dp)
 284                         return -ENOMEM;
 285
 286                 pgd_populate(NULL, dst_pgdp, dst_p4dp);
 287         }
 288
 289         dst_p4dp = p4d_offset(dst_pgdp, start);
 290         src_p4dp = p4d_offset(src_pgdp, start);
 291
 292         do {
 293                 p4d_t p4d = READ_ONCE(*src_p4dp);
 294
 295                 next = p4d_addr_end(start, end);
 296
 297                 if (p4d_none(p4d))
 298                         continue;
 299
 300                 if (p4d_leaf(p4d)) {
 301                         set_p4d(dst_p4dp, __p4d(p4d_val(p4d) | pgprot_val(prot)));
 302                 } else {
 303                         ret = temp_pgtable_map_pud(dst_p4dp, src_p4dp, start, next, prot);
 304                         if (ret)
 305                                 return -ENOMEM;
 306                 }
 307         } while (dst_p4dp++, src_p4dp++, start = next, start != end);
 308
 309         return 0;
 310 }
 311
 312 static int temp_pgtable_mapping(pgd_t *pgdp, unsigned long start, unsigned long end, pgprot_t prot)
 313 {
 314         pgd_t *dst_pgdp = pgd_offset_pgd(pgdp, start);
 315         pgd_t *src_pgdp = pgd_offset_k(start);
 316         unsigned long next;
 317         unsigned long ret;
 318
 319         do {
 320                 pgd_t pgd = READ_ONCE(*src_pgdp);
 321
 322                 next = pgd_addr_end(start, end);
 323
 324                 if (pgd_none(pgd))
 325                         continue;
 326
 327                 if (pgd_leaf(pgd)) {
 328                         set_pgd(dst_pgdp, __pgd(pgd_val(pgd) | pgprot_val(prot)));
 329                 } else {
 330                         ret = temp_pgtable_map_p4d(dst_pgdp, src_pgdp, start, next, prot);
 331                         if (ret)
 332                                 return -ENOMEM;
 333                 }
 334         } while (dst_pgdp++, src_pgdp++, start = next, start != end);
 335
 336         return 0;
 337 }
 338
 339 static unsigned long relocate_restore_code(void)
 340 {
 341         void *page = (void *)get_safe_page(GFP_ATOMIC);
 342
 343         if (!page)
 344                 return -ENOMEM;
 345
 346         copy_page(page, hibernate_core_restore_code);
 347
 348         /* Make the page containing the relocated code executable. */
 349         set_memory_x((unsigned long)page, 1);
 350
 351         return (unsigned long)page;
 352 }
 353
 354 int swsusp_arch_resume(void)
 355 {
 356         unsigned long end = (unsigned long)pfn_to_virt(max_low_pfn);
 357         unsigned long start = PAGE_OFFSET;
 358         int ret;
 359
 360         /*
 361          * Memory allocated by get_safe_page() will be dealt with by the hibernation core,
 362          * we don't need to free it here.
 363          */
 364         resume_pg_dir = (pgd_t *)get_safe_page(GFP_ATOMIC);
 365         if (!resume_pg_dir)
 366                 return -ENOMEM;
 367
 368         /*
 369          * Create a temporary page table and map the whole linear region as executable and
 370          * writable.
 371          */
 372         ret = temp_pgtable_mapping(resume_pg_dir, start, end, __pgprot(_PAGE_WRITE | _PAGE_EXEC));
 373         if (ret)
 374                 return ret;
 375
 376         /* Move the restore code to a new page so that it doesn't get overwritten by itself. */
 377         relocated_restore_code = relocate_restore_code();
 378         if (relocated_restore_code == -ENOMEM)
 379                 return -ENOMEM;
 380
 381         /*
 382          * Map the __hibernate_cpu_resume() address to the temporary page table so that the
 383          * restore code can jumps to it after finished restore the image. The next execution
 384          * code doesn't find itself in a different address space after switching over to the
 385          * original page table used by the hibernated image.
 386          * The __hibernate_cpu_resume() mapping is unnecessary for RV32 since the kernel and
 387          * linear addresses are identical, but different for RV64. To ensure consistency, we
 388          * map it for both RV32 and RV64 kernels.
 389          * Additionally, we should ensure that the page is writable before restoring the image.
 390          */
 391         start = (unsigned long)resume_hdr.restore_cpu_addr;
 392         end = start + PAGE_SIZE;
 393
 394         ret = temp_pgtable_mapping(resume_pg_dir, start, end, __pgprot(_PAGE_WRITE));
 395         if (ret)
 396                 return ret;
 397
 398         hibernate_restore_image(resume_hdr.saved_satp, (PFN_DOWN(__pa(resume_pg_dir)) | satp_mode),
 399                                 resume_hdr.restore_cpu_addr);
 400
 401         return 0;
 402 }
 403
 404 #ifdef CONFIG_PM_SLEEP_SMP
 405 int hibernate_resume_nonboot_cpu_disable(void)
 406 {
 407         if (sleep_cpu < 0) {
 408                 pr_err("Failing to resume from hibernate on an unknown CPU\n");
 409                 return -ENODEV;
 410         }
 411
 412         return freeze_secondary_cpus(sleep_cpu);
 413 }
 414 #endif
 415
 416 static int __init riscv_hibernate_init(void)
 417 {
 418         hibernate_cpu_context = kzalloc(sizeof(*hibernate_cpu_context), GFP_KERNEL);
 419
 420         if (WARN_ON(!hibernate_cpu_context))
 421                 return -ENOMEM;
 422
 423         return 0;
 424 }
 425
 426 early_initcall(riscv_hibernate_init);