kernel/kexec_file.c

   1 /*
   2  * kexec: kexec_file_load system call
   3  *
   4  * Copyright (C) 2014 Red Hat Inc.
   5  * Authors:
   6  *      Vivek Goyal <vgoyal@redhat.com>
   7  *
   8  * This source code is licensed under the GNU General Public License,
   9  * Version 2.  See the file COPYING for more details.
  10  */
  11
  12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  13
  14 #include <linux/capability.h>
  15 #include <linux/mm.h>
  16 #include <linux/file.h>
  17 #include <linux/slab.h>
  18 #include <linux/kexec.h>
  19 #include <linux/mutex.h>
  20 #include <linux/list.h>
  21 #include <linux/fs.h>
  22 #include <crypto/hash.h>
  23 #include <crypto/sha.h>
  24 #include <linux/syscalls.h>
  25 #include <linux/vmalloc.h>
  26 #include "kexec_internal.h"
  27
  28 /*
  29  * Declare these symbols weak so that if architecture provides a purgatory,
  30  * these will be overridden.
  31  */
  32 char __weak kexec_purgatory[0];
  33 size_t __weak kexec_purgatory_size = 0;
  34
  35 static int kexec_calculate_store_digests(struct kimage *image);
  36
  37 /* Architectures can provide this probe function */
  38 int __weak arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
  39                                          unsigned long buf_len)
  40 {
  41         return -ENOEXEC;
  42 }
  43
  44 void * __weak arch_kexec_kernel_image_load(struct kimage *image)
  45 {
  46         return ERR_PTR(-ENOEXEC);
  47 }
  48
  49 int __weak arch_kimage_file_post_load_cleanup(struct kimage *image)
  50 {
  51         return -EINVAL;
  52 }
  53
  54 #ifdef CONFIG_KEXEC_VERIFY_SIG
  55 int __weak arch_kexec_kernel_verify_sig(struct kimage *image, void *buf,
  56                                         unsigned long buf_len)
  57 {
  58         return -EKEYREJECTED;
  59 }
  60 #endif
  61
  62 /*
  63  * Free up memory used by kernel, initrd, and command line. This is temporary
  64  * memory allocation which is not needed any more after these buffers have
  65  * been loaded into separate segments and have been copied elsewhere.
  66  */
  67 void kimage_file_post_load_cleanup(struct kimage *image)
  68 {
  69         struct purgatory_info *pi = &image->purgatory_info;
  70
  71         vfree(image->kernel_buf);
  72         image->kernel_buf = NULL;
  73
  74         vfree(image->initrd_buf);
  75         image->initrd_buf = NULL;
  76
  77         kfree(image->cmdline_buf);
  78         image->cmdline_buf = NULL;
  79
  80         vfree(pi->purgatory_buf);
  81         pi->purgatory_buf = NULL;
  82
  83         vfree(pi->sechdrs);
  84         pi->sechdrs = NULL;
  85
  86         /* See if architecture has anything to cleanup post load */
  87         arch_kimage_file_post_load_cleanup(image);
  88
  89         /*
  90          * Above call should have called into bootloader to free up
  91          * any data stored in kimage->image_loader_data. It should
  92          * be ok now to free it up.
  93          */
  94         kfree(image->image_loader_data);
  95         image->image_loader_data = NULL;
  96 }
  97
  98 /*
  99  * In file mode list of segments is prepared by kernel. Copy relevant
 100  * data from user space, do error checking, prepare segment list
 101  */
 102 static int
 103 kimage_file_prepare_segments(struct kimage *image, int kernel_fd, int initrd_fd,
 104                              const char __user *cmdline_ptr,
 105                              unsigned long cmdline_len, unsigned flags)
 106 {
 107         int ret = 0;
 108         void *ldata;
 109         loff_t size;
 110
 111         ret = kernel_read_file_from_fd(kernel_fd, &image->kernel_buf,
 112                                        &size, INT_MAX, READING_KEXEC_IMAGE);
 113         if (ret)
 114                 return ret;
 115         image->kernel_buf_len = size;
 116
 117         /* Call arch image probe handlers */
 118         ret = arch_kexec_kernel_image_probe(image, image->kernel_buf,
 119                                             image->kernel_buf_len);
 120         if (ret)
 121                 goto out;
 122
 123 #ifdef CONFIG_KEXEC_VERIFY_SIG
 124         ret = arch_kexec_kernel_verify_sig(image, image->kernel_buf,
 125                                            image->kernel_buf_len);
 126         if (ret) {
 127                 pr_debug("kernel signature verification failed.\n");
 128                 goto out;
 129         }
 130         pr_debug("kernel signature verification successful.\n");
 131 #endif
 132         /* It is possible that there no initramfs is being loaded */
 133         if (!(flags & KEXEC_FILE_NO_INITRAMFS)) {
 134                 ret = kernel_read_file_from_fd(initrd_fd, &image->initrd_buf,
 135                                                &size, INT_MAX,
 136                                                READING_KEXEC_INITRAMFS);
 137                 if (ret)
 138                         goto out;
 139                 image->initrd_buf_len = size;
 140         }
 141
 142         if (cmdline_len) {
 143                 image->cmdline_buf = kzalloc(cmdline_len, GFP_KERNEL);
 144                 if (!image->cmdline_buf) {
 145                         ret = -ENOMEM;
 146                         goto out;
 147                 }
 148
 149                 ret = copy_from_user(image->cmdline_buf, cmdline_ptr,
 150                                      cmdline_len);
 151                 if (ret) {
 152                         ret = -EFAULT;
 153                         goto out;
 154                 }
 155
 156                 image->cmdline_buf_len = cmdline_len;
 157
 158                 /* command line should be a string with last byte null */
 159                 if (image->cmdline_buf[cmdline_len - 1] != '\0') {
 160                         ret = -EINVAL;
 161                         goto out;
 162                 }
 163         }
 164
 165         /* Call arch image load handlers */
 166         ldata = arch_kexec_kernel_image_load(image);
 167
 168         if (IS_ERR(ldata)) {
 169                 ret = PTR_ERR(ldata);
 170                 goto out;
 171         }
 172
 173         image->image_loader_data = ldata;
 174 out:
 175         /* In case of error, free up all allocated memory in this function */
 176         if (ret)
 177                 kimage_file_post_load_cleanup(image);
 178         return ret;
 179 }
 180
 181 static int
 182 kimage_file_alloc_init(struct kimage **rimage, int kernel_fd,
 183                        int initrd_fd, const char __user *cmdline_ptr,
 184                        unsigned long cmdline_len, unsigned long flags)
 185 {
 186         int ret;
 187         struct kimage *image;
 188         bool kexec_on_panic = flags & KEXEC_FILE_ON_CRASH;
 189
 190         image = do_kimage_alloc_init();
 191         if (!image)
 192                 return -ENOMEM;
 193
 194         image->file_mode = 1;
 195
 196         if (kexec_on_panic) {
 197                 /* Enable special crash kernel control page alloc policy. */
 198                 image->control_page = crashk_res.start;
 199                 image->type = KEXEC_TYPE_CRASH;
 200         }
 201
 202         ret = kimage_file_prepare_segments(image, kernel_fd, initrd_fd,
 203                                            cmdline_ptr, cmdline_len, flags);
 204         if (ret)
 205                 goto out_free_image;
 206
 207         ret = sanity_check_segment_list(image);
 208         if (ret)
 209                 goto out_free_post_load_bufs;
 210
 211         ret = -ENOMEM;
 212         image->control_code_page = kimage_alloc_control_pages(image,
 213                                            get_order(KEXEC_CONTROL_PAGE_SIZE));
 214         if (!image->control_code_page) {
 215                 pr_err("Could not allocate control_code_buffer\n");
 216                 goto out_free_post_load_bufs;
 217         }
 218
 219         if (!kexec_on_panic) {
 220                 image->swap_page = kimage_alloc_control_pages(image, 0);
 221                 if (!image->swap_page) {
 222                         pr_err("Could not allocate swap buffer\n");
 223                         goto out_free_control_pages;
 224                 }
 225         }
 226
 227         *rimage = image;
 228         return 0;
 229 out_free_control_pages:
 230         kimage_free_page_list(&image->control_pages);
 231 out_free_post_load_bufs:
 232         kimage_file_post_load_cleanup(image);
 233 out_free_image:
 234         kfree(image);
 235         return ret;
 236 }
 237
 238 SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd,
 239                 unsigned long, cmdline_len, const char __user *, cmdline_ptr,
 240                 unsigned long, flags)
 241 {
 242         int ret = 0, i;
 243         struct kimage **dest_image, *image;
 244
 245         /* We only trust the superuser with rebooting the system. */
 246         if (!capable(CAP_SYS_BOOT) || kexec_load_disabled)
 247                 return -EPERM;
 248
 249         /* Make sure we have a legal set of flags */
 250         if (flags != (flags & KEXEC_FILE_FLAGS))
 251                 return -EINVAL;
 252
 253         image = NULL;
 254
 255         if (!mutex_trylock(&kexec_mutex))
 256                 return -EBUSY;
 257
 258         dest_image = &kexec_image;
 259         if (flags & KEXEC_FILE_ON_CRASH) {
 260                 dest_image = &kexec_crash_image;
 261                 if (kexec_crash_image)
 262                         arch_kexec_unprotect_crashkres();
 263         }
 264
 265         if (flags & KEXEC_FILE_UNLOAD)
 266                 goto exchange;
 267
 268         /*
 269          * In case of crash, new kernel gets loaded in reserved region. It is
 270          * same memory where old crash kernel might be loaded. Free any
 271          * current crash dump kernel before we corrupt it.
 272          */
 273         if (flags & KEXEC_FILE_ON_CRASH)
 274                 kimage_free(xchg(&kexec_crash_image, NULL));
 275
 276         ret = kimage_file_alloc_init(&image, kernel_fd, initrd_fd, cmdline_ptr,
 277                                      cmdline_len, flags);
 278         if (ret)
 279                 goto out;
 280
 281         ret = machine_kexec_prepare(image);
 282         if (ret)
 283                 goto out;
 284
 285         ret = kexec_calculate_store_digests(image);
 286         if (ret)
 287                 goto out;
 288
 289         for (i = 0; i < image->nr_segments; i++) {
 290                 struct kexec_segment *ksegment;
 291
 292                 ksegment = &image->segment[i];
 293                 pr_debug("Loading segment %d: buf=0x%p bufsz=0x%zx mem=0x%lx memsz=0x%zx\n",
 294                          i, ksegment->buf, ksegment->bufsz, ksegment->mem,
 295                          ksegment->memsz);
 296
 297                 ret = kimage_load_segment(image, &image->segment[i]);
 298                 if (ret)
 299                         goto out;
 300         }
 301
 302         kimage_terminate(image);
 303
 304         /*
 305          * Free up any temporary buffers allocated which are not needed
 306          * after image has been loaded
 307          */
 308         kimage_file_post_load_cleanup(image);
 309 exchange:
 310         image = xchg(dest_image, image);
 311 out:
 312         if ((flags & KEXEC_FILE_ON_CRASH) && kexec_crash_image)
 313                 arch_kexec_protect_crashkres();
 314
 315         mutex_unlock(&kexec_mutex);
 316         kimage_free(image);
 317         return ret;
 318 }
 319
 320 static int locate_mem_hole_top_down(unsigned long start, unsigned long end,
 321                                     struct kexec_buf *kbuf)
 322 {
 323         struct kimage *image = kbuf->image;
 324         unsigned long temp_start, temp_end;
 325
 326         temp_end = min(end, kbuf->buf_max);
 327         temp_start = temp_end - kbuf->memsz;
 328
 329         do {
 330                 /* align down start */
 331                 temp_start = temp_start & (~(kbuf->buf_align - 1));
 332
 333                 if (temp_start < start || temp_start < kbuf->buf_min)
 334                         return 0;
 335
 336                 temp_end = temp_start + kbuf->memsz - 1;
 337
 338                 /*
 339                  * Make sure this does not conflict with any of existing
 340                  * segments
 341                  */
 342                 if (kimage_is_destination_range(image, temp_start, temp_end)) {
 343                         temp_start = temp_start - PAGE_SIZE;
 344                         continue;
 345                 }
 346
 347                 /* We found a suitable memory range */
 348                 break;
 349         } while (1);
 350
 351         /* If we are here, we found a suitable memory range */
 352         kbuf->mem = temp_start;
 353
 354         /* Success, stop navigating through remaining System RAM ranges */
 355         return 1;
 356 }
 357
 358 static int locate_mem_hole_bottom_up(unsigned long start, unsigned long end,
 359                                      struct kexec_buf *kbuf)
 360 {
 361         struct kimage *image = kbuf->image;
 362         unsigned long temp_start, temp_end;
 363
 364         temp_start = max(start, kbuf->buf_min);
 365
 366         do {
 367                 temp_start = ALIGN(temp_start, kbuf->buf_align);
 368                 temp_end = temp_start + kbuf->memsz - 1;
 369
 370                 if (temp_end > end || temp_end > kbuf->buf_max)
 371                         return 0;
 372                 /*
 373                  * Make sure this does not conflict with any of existing
 374                  * segments
 375                  */
 376                 if (kimage_is_destination_range(image, temp_start, temp_end)) {
 377                         temp_start = temp_start + PAGE_SIZE;
 378                         continue;
 379                 }
 380
 381                 /* We found a suitable memory range */
 382                 break;
 383         } while (1);
 384
 385         /* If we are here, we found a suitable memory range */
 386         kbuf->mem = temp_start;
 387
 388         /* Success, stop navigating through remaining System RAM ranges */
 389         return 1;
 390 }
 391
 392 static int locate_mem_hole_callback(u64 start, u64 end, void *arg)
 393 {
 394         struct kexec_buf *kbuf = (struct kexec_buf *)arg;
 395         unsigned long sz = end - start + 1;
 396
 397         /* Returning 0 will take to next memory range */
 398         if (sz < kbuf->memsz)
 399                 return 0;
 400
 401         if (end < kbuf->buf_min || start > kbuf->buf_max)
 402                 return 0;
 403
 404         /*
 405          * Allocate memory top down with-in ram range. Otherwise bottom up
 406          * allocation.
 407          */
 408         if (kbuf->top_down)
 409                 return locate_mem_hole_top_down(start, end, kbuf);
 410         return locate_mem_hole_bottom_up(start, end, kbuf);
 411 }
 412
 413 /*
 414  * Helper function for placing a buffer in a kexec segment. This assumes
 415  * that kexec_mutex is held.
 416  */
 417 int kexec_add_buffer(struct kimage *image, char *buffer, unsigned long bufsz,
 418                      unsigned long memsz, unsigned long buf_align,
 419                      unsigned long buf_min, unsigned long buf_max,
 420                      bool top_down, unsigned long *load_addr)
 421 {
 422
 423         struct kexec_segment *ksegment;
 424         struct kexec_buf buf, *kbuf;
 425         int ret;
 426
 427         /* Currently adding segment this way is allowed only in file mode */
 428         if (!image->file_mode)
 429                 return -EINVAL;
 430
 431         if (image->nr_segments >= KEXEC_SEGMENT_MAX)
 432                 return -EINVAL;
 433
 434         /*
 435          * Make sure we are not trying to add buffer after allocating
 436          * control pages. All segments need to be placed first before
 437          * any control pages are allocated. As control page allocation
 438          * logic goes through list of segments to make sure there are
 439          * no destination overlaps.
 440          */
 441         if (!list_empty(&image->control_pages)) {
 442                 WARN_ON(1);
 443                 return -EINVAL;
 444         }
 445
 446         memset(&buf, 0, sizeof(struct kexec_buf));
 447         kbuf = &buf;
 448         kbuf->image = image;
 449         kbuf->buffer = buffer;
 450         kbuf->bufsz = bufsz;
 451
 452         kbuf->memsz = ALIGN(memsz, PAGE_SIZE);
 453         kbuf->buf_align = max(buf_align, PAGE_SIZE);
 454         kbuf->buf_min = buf_min;
 455         kbuf->buf_max = buf_max;
 456         kbuf->top_down = top_down;
 457
 458         /* Walk the RAM ranges and allocate a suitable range for the buffer */
 459         if (image->type == KEXEC_TYPE_CRASH)
 460                 ret = walk_iomem_res_desc(crashk_res.desc,
 461                                 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
 462                                 crashk_res.start, crashk_res.end, kbuf,
 463                                 locate_mem_hole_callback);
 464         else
 465                 ret = walk_system_ram_res(0, -1, kbuf,
 466                                           locate_mem_hole_callback);
 467         if (ret != 1) {
 468                 /* A suitable memory range could not be found for buffer */
 469                 return -EADDRNOTAVAIL;
 470         }
 471
 472         /* Found a suitable memory range */
 473         ksegment = &image->segment[image->nr_segments];
 474         ksegment->kbuf = kbuf->buffer;
 475         ksegment->bufsz = kbuf->bufsz;
 476         ksegment->mem = kbuf->mem;
 477         ksegment->memsz = kbuf->memsz;
 478         image->nr_segments++;
 479         *load_addr = ksegment->mem;
 480         return 0;
 481 }
 482
 483 /* Calculate and store the digest of segments */
 484 static int kexec_calculate_store_digests(struct kimage *image)
 485 {
 486         struct crypto_shash *tfm;
 487         struct shash_desc *desc;
 488         int ret = 0, i, j, zero_buf_sz, sha_region_sz;
 489         size_t desc_size, nullsz;
 490         char *digest;
 491         void *zero_buf;
 492         struct kexec_sha_region *sha_regions;
 493         struct purgatory_info *pi = &image->purgatory_info;
 494
 495         zero_buf = __va(page_to_pfn(ZERO_PAGE(0)) << PAGE_SHIFT);
 496         zero_buf_sz = PAGE_SIZE;
 497
 498         tfm = crypto_alloc_shash("sha256", 0, 0);
 499         if (IS_ERR(tfm)) {
 500                 ret = PTR_ERR(tfm);
 501                 goto out;
 502         }
 503
 504         desc_size = crypto_shash_descsize(tfm) + sizeof(*desc);
 505         desc = kzalloc(desc_size, GFP_KERNEL);
 506         if (!desc) {
 507                 ret = -ENOMEM;
 508                 goto out_free_tfm;
 509         }
 510
 511         sha_region_sz = KEXEC_SEGMENT_MAX * sizeof(struct kexec_sha_region);
 512         sha_regions = vzalloc(sha_region_sz);
 513         if (!sha_regions) {
 514                 ret = -ENOMEM;
 515                 goto out_free_desc;
 516         }
 517
 518         desc->tfm   = tfm;
 519         desc->flags = 0;
 520
 521         ret = crypto_shash_init(desc);
 522         if (ret < 0)
 523                 goto out_free_sha_regions;
 524
 525         digest = kzalloc(SHA256_DIGEST_SIZE, GFP_KERNEL);
 526         if (!digest) {
 527                 ret = -ENOMEM;
 528                 goto out_free_sha_regions;
 529         }
 530
 531         for (j = i = 0; i < image->nr_segments; i++) {
 532                 struct kexec_segment *ksegment;
 533
 534                 ksegment = &image->segment[i];
 535                 /*
 536                  * Skip purgatory as it will be modified once we put digest
 537                  * info in purgatory.
 538                  */
 539                 if (ksegment->kbuf == pi->purgatory_buf)
 540                         continue;
 541
 542                 ret = crypto_shash_update(desc, ksegment->kbuf,
 543                                           ksegment->bufsz);
 544                 if (ret)
 545                         break;
 546
 547                 /*
 548                  * Assume rest of the buffer is filled with zero and
 549                  * update digest accordingly.
 550                  */
 551                 nullsz = ksegment->memsz - ksegment->bufsz;
 552                 while (nullsz) {
 553                         unsigned long bytes = nullsz;
 554
 555                         if (bytes > zero_buf_sz)
 556                                 bytes = zero_buf_sz;
 557                         ret = crypto_shash_update(desc, zero_buf, bytes);
 558                         if (ret)
 559                                 break;
 560                         nullsz -= bytes;
 561                 }
 562
 563                 if (ret)
 564                         break;
 565
 566                 sha_regions[j].start = ksegment->mem;
 567                 sha_regions[j].len = ksegment->memsz;
 568                 j++;
 569         }
 570
 571         if (!ret) {
 572                 ret = crypto_shash_final(desc, digest);
 573                 if (ret)
 574                         goto out_free_digest;
 575                 ret = kexec_purgatory_get_set_symbol(image, "sha_regions",
 576                                                 sha_regions, sha_region_sz, 0);
 577                 if (ret)
 578                         goto out_free_digest;
 579
 580                 ret = kexec_purgatory_get_set_symbol(image, "sha256_digest",
 581                                                 digest, SHA256_DIGEST_SIZE, 0);
 582                 if (ret)
 583                         goto out_free_digest;
 584         }
 585
 586 out_free_digest:
 587         kfree(digest);
 588 out_free_sha_regions:
 589         vfree(sha_regions);
 590 out_free_desc:
 591         kfree(desc);
 592 out_free_tfm:
 593         kfree(tfm);
 594 out:
 595         return ret;
 596 }
 597
 598 /* Actually load purgatory. Lot of code taken from kexec-tools */
 599 static int __kexec_load_purgatory(struct kimage *image, unsigned long min,
 600                                   unsigned long max, int top_down)
 601 {
 602         struct purgatory_info *pi = &image->purgatory_info;
 603         unsigned long align, buf_align, bss_align, buf_sz, bss_sz, bss_pad;
 604         unsigned long memsz, entry, load_addr, curr_load_addr, bss_addr, offset;
 605         unsigned char *buf_addr, *src;
 606         int i, ret = 0, entry_sidx = -1;
 607         const Elf_Shdr *sechdrs_c;
 608         Elf_Shdr *sechdrs = NULL;
 609         void *purgatory_buf = NULL;
 610
 611         /*
 612          * sechdrs_c points to section headers in purgatory and are read
 613          * only. No modifications allowed.
 614          */
 615         sechdrs_c = (void *)pi->ehdr + pi->ehdr->e_shoff;
 616
 617         /*
 618          * We can not modify sechdrs_c[] and its fields. It is read only.
 619          * Copy it over to a local copy where one can store some temporary
 620          * data and free it at the end. We need to modify ->sh_addr and
 621          * ->sh_offset fields to keep track of permanent and temporary
 622          * locations of sections.
 623          */
 624         sechdrs = vzalloc(pi->ehdr->e_shnum * sizeof(Elf_Shdr));
 625         if (!sechdrs)
 626                 return -ENOMEM;
 627
 628         memcpy(sechdrs, sechdrs_c, pi->ehdr->e_shnum * sizeof(Elf_Shdr));
 629
 630         /*
 631          * We seem to have multiple copies of sections. First copy is which
 632          * is embedded in kernel in read only section. Some of these sections
 633          * will be copied to a temporary buffer and relocated. And these
 634          * sections will finally be copied to their final destination at
 635          * segment load time.
 636          *
 637          * Use ->sh_offset to reflect section address in memory. It will
 638          * point to original read only copy if section is not allocatable.
 639          * Otherwise it will point to temporary copy which will be relocated.
 640          *
 641          * Use ->sh_addr to contain final address of the section where it
 642          * will go during execution time.
 643          */
 644         for (i = 0; i < pi->ehdr->e_shnum; i++) {
 645                 if (sechdrs[i].sh_type == SHT_NOBITS)
 646                         continue;
 647
 648                 sechdrs[i].sh_offset = (unsigned long)pi->ehdr +
 649                                                 sechdrs[i].sh_offset;
 650         }
 651
 652         /*
 653          * Identify entry point section and make entry relative to section
 654          * start.
 655          */
 656         entry = pi->ehdr->e_entry;
 657         for (i = 0; i < pi->ehdr->e_shnum; i++) {
 658                 if (!(sechdrs[i].sh_flags & SHF_ALLOC))
 659                         continue;
 660
 661                 if (!(sechdrs[i].sh_flags & SHF_EXECINSTR))
 662                         continue;
 663
 664                 /* Make entry section relative */
 665                 if (sechdrs[i].sh_addr <= pi->ehdr->e_entry &&
 666                     ((sechdrs[i].sh_addr + sechdrs[i].sh_size) >
 667                      pi->ehdr->e_entry)) {
 668                         entry_sidx = i;
 669                         entry -= sechdrs[i].sh_addr;
 670                         break;
 671                 }
 672         }
 673
 674         /* Determine how much memory is needed to load relocatable object. */
 675         buf_align = 1;
 676         bss_align = 1;
 677         buf_sz = 0;
 678         bss_sz = 0;
 679
 680         for (i = 0; i < pi->ehdr->e_shnum; i++) {
 681                 if (!(sechdrs[i].sh_flags & SHF_ALLOC))
 682                         continue;
 683
 684                 align = sechdrs[i].sh_addralign;
 685                 if (sechdrs[i].sh_type != SHT_NOBITS) {
 686                         if (buf_align < align)
 687                                 buf_align = align;
 688                         buf_sz = ALIGN(buf_sz, align);
 689                         buf_sz += sechdrs[i].sh_size;
 690                 } else {
 691                         /* bss section */
 692                         if (bss_align < align)
 693                                 bss_align = align;
 694                         bss_sz = ALIGN(bss_sz, align);
 695                         bss_sz += sechdrs[i].sh_size;
 696                 }
 697         }
 698
 699         /* Determine the bss padding required to align bss properly */
 700         bss_pad = 0;
 701         if (buf_sz & (bss_align - 1))
 702                 bss_pad = bss_align - (buf_sz & (bss_align - 1));
 703
 704         memsz = buf_sz + bss_pad + bss_sz;
 705
 706         /* Allocate buffer for purgatory */
 707         purgatory_buf = vzalloc(buf_sz);
 708         if (!purgatory_buf) {
 709                 ret = -ENOMEM;
 710                 goto out;
 711         }
 712
 713         if (buf_align < bss_align)
 714                 buf_align = bss_align;
 715
 716         /* Add buffer to segment list */
 717         ret = kexec_add_buffer(image, purgatory_buf, buf_sz, memsz,
 718                                 buf_align, min, max, top_down,
 719                                 &pi->purgatory_load_addr);
 720         if (ret)
 721                 goto out;
 722
 723         /* Load SHF_ALLOC sections */
 724         buf_addr = purgatory_buf;
 725         load_addr = curr_load_addr = pi->purgatory_load_addr;
 726         bss_addr = load_addr + buf_sz + bss_pad;
 727
 728         for (i = 0; i < pi->ehdr->e_shnum; i++) {
 729                 if (!(sechdrs[i].sh_flags & SHF_ALLOC))
 730                         continue;
 731
 732                 align = sechdrs[i].sh_addralign;
 733                 if (sechdrs[i].sh_type != SHT_NOBITS) {
 734                         curr_load_addr = ALIGN(curr_load_addr, align);
 735                         offset = curr_load_addr - load_addr;
 736                         /* We already modifed ->sh_offset to keep src addr */
 737                         src = (char *) sechdrs[i].sh_offset;
 738                         memcpy(buf_addr + offset, src, sechdrs[i].sh_size);
 739
 740                         /* Store load address and source address of section */
 741                         sechdrs[i].sh_addr = curr_load_addr;
 742
 743                         /*
 744                          * This section got copied to temporary buffer. Update
 745                          * ->sh_offset accordingly.
 746                          */
 747                         sechdrs[i].sh_offset = (unsigned long)(buf_addr + offset);
 748
 749                         /* Advance to the next address */
 750                         curr_load_addr += sechdrs[i].sh_size;
 751                 } else {
 752                         bss_addr = ALIGN(bss_addr, align);
 753                         sechdrs[i].sh_addr = bss_addr;
 754                         bss_addr += sechdrs[i].sh_size;
 755                 }
 756         }
 757
 758         /* Update entry point based on load address of text section */
 759         if (entry_sidx >= 0)
 760                 entry += sechdrs[entry_sidx].sh_addr;
 761
 762         /* Make kernel jump to purgatory after shutdown */
 763         image->start = entry;
 764
 765         /* Used later to get/set symbol values */
 766         pi->sechdrs = sechdrs;
 767
 768         /*
 769          * Used later to identify which section is purgatory and skip it
 770          * from checksumming.
 771          */
 772         pi->purgatory_buf = purgatory_buf;
 773         return ret;
 774 out:
 775         vfree(sechdrs);
 776         vfree(purgatory_buf);
 777         return ret;
 778 }
 779
 780 static int kexec_apply_relocations(struct kimage *image)
 781 {
 782         int i, ret;
 783         struct purgatory_info *pi = &image->purgatory_info;
 784         Elf_Shdr *sechdrs = pi->sechdrs;
 785
 786         /* Apply relocations */
 787         for (i = 0; i < pi->ehdr->e_shnum; i++) {
 788                 Elf_Shdr *section, *symtab;
 789
 790                 if (sechdrs[i].sh_type != SHT_RELA &&
 791                     sechdrs[i].sh_type != SHT_REL)
 792                         continue;
 793
 794                 /*
 795                  * For section of type SHT_RELA/SHT_REL,
 796                  * ->sh_link contains section header index of associated
 797                  * symbol table. And ->sh_info contains section header
 798                  * index of section to which relocations apply.
 799                  */
 800                 if (sechdrs[i].sh_info >= pi->ehdr->e_shnum ||
 801                     sechdrs[i].sh_link >= pi->ehdr->e_shnum)
 802                         return -ENOEXEC;
 803
 804                 section = &sechdrs[sechdrs[i].sh_info];
 805                 symtab = &sechdrs[sechdrs[i].sh_link];
 806
 807                 if (!(section->sh_flags & SHF_ALLOC))
 808                         continue;
 809
 810                 /*
 811                  * symtab->sh_link contain section header index of associated
 812                  * string table.
 813                  */
 814                 if (symtab->sh_link >= pi->ehdr->e_shnum)
 815                         /* Invalid section number? */
 816                         continue;
 817
 818                 /*
 819                  * Respective architecture needs to provide support for applying
 820                  * relocations of type SHT_RELA/SHT_REL.
 821                  */
 822                 if (sechdrs[i].sh_type == SHT_RELA)
 823                         ret = arch_kexec_apply_relocations_add(pi->ehdr,
 824                                                                sechdrs, i);
 825                 else if (sechdrs[i].sh_type == SHT_REL)
 826                         ret = arch_kexec_apply_relocations(pi->ehdr,
 827                                                            sechdrs, i);
 828                 if (ret)
 829                         return ret;
 830         }
 831
 832         return 0;
 833 }
 834
 835 /* Load relocatable purgatory object and relocate it appropriately */
 836 int kexec_load_purgatory(struct kimage *image, unsigned long min,
 837                          unsigned long max, int top_down,
 838                          unsigned long *load_addr)
 839 {
 840         struct purgatory_info *pi = &image->purgatory_info;
 841         int ret;
 842
 843         if (kexec_purgatory_size <= 0)
 844                 return -EINVAL;
 845
 846         if (kexec_purgatory_size < sizeof(Elf_Ehdr))
 847                 return -ENOEXEC;
 848
 849         pi->ehdr = (Elf_Ehdr *)kexec_purgatory;
 850
 851         if (memcmp(pi->ehdr->e_ident, ELFMAG, SELFMAG) != 0
 852             || pi->ehdr->e_type != ET_REL
 853             || !elf_check_arch(pi->ehdr)
 854             || pi->ehdr->e_shentsize != sizeof(Elf_Shdr))
 855                 return -ENOEXEC;
 856
 857         if (pi->ehdr->e_shoff >= kexec_purgatory_size
 858             || (pi->ehdr->e_shnum * sizeof(Elf_Shdr) >
 859             kexec_purgatory_size - pi->ehdr->e_shoff))
 860                 return -ENOEXEC;
 861
 862         ret = __kexec_load_purgatory(image, min, max, top_down);
 863         if (ret)
 864                 return ret;
 865
 866         ret = kexec_apply_relocations(image);
 867         if (ret)
 868                 goto out;
 869
 870         *load_addr = pi->purgatory_load_addr;
 871         return 0;
 872 out:
 873         vfree(pi->sechdrs);
 874         pi->sechdrs = NULL;
 875
 876         vfree(pi->purgatory_buf);
 877         pi->purgatory_buf = NULL;
 878         return ret;
 879 }
 880
 881 static Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi,
 882                                             const char *name)
 883 {
 884         Elf_Sym *syms;
 885         Elf_Shdr *sechdrs;
 886         Elf_Ehdr *ehdr;
 887         int i, k;
 888         const char *strtab;
 889
 890         if (!pi->sechdrs || !pi->ehdr)
 891                 return NULL;
 892
 893         sechdrs = pi->sechdrs;
 894         ehdr = pi->ehdr;
 895
 896         for (i = 0; i < ehdr->e_shnum; i++) {
 897                 if (sechdrs[i].sh_type != SHT_SYMTAB)
 898                         continue;
 899
 900                 if (sechdrs[i].sh_link >= ehdr->e_shnum)
 901                         /* Invalid strtab section number */
 902                         continue;
 903                 strtab = (char *)sechdrs[sechdrs[i].sh_link].sh_offset;
 904                 syms = (Elf_Sym *)sechdrs[i].sh_offset;
 905
 906                 /* Go through symbols for a match */
 907                 for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) {
 908                         if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL)
 909                                 continue;
 910
 911                         if (strcmp(strtab + syms[k].st_name, name) != 0)
 912                                 continue;
 913
 914                         if (syms[k].st_shndx == SHN_UNDEF ||
 915                             syms[k].st_shndx >= ehdr->e_shnum) {
 916                                 pr_debug("Symbol: %s has bad section index %d.\n",
 917                                                 name, syms[k].st_shndx);
 918                                 return NULL;
 919                         }
 920
 921                         /* Found the symbol we are looking for */
 922                         return &syms[k];
 923                 }
 924         }
 925
 926         return NULL;
 927 }
 928
 929 void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name)
 930 {
 931         struct purgatory_info *pi = &image->purgatory_info;
 932         Elf_Sym *sym;
 933         Elf_Shdr *sechdr;
 934
 935         sym = kexec_purgatory_find_symbol(pi, name);
 936         if (!sym)
 937                 return ERR_PTR(-EINVAL);
 938
 939         sechdr = &pi->sechdrs[sym->st_shndx];
 940
 941         /*
 942          * Returns the address where symbol will finally be loaded after
 943          * kexec_load_segment()
 944          */
 945         return (void *)(sechdr->sh_addr + sym->st_value);
 946 }
 947
 948 /*
 949  * Get or set value of a symbol. If "get_value" is true, symbol value is
 950  * returned in buf otherwise symbol value is set based on value in buf.
 951  */
 952 int kexec_purgatory_get_set_symbol(struct kimage *image, const char *name,
 953                                    void *buf, unsigned int size, bool get_value)
 954 {
 955         Elf_Sym *sym;
 956         Elf_Shdr *sechdrs;
 957         struct purgatory_info *pi = &image->purgatory_info;
 958         char *sym_buf;
 959
 960         sym = kexec_purgatory_find_symbol(pi, name);
 961         if (!sym)
 962                 return -EINVAL;
 963
 964         if (sym->st_size != size) {
 965                 pr_err("symbol %s size mismatch: expected %lu actual %u\n",
 966                        name, (unsigned long)sym->st_size, size);
 967                 return -EINVAL;
 968         }
 969
 970         sechdrs = pi->sechdrs;
 971
 972         if (sechdrs[sym->st_shndx].sh_type == SHT_NOBITS) {
 973                 pr_err("symbol %s is in a bss section. Cannot %s\n", name,
 974                        get_value ? "get" : "set");
 975                 return -EINVAL;
 976         }
 977
 978         sym_buf = (unsigned char *)sechdrs[sym->st_shndx].sh_offset +
 979                                         sym->st_value;
 980
 981         if (get_value)
 982                 memcpy((void *)buf, sym_buf, size);
 983         else
 984                 memcpy((void *)sym_buf, buf, size);
 985
 986         return 0;
 987 }