tools/lib/bpf/linker.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2 /*
   3  * BPF static linker
   4  *
   5  * Copyright (c) 2021 Facebook
   6  */
   7 #include <stdbool.h>
   8 #include <stddef.h>
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <unistd.h>
  13 #include <errno.h>
  14 #include <linux/err.h>
  15 #include <linux/btf.h>
  16 #include <elf.h>
  17 #include <libelf.h>
  18 #include <fcntl.h>
  19 #include "libbpf.h"
  20 #include "btf.h"
  21 #include "libbpf_internal.h"
  22 #include "strset.h"
  23
  24 #define BTF_EXTERN_SEC ".extern"
  25
  26 struct src_sec {
  27         const char *sec_name;
  28         /* positional (not necessarily ELF) index in an array of sections */
  29         int id;
  30         /* positional (not necessarily ELF) index of a matching section in a final object file */
  31         int dst_id;
  32         /* section data offset in a matching output section */
  33         int dst_off;
  34         /* whether section is omitted from the final ELF file */
  35         bool skipped;
  36         /* whether section is an ephemeral section, not mapped to an ELF section */
  37         bool ephemeral;
  38
  39         /* ELF info */
  40         size_t sec_idx;
  41         Elf_Scn *scn;
  42         Elf64_Shdr *shdr;
  43         Elf_Data *data;
  44
  45         /* corresponding BTF DATASEC type ID */
  46         int sec_type_id;
  47 };
  48
  49 struct src_obj {
  50         const char *filename;
  51         int fd;
  52         Elf *elf;
  53         /* Section header strings section index */
  54         size_t shstrs_sec_idx;
  55         /* SYMTAB section index */
  56         size_t symtab_sec_idx;
  57
  58         struct btf *btf;
  59         struct btf_ext *btf_ext;
  60
  61         /* List of sections (including ephemeral). Slot zero is unused. */
  62         struct src_sec *secs;
  63         int sec_cnt;
  64
  65         /* mapping of symbol indices from src to dst ELF */
  66         int *sym_map;
  67         /* mapping from the src BTF type IDs to dst ones */
  68         int *btf_type_map;
  69 };
  70
  71 /* single .BTF.ext data section */
  72 struct btf_ext_sec_data {
  73         size_t rec_cnt;
  74         __u32 rec_sz;
  75         void *recs;
  76 };
  77
  78 struct glob_sym {
  79         /* ELF symbol index */
  80         int sym_idx;
  81         /* associated section id for .ksyms, .kconfig, etc, but not .extern */
  82         int sec_id;
  83         /* extern name offset in STRTAB */
  84         int name_off;
  85         /* optional associated BTF type ID */
  86         int btf_id;
  87         /* BTF type ID to which VAR/FUNC type is pointing to; used for
  88          * rewriting types when extern VAR/FUNC is resolved to a concrete
  89          * definition
  90          */
  91         int underlying_btf_id;
  92         /* sec_var index in the corresponding dst_sec, if exists */
  93         int var_idx;
  94
  95         /* extern or resolved/global symbol */
  96         bool is_extern;
  97         /* weak or strong symbol, never goes back from strong to weak */
  98         bool is_weak;
  99 };
 100
 101 struct dst_sec {
 102         char *sec_name;
 103         /* positional (not necessarily ELF) index in an array of sections */
 104         int id;
 105
 106         bool ephemeral;
 107
 108         /* ELF info */
 109         size_t sec_idx;
 110         Elf_Scn *scn;
 111         Elf64_Shdr *shdr;
 112         Elf_Data *data;
 113
 114         /* final output section size */
 115         int sec_sz;
 116         /* final output contents of the section */
 117         void *raw_data;
 118
 119         /* corresponding STT_SECTION symbol index in SYMTAB */
 120         int sec_sym_idx;
 121
 122         /* section's DATASEC variable info, emitted on BTF finalization */
 123         bool has_btf;
 124         int sec_var_cnt;
 125         struct btf_var_secinfo *sec_vars;
 126
 127         /* section's .BTF.ext data */
 128         struct btf_ext_sec_data func_info;
 129         struct btf_ext_sec_data line_info;
 130         struct btf_ext_sec_data core_relo_info;
 131 };
 132
 133 struct bpf_linker {
 134         char *filename;
 135         int fd;
 136         Elf *elf;
 137         Elf64_Ehdr *elf_hdr;
 138
 139         /* Output sections metadata */
 140         struct dst_sec *secs;
 141         int sec_cnt;
 142
 143         struct strset *strtab_strs; /* STRTAB unique strings */
 144         size_t strtab_sec_idx; /* STRTAB section index */
 145         size_t symtab_sec_idx; /* SYMTAB section index */
 146
 147         struct btf *btf;
 148         struct btf_ext *btf_ext;
 149
 150         /* global (including extern) ELF symbols */
 151         int glob_sym_cnt;
 152         struct glob_sym *glob_syms;
 153 };
 154
 155 #define pr_warn_elf(fmt, ...)                                                                   \
 156         libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 157
 158 static int init_output_elf(struct bpf_linker *linker, const char *file);
 159
 160 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
 161                                 const struct bpf_linker_file_opts *opts,
 162                                 struct src_obj *obj);
 163 static int linker_sanity_check_elf(struct src_obj *obj);
 164 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 165 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 166 static int linker_sanity_check_btf(struct src_obj *obj);
 167 static int linker_sanity_check_btf_ext(struct src_obj *obj);
 168 static int linker_fixup_btf(struct src_obj *obj);
 169 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 170 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 171 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
 172                                  Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
 173 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 174 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 175 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 176
 177 static int finalize_btf(struct bpf_linker *linker);
 178 static int finalize_btf_ext(struct bpf_linker *linker);
 179
 180 void bpf_linker__free(struct bpf_linker *linker)
 181 {
 182         int i;
 183
 184         if (!linker)
 185                 return;
 186
 187         free(linker->filename);
 188
 189         if (linker->elf)
 190                 elf_end(linker->elf);
 191
 192         if (linker->fd >= 0)
 193                 close(linker->fd);
 194
 195         strset__free(linker->strtab_strs);
 196
 197         btf__free(linker->btf);
 198         btf_ext__free(linker->btf_ext);
 199
 200         for (i = 1; i < linker->sec_cnt; i++) {
 201                 struct dst_sec *sec = &linker->secs[i];
 202
 203                 free(sec->sec_name);
 204                 free(sec->raw_data);
 205                 free(sec->sec_vars);
 206
 207                 free(sec->func_info.recs);
 208                 free(sec->line_info.recs);
 209                 free(sec->core_relo_info.recs);
 210         }
 211         free(linker->secs);
 212
 213         free(linker->glob_syms);
 214         free(linker);
 215 }
 216
 217 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 218 {
 219         struct bpf_linker *linker;
 220         int err;
 221
 222         if (!OPTS_VALID(opts, bpf_linker_opts))
 223                 return errno = EINVAL, NULL;
 224
 225         if (elf_version(EV_CURRENT) == EV_NONE) {
 226                 pr_warn_elf("libelf initialization failed");
 227                 return errno = EINVAL, NULL;
 228         }
 229
 230         linker = calloc(1, sizeof(*linker));
 231         if (!linker)
 232                 return errno = ENOMEM, NULL;
 233
 234         linker->fd = -1;
 235
 236         err = init_output_elf(linker, filename);
 237         if (err)
 238                 goto err_out;
 239
 240         return linker;
 241
 242 err_out:
 243         bpf_linker__free(linker);
 244         return errno = -err, NULL;
 245 }
 246
 247 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 248 {
 249         struct dst_sec *secs = linker->secs, *sec;
 250         size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 251
 252         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 253         if (!secs)
 254                 return NULL;
 255
 256         /* zero out newly allocated memory */
 257         memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 258
 259         linker->secs = secs;
 260         linker->sec_cnt = new_cnt;
 261
 262         sec = &linker->secs[new_cnt - 1];
 263         sec->id = new_cnt - 1;
 264         sec->sec_name = strdup(sec_name);
 265         if (!sec->sec_name)
 266                 return NULL;
 267
 268         return sec;
 269 }
 270
 271 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 272 {
 273         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 274         Elf64_Sym *syms, *sym;
 275         size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 276
 277         syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 278         if (!syms)
 279                 return NULL;
 280
 281         sym = &syms[sym_cnt];
 282         memset(sym, 0, sizeof(*sym));
 283
 284         symtab->raw_data = syms;
 285         symtab->sec_sz += sizeof(*sym);
 286         symtab->shdr->sh_size += sizeof(*sym);
 287         symtab->data->d_size += sizeof(*sym);
 288
 289         if (sym_idx)
 290                 *sym_idx = sym_cnt;
 291
 292         return sym;
 293 }
 294
 295 static int init_output_elf(struct bpf_linker *linker, const char *file)
 296 {
 297         int err, str_off;
 298         Elf64_Sym *init_sym;
 299         struct dst_sec *sec;
 300
 301         linker->filename = strdup(file);
 302         if (!linker->filename)
 303                 return -ENOMEM;
 304
 305         linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
 306         if (linker->fd < 0) {
 307                 err = -errno;
 308                 pr_warn("failed to create '%s': %d\n", file, err);
 309                 return err;
 310         }
 311
 312         linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 313         if (!linker->elf) {
 314                 pr_warn_elf("failed to create ELF object");
 315                 return -EINVAL;
 316         }
 317
 318         /* ELF header */
 319         linker->elf_hdr = elf64_newehdr(linker->elf);
 320         if (!linker->elf_hdr) {
 321                 pr_warn_elf("failed to create ELF header");
 322                 return -EINVAL;
 323         }
 324
 325         linker->elf_hdr->e_machine = EM_BPF;
 326         linker->elf_hdr->e_type = ET_REL;
 327 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 328         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
 329 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 330         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
 331 #else
 332 #error "Unknown __BYTE_ORDER__"
 333 #endif
 334
 335         /* STRTAB */
 336         /* initialize strset with an empty string to conform to ELF */
 337         linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 338         if (libbpf_get_error(linker->strtab_strs))
 339                 return libbpf_get_error(linker->strtab_strs);
 340
 341         sec = add_dst_sec(linker, ".strtab");
 342         if (!sec)
 343                 return -ENOMEM;
 344
 345         sec->scn = elf_newscn(linker->elf);
 346         if (!sec->scn) {
 347                 pr_warn_elf("failed to create STRTAB section");
 348                 return -EINVAL;
 349         }
 350
 351         sec->shdr = elf64_getshdr(sec->scn);
 352         if (!sec->shdr)
 353                 return -EINVAL;
 354
 355         sec->data = elf_newdata(sec->scn);
 356         if (!sec->data) {
 357                 pr_warn_elf("failed to create STRTAB data");
 358                 return -EINVAL;
 359         }
 360
 361         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 362         if (str_off < 0)
 363                 return str_off;
 364
 365         sec->sec_idx = elf_ndxscn(sec->scn);
 366         linker->elf_hdr->e_shstrndx = sec->sec_idx;
 367         linker->strtab_sec_idx = sec->sec_idx;
 368
 369         sec->shdr->sh_name = str_off;
 370         sec->shdr->sh_type = SHT_STRTAB;
 371         sec->shdr->sh_flags = SHF_STRINGS;
 372         sec->shdr->sh_offset = 0;
 373         sec->shdr->sh_link = 0;
 374         sec->shdr->sh_info = 0;
 375         sec->shdr->sh_addralign = 1;
 376         sec->shdr->sh_size = sec->sec_sz = 0;
 377         sec->shdr->sh_entsize = 0;
 378
 379         /* SYMTAB */
 380         sec = add_dst_sec(linker, ".symtab");
 381         if (!sec)
 382                 return -ENOMEM;
 383
 384         sec->scn = elf_newscn(linker->elf);
 385         if (!sec->scn) {
 386                 pr_warn_elf("failed to create SYMTAB section");
 387                 return -EINVAL;
 388         }
 389
 390         sec->shdr = elf64_getshdr(sec->scn);
 391         if (!sec->shdr)
 392                 return -EINVAL;
 393
 394         sec->data = elf_newdata(sec->scn);
 395         if (!sec->data) {
 396                 pr_warn_elf("failed to create SYMTAB data");
 397                 return -EINVAL;
 398         }
 399
 400         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 401         if (str_off < 0)
 402                 return str_off;
 403
 404         sec->sec_idx = elf_ndxscn(sec->scn);
 405         linker->symtab_sec_idx = sec->sec_idx;
 406
 407         sec->shdr->sh_name = str_off;
 408         sec->shdr->sh_type = SHT_SYMTAB;
 409         sec->shdr->sh_flags = 0;
 410         sec->shdr->sh_offset = 0;
 411         sec->shdr->sh_link = linker->strtab_sec_idx;
 412         /* sh_info should be one greater than the index of the last local
 413          * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 414          */
 415         sec->shdr->sh_info = 0;
 416         sec->shdr->sh_addralign = 8;
 417         sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 418
 419         /* .BTF */
 420         linker->btf = btf__new_empty();
 421         err = libbpf_get_error(linker->btf);
 422         if (err)
 423                 return err;
 424
 425         /* add the special all-zero symbol */
 426         init_sym = add_new_sym(linker, NULL);
 427         if (!init_sym)
 428                 return -EINVAL;
 429
 430         init_sym->st_name = 0;
 431         init_sym->st_info = 0;
 432         init_sym->st_other = 0;
 433         init_sym->st_shndx = SHN_UNDEF;
 434         init_sym->st_value = 0;
 435         init_sym->st_size = 0;
 436
 437         return 0;
 438 }
 439
 440 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
 441                          const struct bpf_linker_file_opts *opts)
 442 {
 443         struct src_obj obj = {};
 444         int err = 0;
 445
 446         if (!OPTS_VALID(opts, bpf_linker_file_opts))
 447                 return libbpf_err(-EINVAL);
 448
 449         if (!linker->elf)
 450                 return libbpf_err(-EINVAL);
 451
 452         err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
 453         err = err ?: linker_append_sec_data(linker, &obj);
 454         err = err ?: linker_append_elf_syms(linker, &obj);
 455         err = err ?: linker_append_elf_relos(linker, &obj);
 456         err = err ?: linker_append_btf(linker, &obj);
 457         err = err ?: linker_append_btf_ext(linker, &obj);
 458
 459         /* free up src_obj resources */
 460         free(obj.btf_type_map);
 461         btf__free(obj.btf);
 462         btf_ext__free(obj.btf_ext);
 463         free(obj.secs);
 464         free(obj.sym_map);
 465         if (obj.elf)
 466                 elf_end(obj.elf);
 467         if (obj.fd >= 0)
 468                 close(obj.fd);
 469
 470         return libbpf_err(err);
 471 }
 472
 473 static bool is_dwarf_sec_name(const char *name)
 474 {
 475         /* approximation, but the actual list is too long */
 476         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 477 }
 478
 479 static bool is_ignored_sec(struct src_sec *sec)
 480 {
 481         Elf64_Shdr *shdr = sec->shdr;
 482         const char *name = sec->sec_name;
 483
 484         /* no special handling of .strtab */
 485         if (shdr->sh_type == SHT_STRTAB)
 486                 return true;
 487
 488         /* ignore .llvm_addrsig section as well */
 489         if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 490                 return true;
 491
 492         /* no subprograms will lead to an empty .text section, ignore it */
 493         if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 494             strcmp(sec->sec_name, ".text") == 0)
 495                 return true;
 496
 497         /* DWARF sections */
 498         if (is_dwarf_sec_name(sec->sec_name))
 499                 return true;
 500
 501         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 502                 name += sizeof(".rel") - 1;
 503                 /* DWARF section relocations */
 504                 if (is_dwarf_sec_name(name))
 505                         return true;
 506
 507                 /* .BTF and .BTF.ext don't need relocations */
 508                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
 509                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
 510                         return true;
 511         }
 512
 513         return false;
 514 }
 515
 516 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 517 {
 518         struct src_sec *secs = obj->secs, *sec;
 519         size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 520
 521         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 522         if (!secs)
 523                 return NULL;
 524
 525         /* zero out newly allocated memory */
 526         memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 527
 528         obj->secs = secs;
 529         obj->sec_cnt = new_cnt;
 530
 531         sec = &obj->secs[new_cnt - 1];
 532         sec->id = new_cnt - 1;
 533         sec->sec_name = sec_name;
 534
 535         return sec;
 536 }
 537
 538 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
 539                                 const struct bpf_linker_file_opts *opts,
 540                                 struct src_obj *obj)
 541 {
 542 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 543         const int host_endianness = ELFDATA2LSB;
 544 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 545         const int host_endianness = ELFDATA2MSB;
 546 #else
 547 #error "Unknown __BYTE_ORDER__"
 548 #endif
 549         int err = 0;
 550         Elf_Scn *scn;
 551         Elf_Data *data;
 552         Elf64_Ehdr *ehdr;
 553         Elf64_Shdr *shdr;
 554         struct src_sec *sec;
 555
 556         pr_debug("linker: adding object file '%s'...\n", filename);
 557
 558         obj->filename = filename;
 559
 560         obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
 561         if (obj->fd < 0) {
 562                 err = -errno;
 563                 pr_warn("failed to open file '%s': %d\n", filename, err);
 564                 return err;
 565         }
 566         obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 567         if (!obj->elf) {
 568                 err = -errno;
 569                 pr_warn_elf("failed to parse ELF file '%s'", filename);
 570                 return err;
 571         }
 572
 573         /* Sanity check ELF file high-level properties */
 574         ehdr = elf64_getehdr(obj->elf);
 575         if (!ehdr) {
 576                 err = -errno;
 577                 pr_warn_elf("failed to get ELF header for %s", filename);
 578                 return err;
 579         }
 580         if (ehdr->e_ident[EI_DATA] != host_endianness) {
 581                 err = -EOPNOTSUPP;
 582                 pr_warn_elf("unsupported byte order of ELF file %s", filename);
 583                 return err;
 584         }
 585         if (ehdr->e_type != ET_REL
 586             || ehdr->e_machine != EM_BPF
 587             || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 588                 err = -EOPNOTSUPP;
 589                 pr_warn_elf("unsupported kind of ELF file %s", filename);
 590                 return err;
 591         }
 592
 593         if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 594                 err = -errno;
 595                 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
 596                 return err;
 597         }
 598
 599         scn = NULL;
 600         while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 601                 size_t sec_idx = elf_ndxscn(scn);
 602                 const char *sec_name;
 603
 604                 shdr = elf64_getshdr(scn);
 605                 if (!shdr) {
 606                         err = -errno;
 607                         pr_warn_elf("failed to get section #%zu header for %s",
 608                                     sec_idx, filename);
 609                         return err;
 610                 }
 611
 612                 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 613                 if (!sec_name) {
 614                         err = -errno;
 615                         pr_warn_elf("failed to get section #%zu name for %s",
 616                                     sec_idx, filename);
 617                         return err;
 618                 }
 619
 620                 data = elf_getdata(scn, 0);
 621                 if (!data) {
 622                         err = -errno;
 623                         pr_warn_elf("failed to get section #%zu (%s) data from %s",
 624                                     sec_idx, sec_name, filename);
 625                         return err;
 626                 }
 627
 628                 sec = add_src_sec(obj, sec_name);
 629                 if (!sec)
 630                         return -ENOMEM;
 631
 632                 sec->scn = scn;
 633                 sec->shdr = shdr;
 634                 sec->data = data;
 635                 sec->sec_idx = elf_ndxscn(scn);
 636
 637                 if (is_ignored_sec(sec)) {
 638                         sec->skipped = true;
 639                         continue;
 640                 }
 641
 642                 switch (shdr->sh_type) {
 643                 case SHT_SYMTAB:
 644                         if (obj->symtab_sec_idx) {
 645                                 err = -EOPNOTSUPP;
 646                                 pr_warn("multiple SYMTAB sections found, not supported\n");
 647                                 return err;
 648                         }
 649                         obj->symtab_sec_idx = sec_idx;
 650                         break;
 651                 case SHT_STRTAB:
 652                         /* we'll construct our own string table */
 653                         break;
 654                 case SHT_PROGBITS:
 655                         if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 656                                 obj->btf = btf__new(data->d_buf, shdr->sh_size);
 657                                 err = libbpf_get_error(obj->btf);
 658                                 if (err) {
 659                                         pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
 660                                         return err;
 661                                 }
 662                                 sec->skipped = true;
 663                                 continue;
 664                         }
 665                         if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 666                                 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 667                                 err = libbpf_get_error(obj->btf_ext);
 668                                 if (err) {
 669                                         pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
 670                                         return err;
 671                                 }
 672                                 sec->skipped = true;
 673                                 continue;
 674                         }
 675
 676                         /* data & code */
 677                         break;
 678                 case SHT_NOBITS:
 679                         /* BSS */
 680                         break;
 681                 case SHT_REL:
 682                         /* relocations */
 683                         break;
 684                 default:
 685                         pr_warn("unrecognized section #%zu (%s) in %s\n",
 686                                 sec_idx, sec_name, filename);
 687                         err = -EINVAL;
 688                         return err;
 689                 }
 690         }
 691
 692         err = err ?: linker_sanity_check_elf(obj);
 693         err = err ?: linker_sanity_check_btf(obj);
 694         err = err ?: linker_sanity_check_btf_ext(obj);
 695         err = err ?: linker_fixup_btf(obj);
 696
 697         return err;
 698 }
 699
 700 static int linker_sanity_check_elf(struct src_obj *obj)
 701 {
 702         struct src_sec *sec;
 703         int i, err;
 704
 705         if (!obj->symtab_sec_idx) {
 706                 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 707                 return -EINVAL;
 708         }
 709         if (!obj->shstrs_sec_idx) {
 710                 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 711                 return -EINVAL;
 712         }
 713
 714         for (i = 1; i < obj->sec_cnt; i++) {
 715                 sec = &obj->secs[i];
 716
 717                 if (sec->sec_name[0] == '\0') {
 718                         pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 719                         return -EINVAL;
 720                 }
 721
 722                 if (is_dwarf_sec_name(sec->sec_name))
 723                         continue;
 724
 725                 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
 726                         pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
 727                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 728                                 obj->filename);
 729                         return -EINVAL;
 730                 }
 731                 if (sec->shdr->sh_addralign != sec->data->d_align) {
 732                         pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
 733                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 734                                 (long long unsigned)sec->data->d_align, obj->filename);
 735                         return -EINVAL;
 736                 }
 737
 738                 if (sec->shdr->sh_size != sec->data->d_size) {
 739                         pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
 740                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 741                                 (long long unsigned)sec->data->d_size, obj->filename);
 742                         return -EINVAL;
 743                 }
 744
 745                 switch (sec->shdr->sh_type) {
 746                 case SHT_SYMTAB:
 747                         err = linker_sanity_check_elf_symtab(obj, sec);
 748                         if (err)
 749                                 return err;
 750                         break;
 751                 case SHT_STRTAB:
 752                         break;
 753                 case SHT_PROGBITS:
 754                         if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 755                                 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
 756                                         pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
 757                                                 sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 758                                                 obj->filename);
 759                                         return -EINVAL;
 760                                 }
 761                         }
 762                         break;
 763                 case SHT_NOBITS:
 764                         break;
 765                 case SHT_REL:
 766                         err = linker_sanity_check_elf_relos(obj, sec);
 767                         if (err)
 768                                 return err;
 769                         break;
 770                 case SHT_LLVM_ADDRSIG:
 771                         break;
 772                 default:
 773                         pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 774                                 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 775                         return -EINVAL;
 776                 }
 777         }
 778
 779         return 0;
 780 }
 781
 782 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 783 {
 784         struct src_sec *link_sec;
 785         Elf64_Sym *sym;
 786         int i, n;
 787
 788         if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 789                 return -EINVAL;
 790         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 791                 return -EINVAL;
 792
 793         if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 794                 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 795                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 796                 return -EINVAL;
 797         }
 798         link_sec = &obj->secs[sec->shdr->sh_link];
 799         if (link_sec->shdr->sh_type != SHT_STRTAB) {
 800                 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 801                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 802                 return -EINVAL;
 803         }
 804
 805         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 806         sym = sec->data->d_buf;
 807         for (i = 0; i < n; i++, sym++) {
 808                 int sym_type = ELF64_ST_TYPE(sym->st_info);
 809                 int sym_bind = ELF64_ST_BIND(sym->st_info);
 810                 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
 811
 812                 if (i == 0) {
 813                         if (sym->st_name != 0 || sym->st_info != 0
 814                             || sym->st_other != 0 || sym->st_shndx != 0
 815                             || sym->st_value != 0 || sym->st_size != 0) {
 816                                 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
 817                                 return -EINVAL;
 818                         }
 819                         continue;
 820                 }
 821                 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
 822                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
 823                                 i, sec->sec_idx, sym_bind);
 824                         return -EINVAL;
 825                 }
 826                 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
 827                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
 828                                 i, sec->sec_idx, sym_vis);
 829                         return -EINVAL;
 830                 }
 831                 if (sym->st_shndx == 0) {
 832                         if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
 833                             || sym->st_value != 0 || sym->st_size != 0) {
 834                                 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
 835                                         i, obj->filename);
 836
 837                                 return -EINVAL;
 838                         }
 839                         continue;
 840                 }
 841                 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
 842                         pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 843                                 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 844                         return -EINVAL;
 845                 }
 846                 if (sym_type == STT_SECTION) {
 847                         if (sym->st_value != 0)
 848                                 return -EINVAL;
 849                         continue;
 850                 }
 851         }
 852
 853         return 0;
 854 }
 855
 856 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 857 {
 858         struct src_sec *link_sec, *sym_sec;
 859         Elf64_Rel *relo;
 860         int i, n;
 861
 862         if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 863                 return -EINVAL;
 864         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 865                 return -EINVAL;
 866
 867         /* SHT_REL's sh_link should point to SYMTAB */
 868         if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 869                 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
 870                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 871                 return -EINVAL;
 872         }
 873
 874         /* SHT_REL's sh_info points to relocated section */
 875         if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
 876                 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
 877                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 878                 return -EINVAL;
 879         }
 880         link_sec = &obj->secs[sec->shdr->sh_info];
 881
 882         /* .rel<secname> -> <secname> pattern is followed */
 883         if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
 884             || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
 885                 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
 886                         sec->sec_idx, obj->filename);
 887                 return -EINVAL;
 888         }
 889
 890         /* don't further validate relocations for ignored sections */
 891         if (link_sec->skipped)
 892                 return 0;
 893
 894         /* relocatable section is data or instructions */
 895         if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
 896                 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
 897                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 898                 return -EINVAL;
 899         }
 900
 901         /* check sanity of each relocation */
 902         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 903         relo = sec->data->d_buf;
 904         sym_sec = &obj->secs[obj->symtab_sec_idx];
 905         for (i = 0; i < n; i++, relo++) {
 906                 size_t sym_idx = ELF64_R_SYM(relo->r_info);
 907                 size_t sym_type = ELF64_R_TYPE(relo->r_info);
 908
 909                 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
 910                     sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
 911                         pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
 912                                 i, sec->sec_idx, sym_type, obj->filename);
 913                         return -EINVAL;
 914                 }
 915
 916                 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
 917                         pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
 918                                 i, sec->sec_idx, sym_idx, obj->filename);
 919                         return -EINVAL;
 920                 }
 921
 922                 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
 923                         if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
 924                                 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
 925                                         i, sec->sec_idx, sym_idx, obj->filename);
 926                                 return -EINVAL;
 927                         }
 928                 }
 929         }
 930
 931         return 0;
 932 }
 933
 934 static int check_btf_type_id(__u32 *type_id, void *ctx)
 935 {
 936         struct btf *btf = ctx;
 937
 938         if (*type_id >= btf__type_cnt(btf))
 939                 return -EINVAL;
 940
 941         return 0;
 942 }
 943
 944 static int check_btf_str_off(__u32 *str_off, void *ctx)
 945 {
 946         struct btf *btf = ctx;
 947         const char *s;
 948
 949         s = btf__str_by_offset(btf, *str_off);
 950
 951         if (!s)
 952                 return -EINVAL;
 953
 954         return 0;
 955 }
 956
 957 static int linker_sanity_check_btf(struct src_obj *obj)
 958 {
 959         struct btf_type *t;
 960         int i, n, err = 0;
 961
 962         if (!obj->btf)
 963                 return 0;
 964
 965         n = btf__type_cnt(obj->btf);
 966         for (i = 1; i < n; i++) {
 967                 t = btf_type_by_id(obj->btf, i);
 968
 969                 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
 970                 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
 971                 if (err)
 972                         return err;
 973         }
 974
 975         return 0;
 976 }
 977
 978 static int linker_sanity_check_btf_ext(struct src_obj *obj)
 979 {
 980         int err = 0;
 981
 982         if (!obj->btf_ext)
 983                 return 0;
 984
 985         /* can't use .BTF.ext without .BTF */
 986         if (!obj->btf)
 987                 return -EINVAL;
 988
 989         err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
 990         err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
 991         if (err)
 992                 return err;
 993
 994         return 0;
 995 }
 996
 997 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
 998 {
 999         Elf_Scn *scn;
1000         Elf_Data *data;
1001         Elf64_Shdr *shdr;
1002         int name_off;
1003
1004         dst_sec->sec_sz = 0;
1005         dst_sec->sec_idx = 0;
1006         dst_sec->ephemeral = src_sec->ephemeral;
1007
1008         /* ephemeral sections are just thin section shells lacking most parts */
1009         if (src_sec->ephemeral)
1010                 return 0;
1011
1012         scn = elf_newscn(linker->elf);
1013         if (!scn)
1014                 return -ENOMEM;
1015         data = elf_newdata(scn);
1016         if (!data)
1017                 return -ENOMEM;
1018         shdr = elf64_getshdr(scn);
1019         if (!shdr)
1020                 return -ENOMEM;
1021
1022         dst_sec->scn = scn;
1023         dst_sec->shdr = shdr;
1024         dst_sec->data = data;
1025         dst_sec->sec_idx = elf_ndxscn(scn);
1026
1027         name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1028         if (name_off < 0)
1029                 return name_off;
1030
1031         shdr->sh_name = name_off;
1032         shdr->sh_type = src_sec->shdr->sh_type;
1033         shdr->sh_flags = src_sec->shdr->sh_flags;
1034         shdr->sh_size = 0;
1035         /* sh_link and sh_info have different meaning for different types of
1036          * sections, so we leave it up to the caller code to fill them in, if
1037          * necessary
1038          */
1039         shdr->sh_link = 0;
1040         shdr->sh_info = 0;
1041         shdr->sh_addralign = src_sec->shdr->sh_addralign;
1042         shdr->sh_entsize = src_sec->shdr->sh_entsize;
1043
1044         data->d_type = src_sec->data->d_type;
1045         data->d_size = 0;
1046         data->d_buf = NULL;
1047         data->d_align = src_sec->data->d_align;
1048         data->d_off = 0;
1049
1050         return 0;
1051 }
1052
1053 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1054 {
1055         struct dst_sec *sec;
1056         int i;
1057
1058         for (i = 1; i < linker->sec_cnt; i++) {
1059                 sec = &linker->secs[i];
1060
1061                 if (strcmp(sec->sec_name, sec_name) == 0)
1062                         return sec;
1063         }
1064
1065         return NULL;
1066 }
1067
1068 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1069 {
1070         if (dst->ephemeral || src->ephemeral)
1071                 return true;
1072
1073         if (dst->shdr->sh_type != src->shdr->sh_type) {
1074                 pr_warn("sec %s types mismatch\n", dst->sec_name);
1075                 return false;
1076         }
1077         if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1078                 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1079                 return false;
1080         }
1081         if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1082                 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1083                 return false;
1084         }
1085
1086         return true;
1087 }
1088
1089 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1090 {
1091         if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1092                 return false;
1093         if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1094                 return false;
1095         return true;
1096 }
1097
1098 static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1099 {
1100         void *tmp;
1101         size_t dst_align, src_align;
1102         size_t dst_align_sz, dst_final_sz;
1103         int err;
1104
1105         /* Ephemeral source section doesn't contribute anything to ELF
1106          * section data.
1107          */
1108         if (src->ephemeral)
1109                 return 0;
1110
1111         /* Some sections (like .maps) can contain both externs (and thus be
1112          * ephemeral) and non-externs (map definitions). So it's possible that
1113          * it has to be "upgraded" from ephemeral to non-ephemeral when the
1114          * first non-ephemeral entity appears. In such case, we add ELF
1115          * section, data, etc.
1116          */
1117         if (dst->ephemeral) {
1118                 err = init_sec(linker, dst, src);
1119                 if (err)
1120                         return err;
1121         }
1122
1123         dst_align = dst->shdr->sh_addralign;
1124         src_align = src->shdr->sh_addralign;
1125         if (dst_align == 0)
1126                 dst_align = 1;
1127         if (dst_align < src_align)
1128                 dst_align = src_align;
1129
1130         dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1131
1132         /* no need to re-align final size */
1133         dst_final_sz = dst_align_sz + src->shdr->sh_size;
1134
1135         if (src->shdr->sh_type != SHT_NOBITS) {
1136                 tmp = realloc(dst->raw_data, dst_final_sz);
1137                 /* If dst_align_sz == 0, realloc() behaves in a special way:
1138                  * 1. When dst->raw_data is NULL it returns:
1139                  *    "either NULL or a pointer suitable to be passed to free()" [1].
1140                  * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1141                  *    thus invalidating any "pointer suitable to be passed to free()" obtained
1142                  *    at step (1).
1143                  *
1144                  * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1145                  * dst->raw_data would be freed again in bpf_linker__free().
1146                  *
1147                  * [1] man 3 realloc
1148                  */
1149                 if (!tmp && dst_align_sz > 0)
1150                         return -ENOMEM;
1151                 dst->raw_data = tmp;
1152
1153                 /* pad dst section, if it's alignment forced size increase */
1154                 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1155                 /* now copy src data at a properly aligned offset */
1156                 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1157         }
1158
1159         dst->sec_sz = dst_final_sz;
1160         dst->shdr->sh_size = dst_final_sz;
1161         dst->data->d_size = dst_final_sz;
1162
1163         dst->shdr->sh_addralign = dst_align;
1164         dst->data->d_align = dst_align;
1165
1166         src->dst_off = dst_align_sz;
1167
1168         return 0;
1169 }
1170
1171 static bool is_data_sec(struct src_sec *sec)
1172 {
1173         if (!sec || sec->skipped)
1174                 return false;
1175         /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1176         if (sec->ephemeral)
1177                 return true;
1178         return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1179 }
1180
1181 static bool is_relo_sec(struct src_sec *sec)
1182 {
1183         if (!sec || sec->skipped || sec->ephemeral)
1184                 return false;
1185         return sec->shdr->sh_type == SHT_REL;
1186 }
1187
1188 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1189 {
1190         int i, err;
1191
1192         for (i = 1; i < obj->sec_cnt; i++) {
1193                 struct src_sec *src_sec;
1194                 struct dst_sec *dst_sec;
1195
1196                 src_sec = &obj->secs[i];
1197                 if (!is_data_sec(src_sec))
1198                         continue;
1199
1200                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1201                 if (!dst_sec) {
1202                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1203                         if (!dst_sec)
1204                                 return -ENOMEM;
1205                         err = init_sec(linker, dst_sec, src_sec);
1206                         if (err) {
1207                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1208                                 return err;
1209                         }
1210                 } else {
1211                         if (!secs_match(dst_sec, src_sec)) {
1212                                 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1213                                 return -1;
1214                         }
1215
1216                         /* "license" and "version" sections are deduped */
1217                         if (strcmp(src_sec->sec_name, "license") == 0
1218                             || strcmp(src_sec->sec_name, "version") == 0) {
1219                                 if (!sec_content_is_same(dst_sec, src_sec)) {
1220                                         pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1221                                         return -EINVAL;
1222                                 }
1223                                 src_sec->skipped = true;
1224                                 src_sec->dst_id = dst_sec->id;
1225                                 continue;
1226                         }
1227                 }
1228
1229                 /* record mapped section index */
1230                 src_sec->dst_id = dst_sec->id;
1231
1232                 err = extend_sec(linker, dst_sec, src_sec);
1233                 if (err)
1234                         return err;
1235         }
1236
1237         return 0;
1238 }
1239
1240 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1241 {
1242         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1243         Elf64_Sym *sym = symtab->data->d_buf;
1244         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1245         int str_sec_idx = symtab->shdr->sh_link;
1246         const char *sym_name;
1247
1248         obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1249         if (!obj->sym_map)
1250                 return -ENOMEM;
1251
1252         for (i = 0; i < n; i++, sym++) {
1253                 /* We already validated all-zero symbol #0 and we already
1254                  * appended it preventively to the final SYMTAB, so skip it.
1255                  */
1256                 if (i == 0)
1257                         continue;
1258
1259                 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1260                 if (!sym_name) {
1261                         pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1262                         return -EINVAL;
1263                 }
1264
1265                 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1266                 if (err)
1267                         return err;
1268         }
1269
1270         return 0;
1271 }
1272
1273 static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1274 {
1275         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1276         Elf64_Sym *syms = symtab->raw_data;
1277
1278         return &syms[sym_idx];
1279 }
1280
1281 static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1282 {
1283         struct glob_sym *glob_sym;
1284         const char *name;
1285         int i;
1286
1287         for (i = 0; i < linker->glob_sym_cnt; i++) {
1288                 glob_sym = &linker->glob_syms[i];
1289                 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1290
1291                 if (strcmp(name, sym_name) == 0)
1292                         return glob_sym;
1293         }
1294
1295         return NULL;
1296 }
1297
1298 static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1299 {
1300         struct glob_sym *syms, *sym;
1301
1302         syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1303                                    sizeof(*linker->glob_syms));
1304         if (!syms)
1305                 return NULL;
1306
1307         sym = &syms[linker->glob_sym_cnt];
1308         memset(sym, 0, sizeof(*sym));
1309         sym->var_idx = -1;
1310
1311         linker->glob_syms = syms;
1312         linker->glob_sym_cnt++;
1313
1314         return sym;
1315 }
1316
1317 static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1318                                  const struct btf *btf1, __u32 id1,
1319                                  const struct btf *btf2, __u32 id2)
1320 {
1321         const struct btf_type *t1, *t2;
1322         bool is_static1, is_static2;
1323         const char *n1, *n2;
1324         int i, n;
1325
1326 recur:
1327         n1 = n2 = NULL;
1328         t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1329         t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1330
1331         /* check if only one side is FWD, otherwise handle with common logic */
1332         if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1333                 n1 = btf__str_by_offset(btf1, t1->name_off);
1334                 n2 = btf__str_by_offset(btf2, t2->name_off);
1335                 if (strcmp(n1, n2) != 0) {
1336                         pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1337                                 sym_name, n1, n2);
1338                         return false;
1339                 }
1340                 /* validate if FWD kind matches concrete kind */
1341                 if (btf_is_fwd(t1)) {
1342                         if (btf_kflag(t1) && btf_is_union(t2))
1343                                 return true;
1344                         if (!btf_kflag(t1) && btf_is_struct(t2))
1345                                 return true;
1346                         pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1347                                 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1348                 } else {
1349                         if (btf_kflag(t2) && btf_is_union(t1))
1350                                 return true;
1351                         if (!btf_kflag(t2) && btf_is_struct(t1))
1352                                 return true;
1353                         pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1354                                 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1355                 }
1356                 return false;
1357         }
1358
1359         if (btf_kind(t1) != btf_kind(t2)) {
1360                 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1361                         sym_name, btf_kind_str(t1), btf_kind_str(t2));
1362                 return false;
1363         }
1364
1365         switch (btf_kind(t1)) {
1366         case BTF_KIND_STRUCT:
1367         case BTF_KIND_UNION:
1368         case BTF_KIND_ENUM:
1369         case BTF_KIND_ENUM64:
1370         case BTF_KIND_FWD:
1371         case BTF_KIND_FUNC:
1372         case BTF_KIND_VAR:
1373                 n1 = btf__str_by_offset(btf1, t1->name_off);
1374                 n2 = btf__str_by_offset(btf2, t2->name_off);
1375                 if (strcmp(n1, n2) != 0) {
1376                         pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1377                                 sym_name, btf_kind_str(t1), n1, n2);
1378                         return false;
1379                 }
1380                 break;
1381         default:
1382                 break;
1383         }
1384
1385         switch (btf_kind(t1)) {
1386         case BTF_KIND_UNKN: /* void */
1387         case BTF_KIND_FWD:
1388                 return true;
1389         case BTF_KIND_INT:
1390         case BTF_KIND_FLOAT:
1391         case BTF_KIND_ENUM:
1392         case BTF_KIND_ENUM64:
1393                 /* ignore encoding for int and enum values for enum */
1394                 if (t1->size != t2->size) {
1395                         pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1396                                 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1397                         return false;
1398                 }
1399                 return true;
1400         case BTF_KIND_PTR:
1401                 /* just validate overall shape of the referenced type, so no
1402                  * contents comparison for struct/union, and allowd fwd vs
1403                  * struct/union
1404                  */
1405                 exact = false;
1406                 id1 = t1->type;
1407                 id2 = t2->type;
1408                 goto recur;
1409         case BTF_KIND_ARRAY:
1410                 /* ignore index type and array size */
1411                 id1 = btf_array(t1)->type;
1412                 id2 = btf_array(t2)->type;
1413                 goto recur;
1414         case BTF_KIND_FUNC:
1415                 /* extern and global linkages are compatible */
1416                 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1417                 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1418                 if (is_static1 != is_static2) {
1419                         pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1420                         return false;
1421                 }
1422
1423                 id1 = t1->type;
1424                 id2 = t2->type;
1425                 goto recur;
1426         case BTF_KIND_VAR:
1427                 /* extern and global linkages are compatible */
1428                 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1429                 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1430                 if (is_static1 != is_static2) {
1431                         pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1432                         return false;
1433                 }
1434
1435                 id1 = t1->type;
1436                 id2 = t2->type;
1437                 goto recur;
1438         case BTF_KIND_STRUCT:
1439         case BTF_KIND_UNION: {
1440                 const struct btf_member *m1, *m2;
1441
1442                 if (!exact)
1443                         return true;
1444
1445                 if (btf_vlen(t1) != btf_vlen(t2)) {
1446                         pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1447                                 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1448                         return false;
1449                 }
1450
1451                 n = btf_vlen(t1);
1452                 m1 = btf_members(t1);
1453                 m2 = btf_members(t2);
1454                 for (i = 0; i < n; i++, m1++, m2++) {
1455                         n1 = btf__str_by_offset(btf1, m1->name_off);
1456                         n2 = btf__str_by_offset(btf2, m2->name_off);
1457                         if (strcmp(n1, n2) != 0) {
1458                                 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1459                                         sym_name, i, n1, n2);
1460                                 return false;
1461                         }
1462                         if (m1->offset != m2->offset) {
1463                                 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1464                                         sym_name, i, n1);
1465                                 return false;
1466                         }
1467                         if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1468                                 return false;
1469                 }
1470
1471                 return true;
1472         }
1473         case BTF_KIND_FUNC_PROTO: {
1474                 const struct btf_param *m1, *m2;
1475
1476                 if (btf_vlen(t1) != btf_vlen(t2)) {
1477                         pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1478                                 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1479                         return false;
1480                 }
1481
1482                 n = btf_vlen(t1);
1483                 m1 = btf_params(t1);
1484                 m2 = btf_params(t2);
1485                 for (i = 0; i < n; i++, m1++, m2++) {
1486                         /* ignore func arg names */
1487                         if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1488                                 return false;
1489                 }
1490
1491                 /* now check return type as well */
1492                 id1 = t1->type;
1493                 id2 = t2->type;
1494                 goto recur;
1495         }
1496
1497         /* skip_mods_and_typedefs() make this impossible */
1498         case BTF_KIND_TYPEDEF:
1499         case BTF_KIND_VOLATILE:
1500         case BTF_KIND_CONST:
1501         case BTF_KIND_RESTRICT:
1502         /* DATASECs are never compared with each other */
1503         case BTF_KIND_DATASEC:
1504         default:
1505                 pr_warn("global '%s': unsupported BTF kind %s\n",
1506                         sym_name, btf_kind_str(t1));
1507                 return false;
1508         }
1509 }
1510
1511 static bool map_defs_match(const char *sym_name,
1512                            const struct btf *main_btf,
1513                            const struct btf_map_def *main_def,
1514                            const struct btf_map_def *main_inner_def,
1515                            const struct btf *extra_btf,
1516                            const struct btf_map_def *extra_def,
1517                            const struct btf_map_def *extra_inner_def)
1518 {
1519         const char *reason;
1520
1521         if (main_def->map_type != extra_def->map_type) {
1522                 reason = "type";
1523                 goto mismatch;
1524         }
1525
1526         /* check key type/size match */
1527         if (main_def->key_size != extra_def->key_size) {
1528                 reason = "key_size";
1529                 goto mismatch;
1530         }
1531         if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1532                 reason = "key type";
1533                 goto mismatch;
1534         }
1535         if ((main_def->parts & MAP_DEF_KEY_TYPE)
1536              && !glob_sym_btf_matches(sym_name, true /*exact*/,
1537                                       main_btf, main_def->key_type_id,
1538                                       extra_btf, extra_def->key_type_id)) {
1539                 reason = "key type";
1540                 goto mismatch;
1541         }
1542
1543         /* validate value type/size match */
1544         if (main_def->value_size != extra_def->value_size) {
1545                 reason = "value_size";
1546                 goto mismatch;
1547         }
1548         if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1549                 reason = "value type";
1550                 goto mismatch;
1551         }
1552         if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1553              && !glob_sym_btf_matches(sym_name, true /*exact*/,
1554                                       main_btf, main_def->value_type_id,
1555                                       extra_btf, extra_def->value_type_id)) {
1556                 reason = "key type";
1557                 goto mismatch;
1558         }
1559
1560         if (main_def->max_entries != extra_def->max_entries) {
1561                 reason = "max_entries";
1562                 goto mismatch;
1563         }
1564         if (main_def->map_flags != extra_def->map_flags) {
1565                 reason = "map_flags";
1566                 goto mismatch;
1567         }
1568         if (main_def->numa_node != extra_def->numa_node) {
1569                 reason = "numa_node";
1570                 goto mismatch;
1571         }
1572         if (main_def->pinning != extra_def->pinning) {
1573                 reason = "pinning";
1574                 goto mismatch;
1575         }
1576
1577         if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1578                 reason = "inner map";
1579                 goto mismatch;
1580         }
1581
1582         if (main_def->parts & MAP_DEF_INNER_MAP) {
1583                 char inner_map_name[128];
1584
1585                 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1586
1587                 return map_defs_match(inner_map_name,
1588                                       main_btf, main_inner_def, NULL,
1589                                       extra_btf, extra_inner_def, NULL);
1590         }
1591
1592         return true;
1593
1594 mismatch:
1595         pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1596         return false;
1597 }
1598
1599 static bool glob_map_defs_match(const char *sym_name,
1600                                 struct bpf_linker *linker, struct glob_sym *glob_sym,
1601                                 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1602 {
1603         struct btf_map_def dst_def = {}, dst_inner_def = {};
1604         struct btf_map_def src_def = {}, src_inner_def = {};
1605         const struct btf_type *t;
1606         int err;
1607
1608         t = btf__type_by_id(obj->btf, btf_id);
1609         if (!btf_is_var(t)) {
1610                 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1611                 return false;
1612         }
1613         t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1614
1615         err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1616         if (err) {
1617                 pr_warn("global '%s': invalid map definition\n", sym_name);
1618                 return false;
1619         }
1620
1621         /* re-parse existing map definition */
1622         t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1623         t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1624         err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1625         if (err) {
1626                 /* this should not happen, because we already validated it */
1627                 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1628                 return false;
1629         }
1630
1631         /* Currently extern map definition has to be complete and match
1632          * concrete map definition exactly. This restriction might be lifted
1633          * in the future.
1634          */
1635         return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1636                               obj->btf, &src_def, &src_inner_def);
1637 }
1638
1639 static bool glob_syms_match(const char *sym_name,
1640                             struct bpf_linker *linker, struct glob_sym *glob_sym,
1641                             struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1642 {
1643         const struct btf_type *src_t;
1644
1645         /* if we are dealing with externs, BTF types describing both global
1646          * and extern VARs/FUNCs should be completely present in all files
1647          */
1648         if (!glob_sym->btf_id || !btf_id) {
1649                 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1650                 return false;
1651         }
1652
1653         src_t = btf__type_by_id(obj->btf, btf_id);
1654         if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1655                 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1656                         btf_kind_str(src_t), sym_name);
1657                 return false;
1658         }
1659
1660         /* deal with .maps definitions specially */
1661         if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1662                 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1663
1664         if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1665                                   linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1666                 return false;
1667
1668         return true;
1669 }
1670
1671 static bool btf_is_non_static(const struct btf_type *t)
1672 {
1673         return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1674                || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1675 }
1676
1677 static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1678                              int *out_btf_sec_id, int *out_btf_id)
1679 {
1680         int i, j, n, m, btf_id = 0;
1681         const struct btf_type *t;
1682         const struct btf_var_secinfo *vi;
1683         const char *name;
1684
1685         if (!obj->btf) {
1686                 pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1687                 return -EINVAL;
1688         }
1689
1690         n = btf__type_cnt(obj->btf);
1691         for (i = 1; i < n; i++) {
1692                 t = btf__type_by_id(obj->btf, i);
1693
1694                 /* some global and extern FUNCs and VARs might not be associated with any
1695                  * DATASEC, so try to detect them in the same pass
1696                  */
1697                 if (btf_is_non_static(t)) {
1698                         name = btf__str_by_offset(obj->btf, t->name_off);
1699                         if (strcmp(name, sym_name) != 0)
1700                                 continue;
1701
1702                         /* remember and still try to find DATASEC */
1703                         btf_id = i;
1704                         continue;
1705                 }
1706
1707                 if (!btf_is_datasec(t))
1708                         continue;
1709
1710                 vi = btf_var_secinfos(t);
1711                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1712                         t = btf__type_by_id(obj->btf, vi->type);
1713                         name = btf__str_by_offset(obj->btf, t->name_off);
1714
1715                         if (strcmp(name, sym_name) != 0)
1716                                 continue;
1717                         if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1718                                 continue;
1719                         if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1720                                 continue;
1721
1722                         if (btf_id && btf_id != vi->type) {
1723                                 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1724                                         sym_name, btf_id, vi->type);
1725                                 return -EINVAL;
1726                         }
1727
1728                         *out_btf_sec_id = i;
1729                         *out_btf_id = vi->type;
1730
1731                         return 0;
1732                 }
1733         }
1734
1735         /* free-floating extern or global FUNC */
1736         if (btf_id) {
1737                 *out_btf_sec_id = 0;
1738                 *out_btf_id = btf_id;
1739                 return 0;
1740         }
1741
1742         pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1743         return -ENOENT;
1744 }
1745
1746 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1747 {
1748         struct src_sec *sec;
1749         int i;
1750
1751         for (i = 1; i < obj->sec_cnt; i++) {
1752                 sec = &obj->secs[i];
1753
1754                 if (strcmp(sec->sec_name, sec_name) == 0)
1755                         return sec;
1756         }
1757
1758         return NULL;
1759 }
1760
1761 static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1762                                     struct btf *src_btf, int src_id)
1763 {
1764         struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1765         struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1766         struct btf_param *src_p, *dst_p;
1767         const char *s;
1768         int i, n, off;
1769
1770         /* We already made sure that source and destination types (FUNC or
1771          * VAR) match in terms of types and argument names.
1772          */
1773         if (btf_is_var(dst_t)) {
1774                 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1775                 return 0;
1776         }
1777
1778         dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1779
1780         /* now onto FUNC_PROTO types */
1781         src_t = btf_type_by_id(src_btf, src_t->type);
1782         dst_t = btf_type_by_id(dst_btf, dst_t->type);
1783
1784         /* Fill in all the argument names, which for extern FUNCs are missing.
1785          * We'll end up with two copies of FUNCs/VARs for externs, but that
1786          * will be taken care of by BTF dedup at the very end.
1787          * It might be that BTF types for extern in one file has less/more BTF
1788          * information (e.g., FWD instead of full STRUCT/UNION information),
1789          * but that should be (in most cases, subject to BTF dedup rules)
1790          * handled and resolved by BTF dedup algorithm as well, so we won't
1791          * worry about it. Our only job is to make sure that argument names
1792          * are populated on both sides, otherwise BTF dedup will pedantically
1793          * consider them different.
1794          */
1795         src_p = btf_params(src_t);
1796         dst_p = btf_params(dst_t);
1797         for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1798                 if (!src_p->name_off)
1799                         continue;
1800
1801                 /* src_btf has more complete info, so add name to dst_btf */
1802                 s = btf__str_by_offset(src_btf, src_p->name_off);
1803                 off = btf__add_str(dst_btf, s);
1804                 if (off < 0)
1805                         return off;
1806                 dst_p->name_off = off;
1807         }
1808         return 0;
1809 }
1810
1811 static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1812 {
1813         sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1814 }
1815
1816 static void sym_update_type(Elf64_Sym *sym, int sym_type)
1817 {
1818         sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1819 }
1820
1821 static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1822 {
1823         /* libelf doesn't provide setters for ST_VISIBILITY,
1824          * but it is stored in the lower 2 bits of st_other
1825          */
1826         sym->st_other &= ~0x03;
1827         sym->st_other |= sym_vis;
1828 }
1829
1830 static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1831                                  Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1832 {
1833         struct src_sec *src_sec = NULL;
1834         struct dst_sec *dst_sec = NULL;
1835         struct glob_sym *glob_sym = NULL;
1836         int name_off, sym_type, sym_bind, sym_vis, err;
1837         int btf_sec_id = 0, btf_id = 0;
1838         size_t dst_sym_idx;
1839         Elf64_Sym *dst_sym;
1840         bool sym_is_extern;
1841
1842         sym_type = ELF64_ST_TYPE(sym->st_info);
1843         sym_bind = ELF64_ST_BIND(sym->st_info);
1844         sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1845         sym_is_extern = sym->st_shndx == SHN_UNDEF;
1846
1847         if (sym_is_extern) {
1848                 if (!obj->btf) {
1849                         pr_warn("externs without BTF info are not supported\n");
1850                         return -ENOTSUP;
1851                 }
1852         } else if (sym->st_shndx < SHN_LORESERVE) {
1853                 src_sec = &obj->secs[sym->st_shndx];
1854                 if (src_sec->skipped)
1855                         return 0;
1856                 dst_sec = &linker->secs[src_sec->dst_id];
1857
1858                 /* allow only one STT_SECTION symbol per section */
1859                 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1860                         obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1861                         return 0;
1862                 }
1863         }
1864
1865         if (sym_bind == STB_LOCAL)
1866                 goto add_sym;
1867
1868         /* find matching BTF info */
1869         err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1870         if (err)
1871                 return err;
1872
1873         if (sym_is_extern && btf_sec_id) {
1874                 const char *sec_name = NULL;
1875                 const struct btf_type *t;
1876
1877                 t = btf__type_by_id(obj->btf, btf_sec_id);
1878                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1879
1880                 /* Clang puts unannotated extern vars into
1881                  * '.extern' BTF DATASEC. Treat them the same
1882                  * as unannotated extern funcs (which are
1883                  * currently not put into any DATASECs).
1884                  * Those don't have associated src_sec/dst_sec.
1885                  */
1886                 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1887                         src_sec = find_src_sec_by_name(obj, sec_name);
1888                         if (!src_sec) {
1889                                 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1890                                 return -ENOENT;
1891                         }
1892                         dst_sec = &linker->secs[src_sec->dst_id];
1893                 }
1894         }
1895
1896         glob_sym = find_glob_sym(linker, sym_name);
1897         if (glob_sym) {
1898                 /* Preventively resolve to existing symbol. This is
1899                  * needed for further relocation symbol remapping in
1900                  * the next step of linking.
1901                  */
1902                 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1903
1904                 /* If both symbols are non-externs, at least one of
1905                  * them has to be STB_WEAK, otherwise they are in
1906                  * a conflict with each other.
1907                  */
1908                 if (!sym_is_extern && !glob_sym->is_extern
1909                     && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1910                         pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1911                                 src_sym_idx, sym_name, obj->filename);
1912                         return -EINVAL;
1913                 }
1914
1915                 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1916                         return -EINVAL;
1917
1918                 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1919
1920                 /* If new symbol is strong, then force dst_sym to be strong as
1921                  * well; this way a mix of weak and non-weak extern
1922                  * definitions will end up being strong.
1923                  */
1924                 if (sym_bind == STB_GLOBAL) {
1925                         /* We still need to preserve type (NOTYPE or
1926                          * OBJECT/FUNC, depending on whether the symbol is
1927                          * extern or not)
1928                          */
1929                         sym_update_bind(dst_sym, STB_GLOBAL);
1930                         glob_sym->is_weak = false;
1931                 }
1932
1933                 /* Non-default visibility is "contaminating", with stricter
1934                  * visibility overwriting more permissive ones, even if more
1935                  * permissive visibility comes from just an extern definition.
1936                  * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
1937                  * ensured by ELF symbol sanity checks above.
1938                  */
1939                 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1940                         sym_update_visibility(dst_sym, sym_vis);
1941
1942                 /* If the new symbol is extern, then regardless if
1943                  * existing symbol is extern or resolved global, just
1944                  * keep the existing one untouched.
1945                  */
1946                 if (sym_is_extern)
1947                         return 0;
1948
1949                 /* If existing symbol is a strong resolved symbol, bail out,
1950                  * because we lost resolution battle have nothing to
1951                  * contribute. We already checked abover that there is no
1952                  * strong-strong conflict. We also already tightened binding
1953                  * and visibility, so nothing else to contribute at that point.
1954                  */
1955                 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1956                         return 0;
1957
1958                 /* At this point, new symbol is strong non-extern,
1959                  * so overwrite glob_sym with new symbol information.
1960                  * Preserve binding and visibility.
1961                  */
1962                 sym_update_type(dst_sym, sym_type);
1963                 dst_sym->st_shndx = dst_sec->sec_idx;
1964                 dst_sym->st_value = src_sec->dst_off + sym->st_value;
1965                 dst_sym->st_size = sym->st_size;
1966
1967                 /* see comment below about dst_sec->id vs dst_sec->sec_idx */
1968                 glob_sym->sec_id = dst_sec->id;
1969                 glob_sym->is_extern = false;
1970
1971                 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1972                                              obj->btf, btf_id))
1973                         return -EINVAL;
1974
1975                 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
1976                 glob_sym->underlying_btf_id = 0;
1977
1978                 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1979                 return 0;
1980         }
1981
1982 add_sym:
1983         name_off = strset__add_str(linker->strtab_strs, sym_name);
1984         if (name_off < 0)
1985                 return name_off;
1986
1987         dst_sym = add_new_sym(linker, &dst_sym_idx);
1988         if (!dst_sym)
1989                 return -ENOMEM;
1990
1991         dst_sym->st_name = name_off;
1992         dst_sym->st_info = sym->st_info;
1993         dst_sym->st_other = sym->st_other;
1994         dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1995         dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1996         dst_sym->st_size = sym->st_size;
1997
1998         obj->sym_map[src_sym_idx] = dst_sym_idx;
1999
2000         if (sym_type == STT_SECTION && dst_sym) {
2001                 dst_sec->sec_sym_idx = dst_sym_idx;
2002                 dst_sym->st_value = 0;
2003         }
2004
2005         if (sym_bind != STB_LOCAL) {
2006                 glob_sym = add_glob_sym(linker);
2007                 if (!glob_sym)
2008                         return -ENOMEM;
2009
2010                 glob_sym->sym_idx = dst_sym_idx;
2011                 /* we use dst_sec->id (and not dst_sec->sec_idx), because
2012                  * ephemeral sections (.kconfig, .ksyms, etc) don't have
2013                  * sec_idx (as they don't have corresponding ELF section), but
2014                  * still have id. .extern doesn't have even ephemeral section
2015                  * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2016                  */
2017                 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2018                 glob_sym->name_off = name_off;
2019                 /* we will fill btf_id in during BTF merging step */
2020                 glob_sym->btf_id = 0;
2021                 glob_sym->is_extern = sym_is_extern;
2022                 glob_sym->is_weak = sym_bind == STB_WEAK;
2023         }
2024
2025         return 0;
2026 }
2027
2028 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2029 {
2030         struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2031         int i, err;
2032
2033         for (i = 1; i < obj->sec_cnt; i++) {
2034                 struct src_sec *src_sec, *src_linked_sec;
2035                 struct dst_sec *dst_sec, *dst_linked_sec;
2036                 Elf64_Rel *src_rel, *dst_rel;
2037                 int j, n;
2038
2039                 src_sec = &obj->secs[i];
2040                 if (!is_relo_sec(src_sec))
2041                         continue;
2042
2043                 /* shdr->sh_info points to relocatable section */
2044                 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2045                 if (src_linked_sec->skipped)
2046                         continue;
2047
2048                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2049                 if (!dst_sec) {
2050                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
2051                         if (!dst_sec)
2052                                 return -ENOMEM;
2053                         err = init_sec(linker, dst_sec, src_sec);
2054                         if (err) {
2055                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2056                                 return err;
2057                         }
2058                 } else if (!secs_match(dst_sec, src_sec)) {
2059                         pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2060                         return -1;
2061                 }
2062
2063                 /* shdr->sh_link points to SYMTAB */
2064                 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2065
2066                 /* shdr->sh_info points to relocated section */
2067                 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2068                 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2069
2070                 src_sec->dst_id = dst_sec->id;
2071                 err = extend_sec(linker, dst_sec, src_sec);
2072                 if (err)
2073                         return err;
2074
2075                 src_rel = src_sec->data->d_buf;
2076                 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2077                 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2078                 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2079                         size_t src_sym_idx, dst_sym_idx, sym_type;
2080                         Elf64_Sym *src_sym;
2081
2082                         src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2083                         src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2084
2085                         dst_sym_idx = obj->sym_map[src_sym_idx];
2086                         dst_rel->r_offset += src_linked_sec->dst_off;
2087                         sym_type = ELF64_R_TYPE(src_rel->r_info);
2088                         dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2089
2090                         if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2091                                 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2092                                 struct bpf_insn *insn;
2093
2094                                 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2095                                         /* calls to the very first static function inside
2096                                          * .text section at offset 0 will
2097                                          * reference section symbol, not the
2098                                          * function symbol. Fix that up,
2099                                          * otherwise it won't be possible to
2100                                          * relocate calls to two different
2101                                          * static functions with the same name
2102                                          * (rom two different object files)
2103                                          */
2104                                         insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2105                                         if (insn->code == (BPF_JMP | BPF_CALL))
2106                                                 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2107                                         else
2108                                                 insn->imm += sec->dst_off;
2109                                 } else {
2110                                         pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2111                                         return -EINVAL;
2112                                 }
2113                         }
2114
2115                 }
2116         }
2117
2118         return 0;
2119 }
2120
2121 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2122                                    int sym_type, const char *sym_name)
2123 {
2124         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2125         Elf64_Sym *sym = symtab->data->d_buf;
2126         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2127         int str_sec_idx = symtab->shdr->sh_link;
2128         const char *name;
2129
2130         for (i = 0; i < n; i++, sym++) {
2131                 if (sym->st_shndx != sec_idx)
2132                         continue;
2133                 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2134                         continue;
2135
2136                 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2137                 if (!name)
2138                         return NULL;
2139
2140                 if (strcmp(sym_name, name) != 0)
2141                         continue;
2142
2143                 return sym;
2144         }
2145
2146         return NULL;
2147 }
2148
2149 static int linker_fixup_btf(struct src_obj *obj)
2150 {
2151         const char *sec_name;
2152         struct src_sec *sec;
2153         int i, j, n, m;
2154
2155         if (!obj->btf)
2156                 return 0;
2157
2158         n = btf__type_cnt(obj->btf);
2159         for (i = 1; i < n; i++) {
2160                 struct btf_var_secinfo *vi;
2161                 struct btf_type *t;
2162
2163                 t = btf_type_by_id(obj->btf, i);
2164                 if (btf_kind(t) != BTF_KIND_DATASEC)
2165                         continue;
2166
2167                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2168                 sec = find_src_sec_by_name(obj, sec_name);
2169                 if (sec) {
2170                         /* record actual section size, unless ephemeral */
2171                         if (sec->shdr)
2172                                 t->size = sec->shdr->sh_size;
2173                 } else {
2174                         /* BTF can have some sections that are not represented
2175                          * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2176                          * for special extern variables.
2177                          *
2178                          * For all but one such special (ephemeral)
2179                          * sections, we pre-create "section shells" to be able
2180                          * to keep track of extra per-section metadata later
2181                          * (e.g., those BTF extern variables).
2182                          *
2183                          * .extern is even more special, though, because it
2184                          * contains extern variables that need to be resolved
2185                          * by static linker, not libbpf and kernel. When such
2186                          * externs are resolved, we are going to remove them
2187                          * from .extern BTF section and might end up not
2188                          * needing it at all. Each resolved extern should have
2189                          * matching non-extern VAR/FUNC in other sections.
2190                          *
2191                          * We do support leaving some of the externs
2192                          * unresolved, though, to support cases of building
2193                          * libraries, which will later be linked against final
2194                          * BPF applications. So if at finalization we still
2195                          * see unresolved externs, we'll create .extern
2196                          * section on our own.
2197                          */
2198                         if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2199                                 continue;
2200
2201                         sec = add_src_sec(obj, sec_name);
2202                         if (!sec)
2203                                 return -ENOMEM;
2204
2205                         sec->ephemeral = true;
2206                         sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2207                 }
2208
2209                 /* remember ELF section and its BTF type ID match */
2210                 sec->sec_type_id = i;
2211
2212                 /* fix up variable offsets */
2213                 vi = btf_var_secinfos(t);
2214                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2215                         const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2216                         const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
2217                         int var_linkage = btf_var(vt)->linkage;
2218                         Elf64_Sym *sym;
2219
2220                         /* no need to patch up static or extern vars */
2221                         if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2222                                 continue;
2223
2224                         sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2225                         if (!sym) {
2226                                 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2227                                 return -ENOENT;
2228                         }
2229
2230                         vi->offset = sym->st_value;
2231                 }
2232         }
2233
2234         return 0;
2235 }
2236
2237 static int remap_type_id(__u32 *type_id, void *ctx)
2238 {
2239         int *id_map = ctx;
2240         int new_id = id_map[*type_id];
2241
2242         /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2243         if (new_id == 0 && *type_id != 0) {
2244                 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2245                 return -EINVAL;
2246         }
2247
2248         *type_id = id_map[*type_id];
2249
2250         return 0;
2251 }
2252
2253 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2254 {
2255         const struct btf_type *t;
2256         int i, j, n, start_id, id;
2257         const char *name;
2258
2259         if (!obj->btf)
2260                 return 0;
2261
2262         start_id = btf__type_cnt(linker->btf);
2263         n = btf__type_cnt(obj->btf);
2264
2265         obj->btf_type_map = calloc(n + 1, sizeof(int));
2266         if (!obj->btf_type_map)
2267                 return -ENOMEM;
2268
2269         for (i = 1; i < n; i++) {
2270                 struct glob_sym *glob_sym = NULL;
2271
2272                 t = btf__type_by_id(obj->btf, i);
2273
2274                 /* DATASECs are handled specially below */
2275                 if (btf_kind(t) == BTF_KIND_DATASEC)
2276                         continue;
2277
2278                 if (btf_is_non_static(t)) {
2279                         /* there should be glob_sym already */
2280                         name = btf__str_by_offset(obj->btf, t->name_off);
2281                         glob_sym = find_glob_sym(linker, name);
2282
2283                         /* VARs without corresponding glob_sym are those that
2284                          * belong to skipped/deduplicated sections (i.e.,
2285                          * license and version), so just skip them
2286                          */
2287                         if (!glob_sym)
2288                                 continue;
2289
2290                         /* linker_append_elf_sym() might have requested
2291                          * updating underlying type ID, if extern was resolved
2292                          * to strong symbol or weak got upgraded to non-weak
2293                          */
2294                         if (glob_sym->underlying_btf_id == 0)
2295                                 glob_sym->underlying_btf_id = -t->type;
2296
2297                         /* globals from previous object files that match our
2298                          * VAR/FUNC already have a corresponding associated
2299                          * BTF type, so just make sure to use it
2300                          */
2301                         if (glob_sym->btf_id) {
2302                                 /* reuse existing BTF type for global var/func */
2303                                 obj->btf_type_map[i] = glob_sym->btf_id;
2304                                 continue;
2305                         }
2306                 }
2307
2308                 id = btf__add_type(linker->btf, obj->btf, t);
2309                 if (id < 0) {
2310                         pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2311                         return id;
2312                 }
2313
2314                 obj->btf_type_map[i] = id;
2315
2316                 /* record just appended BTF type for var/func */
2317                 if (glob_sym) {
2318                         glob_sym->btf_id = id;
2319                         glob_sym->underlying_btf_id = -t->type;
2320                 }
2321         }
2322
2323         /* remap all the types except DATASECs */
2324         n = btf__type_cnt(linker->btf);
2325         for (i = start_id; i < n; i++) {
2326                 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2327
2328                 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2329                         return -EINVAL;
2330         }
2331
2332         /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2333          * actual type), if necessary
2334          */
2335         for (i = 0; i < linker->glob_sym_cnt; i++) {
2336                 struct glob_sym *glob_sym = &linker->glob_syms[i];
2337                 struct btf_type *glob_t;
2338
2339                 if (glob_sym->underlying_btf_id >= 0)
2340                         continue;
2341
2342                 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2343
2344                 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2345                 glob_t->type = glob_sym->underlying_btf_id;
2346         }
2347
2348         /* append DATASEC info */
2349         for (i = 1; i < obj->sec_cnt; i++) {
2350                 struct src_sec *src_sec;
2351                 struct dst_sec *dst_sec;
2352                 const struct btf_var_secinfo *src_var;
2353                 struct btf_var_secinfo *dst_var;
2354
2355                 src_sec = &obj->secs[i];
2356                 if (!src_sec->sec_type_id || src_sec->skipped)
2357                         continue;
2358                 dst_sec = &linker->secs[src_sec->dst_id];
2359
2360                 /* Mark section as having BTF regardless of the presence of
2361                  * variables. In some cases compiler might generate empty BTF
2362                  * with no variables information. E.g., when promoting local
2363                  * array/structure variable initial values and BPF object
2364                  * file otherwise has no read-only static variables in
2365                  * .rodata. We need to preserve such empty BTF and just set
2366                  * correct section size.
2367                  */
2368                 dst_sec->has_btf = true;
2369
2370                 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2371                 src_var = btf_var_secinfos(t);
2372                 n = btf_vlen(t);
2373                 for (j = 0; j < n; j++, src_var++) {
2374                         void *sec_vars = dst_sec->sec_vars;
2375                         int new_id = obj->btf_type_map[src_var->type];
2376                         struct glob_sym *glob_sym = NULL;
2377
2378                         t = btf_type_by_id(linker->btf, new_id);
2379                         if (btf_is_non_static(t)) {
2380                                 name = btf__str_by_offset(linker->btf, t->name_off);
2381                                 glob_sym = find_glob_sym(linker, name);
2382                                 if (glob_sym->sec_id != dst_sec->id) {
2383                                         pr_warn("global '%s': section mismatch %d vs %d\n",
2384                                                 name, glob_sym->sec_id, dst_sec->id);
2385                                         return -EINVAL;
2386                                 }
2387                         }
2388
2389                         /* If there is already a member (VAR or FUNC) mapped
2390                          * to the same type, don't add a duplicate entry.
2391                          * This will happen when multiple object files define
2392                          * the same extern VARs/FUNCs.
2393                          */
2394                         if (glob_sym && glob_sym->var_idx >= 0) {
2395                                 __s64 sz;
2396
2397                                 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2398                                 /* Because underlying BTF type might have
2399                                  * changed, so might its size have changed, so
2400                                  * re-calculate and update it in sec_var.
2401                                  */
2402                                 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2403                                 if (sz < 0) {
2404                                         pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2405                                                 name, (int)sz);
2406                                         return -EINVAL;
2407                                 }
2408                                 dst_var->size = sz;
2409                                 continue;
2410                         }
2411
2412                         sec_vars = libbpf_reallocarray(sec_vars,
2413                                                        dst_sec->sec_var_cnt + 1,
2414                                                        sizeof(*dst_sec->sec_vars));
2415                         if (!sec_vars)
2416                                 return -ENOMEM;
2417
2418                         dst_sec->sec_vars = sec_vars;
2419                         dst_sec->sec_var_cnt++;
2420
2421                         dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2422                         dst_var->type = obj->btf_type_map[src_var->type];
2423                         dst_var->size = src_var->size;
2424                         dst_var->offset = src_sec->dst_off + src_var->offset;
2425
2426                         if (glob_sym)
2427                                 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2428                 }
2429         }
2430
2431         return 0;
2432 }
2433
2434 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2435 {
2436         void *tmp;
2437
2438         tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2439         if (!tmp)
2440                 return NULL;
2441         ext_data->recs = tmp;
2442
2443         tmp += ext_data->rec_cnt * ext_data->rec_sz;
2444         memcpy(tmp, src_rec, ext_data->rec_sz);
2445
2446         ext_data->rec_cnt++;
2447
2448         return tmp;
2449 }
2450
2451 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2452 {
2453         const struct btf_ext_info_sec *ext_sec;
2454         const char *sec_name, *s;
2455         struct src_sec *src_sec;
2456         struct dst_sec *dst_sec;
2457         int rec_sz, str_off, i;
2458
2459         if (!obj->btf_ext)
2460                 return 0;
2461
2462         rec_sz = obj->btf_ext->func_info.rec_size;
2463         for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2464                 struct bpf_func_info_min *src_rec, *dst_rec;
2465
2466                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2467                 src_sec = find_src_sec_by_name(obj, sec_name);
2468                 if (!src_sec) {
2469                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2470                         return -EINVAL;
2471                 }
2472                 dst_sec = &linker->secs[src_sec->dst_id];
2473
2474                 if (dst_sec->func_info.rec_sz == 0)
2475                         dst_sec->func_info.rec_sz = rec_sz;
2476                 if (dst_sec->func_info.rec_sz != rec_sz) {
2477                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2478                         return -EINVAL;
2479                 }
2480
2481                 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2482                         dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2483                         if (!dst_rec)
2484                                 return -ENOMEM;
2485
2486                         dst_rec->insn_off += src_sec->dst_off;
2487                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2488                 }
2489         }
2490
2491         rec_sz = obj->btf_ext->line_info.rec_size;
2492         for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2493                 struct bpf_line_info_min *src_rec, *dst_rec;
2494
2495                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2496                 src_sec = find_src_sec_by_name(obj, sec_name);
2497                 if (!src_sec) {
2498                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2499                         return -EINVAL;
2500                 }
2501                 dst_sec = &linker->secs[src_sec->dst_id];
2502
2503                 if (dst_sec->line_info.rec_sz == 0)
2504                         dst_sec->line_info.rec_sz = rec_sz;
2505                 if (dst_sec->line_info.rec_sz != rec_sz) {
2506                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2507                         return -EINVAL;
2508                 }
2509
2510                 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2511                         dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2512                         if (!dst_rec)
2513                                 return -ENOMEM;
2514
2515                         dst_rec->insn_off += src_sec->dst_off;
2516
2517                         s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2518                         str_off = btf__add_str(linker->btf, s);
2519                         if (str_off < 0)
2520                                 return -ENOMEM;
2521                         dst_rec->file_name_off = str_off;
2522
2523                         s = btf__str_by_offset(obj->btf, src_rec->line_off);
2524                         str_off = btf__add_str(linker->btf, s);
2525                         if (str_off < 0)
2526                                 return -ENOMEM;
2527                         dst_rec->line_off = str_off;
2528
2529                         /* dst_rec->line_col is fine */
2530                 }
2531         }
2532
2533         rec_sz = obj->btf_ext->core_relo_info.rec_size;
2534         for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2535                 struct bpf_core_relo *src_rec, *dst_rec;
2536
2537                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2538                 src_sec = find_src_sec_by_name(obj, sec_name);
2539                 if (!src_sec) {
2540                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2541                         return -EINVAL;
2542                 }
2543                 dst_sec = &linker->secs[src_sec->dst_id];
2544
2545                 if (dst_sec->core_relo_info.rec_sz == 0)
2546                         dst_sec->core_relo_info.rec_sz = rec_sz;
2547                 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2548                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2549                         return -EINVAL;
2550                 }
2551
2552                 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2553                         dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2554                         if (!dst_rec)
2555                                 return -ENOMEM;
2556
2557                         dst_rec->insn_off += src_sec->dst_off;
2558                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2559
2560                         s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2561                         str_off = btf__add_str(linker->btf, s);
2562                         if (str_off < 0)
2563                                 return -ENOMEM;
2564                         dst_rec->access_str_off = str_off;
2565
2566                         /* dst_rec->kind is fine */
2567                 }
2568         }
2569
2570         return 0;
2571 }
2572
2573 int bpf_linker__finalize(struct bpf_linker *linker)
2574 {
2575         struct dst_sec *sec;
2576         size_t strs_sz;
2577         const void *strs;
2578         int err, i;
2579
2580         if (!linker->elf)
2581                 return libbpf_err(-EINVAL);
2582
2583         err = finalize_btf(linker);
2584         if (err)
2585                 return libbpf_err(err);
2586
2587         /* Finalize strings */
2588         strs_sz = strset__data_size(linker->strtab_strs);
2589         strs = strset__data(linker->strtab_strs);
2590
2591         sec = &linker->secs[linker->strtab_sec_idx];
2592         sec->data->d_align = 1;
2593         sec->data->d_off = 0LL;
2594         sec->data->d_buf = (void *)strs;
2595         sec->data->d_type = ELF_T_BYTE;
2596         sec->data->d_size = strs_sz;
2597         sec->shdr->sh_size = strs_sz;
2598
2599         for (i = 1; i < linker->sec_cnt; i++) {
2600                 sec = &linker->secs[i];
2601
2602                 /* STRTAB is handled specially above */
2603                 if (sec->sec_idx == linker->strtab_sec_idx)
2604                         continue;
2605
2606                 /* special ephemeral sections (.ksyms, .kconfig, etc) */
2607                 if (!sec->scn)
2608                         continue;
2609
2610                 sec->data->d_buf = sec->raw_data;
2611         }
2612
2613         /* Finalize ELF layout */
2614         if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2615                 err = -errno;
2616                 pr_warn_elf("failed to finalize ELF layout");
2617                 return libbpf_err(err);
2618         }
2619
2620         /* Write out final ELF contents */
2621         if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2622                 err = -errno;
2623                 pr_warn_elf("failed to write ELF contents");
2624                 return libbpf_err(err);
2625         }
2626
2627         elf_end(linker->elf);
2628         close(linker->fd);
2629
2630         linker->elf = NULL;
2631         linker->fd = -1;
2632
2633         return 0;
2634 }
2635
2636 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2637                              size_t align, const void *raw_data, size_t raw_sz)
2638 {
2639         Elf_Scn *scn;
2640         Elf_Data *data;
2641         Elf64_Shdr *shdr;
2642         int name_off;
2643
2644         name_off = strset__add_str(linker->strtab_strs, sec_name);
2645         if (name_off < 0)
2646                 return name_off;
2647
2648         scn = elf_newscn(linker->elf);
2649         if (!scn)
2650                 return -ENOMEM;
2651         data = elf_newdata(scn);
2652         if (!data)
2653                 return -ENOMEM;
2654         shdr = elf64_getshdr(scn);
2655         if (!shdr)
2656                 return -EINVAL;
2657
2658         shdr->sh_name = name_off;
2659         shdr->sh_type = SHT_PROGBITS;
2660         shdr->sh_flags = 0;
2661         shdr->sh_size = raw_sz;
2662         shdr->sh_link = 0;
2663         shdr->sh_info = 0;
2664         shdr->sh_addralign = align;
2665         shdr->sh_entsize = 0;
2666
2667         data->d_type = ELF_T_BYTE;
2668         data->d_size = raw_sz;
2669         data->d_buf = (void *)raw_data;
2670         data->d_align = align;
2671         data->d_off = 0;
2672
2673         return 0;
2674 }
2675
2676 static int finalize_btf(struct bpf_linker *linker)
2677 {
2678         LIBBPF_OPTS(btf_dedup_opts, opts);
2679         struct btf *btf = linker->btf;
2680         const void *raw_data;
2681         int i, j, id, err;
2682         __u32 raw_sz;
2683
2684         /* bail out if no BTF data was produced */
2685         if (btf__type_cnt(linker->btf) == 1)
2686                 return 0;
2687
2688         for (i = 1; i < linker->sec_cnt; i++) {
2689                 struct dst_sec *sec = &linker->secs[i];
2690
2691                 if (!sec->has_btf)
2692                         continue;
2693
2694                 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2695                 if (id < 0) {
2696                         pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2697                                 sec->sec_name, id);
2698                         return id;
2699                 }
2700
2701                 for (j = 0; j < sec->sec_var_cnt; j++) {
2702                         struct btf_var_secinfo *vi = &sec->sec_vars[j];
2703
2704                         if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2705                                 return -EINVAL;
2706                 }
2707         }
2708
2709         err = finalize_btf_ext(linker);
2710         if (err) {
2711                 pr_warn(".BTF.ext generation failed: %d\n", err);
2712                 return err;
2713         }
2714
2715         opts.btf_ext = linker->btf_ext;
2716         err = btf__dedup(linker->btf, &opts);
2717         if (err) {
2718                 pr_warn("BTF dedup failed: %d\n", err);
2719                 return err;
2720         }
2721
2722         /* Emit .BTF section */
2723         raw_data = btf__raw_data(linker->btf, &raw_sz);
2724         if (!raw_data)
2725                 return -ENOMEM;
2726
2727         err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2728         if (err) {
2729                 pr_warn("failed to write out .BTF ELF section: %d\n", err);
2730                 return err;
2731         }
2732
2733         /* Emit .BTF.ext section */
2734         if (linker->btf_ext) {
2735                 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
2736                 if (!raw_data)
2737                         return -ENOMEM;
2738
2739                 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2740                 if (err) {
2741                         pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2742                         return err;
2743                 }
2744         }
2745
2746         return 0;
2747 }
2748
2749 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2750                              const char *sec_name, struct btf_ext_sec_data *sec_data)
2751 {
2752         struct btf_ext_info_sec *sec_info;
2753         void *cur = output;
2754         int str_off;
2755         size_t sz;
2756
2757         if (!sec_data->rec_cnt)
2758                 return 0;
2759
2760         str_off = btf__add_str(linker->btf, sec_name);
2761         if (str_off < 0)
2762                 return -ENOMEM;
2763
2764         sec_info = cur;
2765         sec_info->sec_name_off = str_off;
2766         sec_info->num_info = sec_data->rec_cnt;
2767         cur += sizeof(struct btf_ext_info_sec);
2768
2769         sz = sec_data->rec_cnt * sec_data->rec_sz;
2770         memcpy(cur, sec_data->recs, sz);
2771         cur += sz;
2772
2773         return cur - output;
2774 }
2775
2776 static int finalize_btf_ext(struct bpf_linker *linker)
2777 {
2778         size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2779         size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2780         struct btf_ext_header *hdr;
2781         void *data, *cur;
2782         int i, err, sz;
2783
2784         /* validate that all sections have the same .BTF.ext record sizes
2785          * and calculate total data size for each type of data (func info,
2786          * line info, core relos)
2787          */
2788         for (i = 1; i < linker->sec_cnt; i++) {
2789                 struct dst_sec *sec = &linker->secs[i];
2790
2791                 if (sec->func_info.rec_cnt) {
2792                         if (func_rec_sz == 0)
2793                                 func_rec_sz = sec->func_info.rec_sz;
2794                         if (func_rec_sz != sec->func_info.rec_sz) {
2795                                 pr_warn("mismatch in func_info record size %zu != %u\n",
2796                                         func_rec_sz, sec->func_info.rec_sz);
2797                                 return -EINVAL;
2798                         }
2799
2800                         funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2801                 }
2802                 if (sec->line_info.rec_cnt) {
2803                         if (line_rec_sz == 0)
2804                                 line_rec_sz = sec->line_info.rec_sz;
2805                         if (line_rec_sz != sec->line_info.rec_sz) {
2806                                 pr_warn("mismatch in line_info record size %zu != %u\n",
2807                                         line_rec_sz, sec->line_info.rec_sz);
2808                                 return -EINVAL;
2809                         }
2810
2811                         lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2812                 }
2813                 if (sec->core_relo_info.rec_cnt) {
2814                         if (core_relo_rec_sz == 0)
2815                                 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2816                         if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2817                                 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2818                                         core_relo_rec_sz, sec->core_relo_info.rec_sz);
2819                                 return -EINVAL;
2820                         }
2821
2822                         core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2823                 }
2824         }
2825
2826         if (!funcs_sz && !lines_sz && !core_relos_sz)
2827                 return 0;
2828
2829         total_sz += sizeof(struct btf_ext_header);
2830         if (funcs_sz) {
2831                 funcs_sz += sizeof(__u32); /* record size prefix */
2832                 total_sz += funcs_sz;
2833         }
2834         if (lines_sz) {
2835                 lines_sz += sizeof(__u32); /* record size prefix */
2836                 total_sz += lines_sz;
2837         }
2838         if (core_relos_sz) {
2839                 core_relos_sz += sizeof(__u32); /* record size prefix */
2840                 total_sz += core_relos_sz;
2841         }
2842
2843         cur = data = calloc(1, total_sz);
2844         if (!data)
2845                 return -ENOMEM;
2846
2847         hdr = cur;
2848         hdr->magic = BTF_MAGIC;
2849         hdr->version = BTF_VERSION;
2850         hdr->flags = 0;
2851         hdr->hdr_len = sizeof(struct btf_ext_header);
2852         cur += sizeof(struct btf_ext_header);
2853
2854         /* All offsets are in bytes relative to the end of this header */
2855         hdr->func_info_off = 0;
2856         hdr->func_info_len = funcs_sz;
2857         hdr->line_info_off = funcs_sz;
2858         hdr->line_info_len = lines_sz;
2859         hdr->core_relo_off = funcs_sz + lines_sz;
2860         hdr->core_relo_len = core_relos_sz;
2861
2862         if (funcs_sz) {
2863                 *(__u32 *)cur = func_rec_sz;
2864                 cur += sizeof(__u32);
2865
2866                 for (i = 1; i < linker->sec_cnt; i++) {
2867                         struct dst_sec *sec = &linker->secs[i];
2868
2869                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2870                         if (sz < 0) {
2871                                 err = sz;
2872                                 goto out;
2873                         }
2874
2875                         cur += sz;
2876                 }
2877         }
2878
2879         if (lines_sz) {
2880                 *(__u32 *)cur = line_rec_sz;
2881                 cur += sizeof(__u32);
2882
2883                 for (i = 1; i < linker->sec_cnt; i++) {
2884                         struct dst_sec *sec = &linker->secs[i];
2885
2886                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2887                         if (sz < 0) {
2888                                 err = sz;
2889                                 goto out;
2890                         }
2891
2892                         cur += sz;
2893                 }
2894         }
2895
2896         if (core_relos_sz) {
2897                 *(__u32 *)cur = core_relo_rec_sz;
2898                 cur += sizeof(__u32);
2899
2900                 for (i = 1; i < linker->sec_cnt; i++) {
2901                         struct dst_sec *sec = &linker->secs[i];
2902
2903                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2904                         if (sz < 0) {
2905                                 err = sz;
2906                                 goto out;
2907                         }
2908
2909                         cur += sz;
2910                 }
2911         }
2912
2913         linker->btf_ext = btf_ext__new(data, total_sz);
2914         err = libbpf_get_error(linker->btf_ext);
2915         if (err) {
2916                 linker->btf_ext = NULL;
2917                 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2918                 goto out;
2919         }
2920
2921 out:
2922         free(data);
2923         return err;
2924 }