kernel/bpf/stackmap.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2016 Facebook
   3  */
   4 #include <linux/bpf.h>
   5 #include <linux/jhash.h>
   6 #include <linux/filter.h>
   7 #include <linux/kernel.h>
   8 #include <linux/stacktrace.h>
   9 #include <linux/perf_event.h>
  10 #include <linux/elf.h>
  11 #include <linux/pagemap.h>
  12 #include <linux/irq_work.h>
  13 #include <linux/btf_ids.h>
  14 #include "percpu_freelist.h"
  15
  16 #define STACK_CREATE_FLAG_MASK                                  \
  17         (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY |        \
  18          BPF_F_STACK_BUILD_ID)
  19
  20 struct stack_map_bucket {
  21         struct pcpu_freelist_node fnode;
  22         u32 hash;
  23         u32 nr;
  24         u64 data[];
  25 };
  26
  27 struct bpf_stack_map {
  28         struct bpf_map map;
  29         void *elems;
  30         struct pcpu_freelist freelist;
  31         u32 n_buckets;
  32         struct stack_map_bucket *buckets[];
  33 };
  34
  35 /* irq_work to run up_read() for build_id lookup in nmi context */
  36 struct stack_map_irq_work {
  37         struct irq_work irq_work;
  38         struct mm_struct *mm;
  39 };
  40
  41 static void do_up_read(struct irq_work *entry)
  42 {
  43         struct stack_map_irq_work *work;
  44
  45         if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT)))
  46                 return;
  47
  48         work = container_of(entry, struct stack_map_irq_work, irq_work);
  49         mmap_read_unlock_non_owner(work->mm);
  50 }
  51
  52 static DEFINE_PER_CPU(struct stack_map_irq_work, up_read_work);
  53
  54 static inline bool stack_map_use_build_id(struct bpf_map *map)
  55 {
  56         return (map->map_flags & BPF_F_STACK_BUILD_ID);
  57 }
  58
  59 static inline int stack_map_data_size(struct bpf_map *map)
  60 {
  61         return stack_map_use_build_id(map) ?
  62                 sizeof(struct bpf_stack_build_id) : sizeof(u64);
  63 }
  64
  65 static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)
  66 {
  67         u64 elem_size = sizeof(struct stack_map_bucket) +
  68                         (u64)smap->map.value_size;
  69         int err;
  70
  71         smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,
  72                                          smap->map.numa_node);
  73         if (!smap->elems)
  74                 return -ENOMEM;
  75
  76         err = pcpu_freelist_init(&smap->freelist);
  77         if (err)
  78                 goto free_elems;
  79
  80         pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,
  81                                smap->map.max_entries);
  82         return 0;
  83
  84 free_elems:
  85         bpf_map_area_free(smap->elems);
  86         return err;
  87 }
  88
  89 /* Called from syscall */
  90 static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
  91 {
  92         u32 value_size = attr->value_size;
  93         struct bpf_stack_map *smap;
  94         struct bpf_map_memory mem;
  95         u64 cost, n_buckets;
  96         int err;
  97
  98         if (!bpf_capable())
  99                 return ERR_PTR(-EPERM);
 100
 101         if (attr->map_flags & ~STACK_CREATE_FLAG_MASK)
 102                 return ERR_PTR(-EINVAL);
 103
 104         /* check sanity of attributes */
 105         if (attr->max_entries == 0 || attr->key_size != 4 ||
 106             value_size < 8 || value_size % 8)
 107                 return ERR_PTR(-EINVAL);
 108
 109         BUILD_BUG_ON(sizeof(struct bpf_stack_build_id) % sizeof(u64));
 110         if (attr->map_flags & BPF_F_STACK_BUILD_ID) {
 111                 if (value_size % sizeof(struct bpf_stack_build_id) ||
 112                     value_size / sizeof(struct bpf_stack_build_id)
 113                     > sysctl_perf_event_max_stack)
 114                         return ERR_PTR(-EINVAL);
 115         } else if (value_size / 8 > sysctl_perf_event_max_stack)
 116                 return ERR_PTR(-EINVAL);
 117
 118         /* hash table size must be power of 2; roundup_pow_of_two() can overflow
 119          * into UB on 32-bit arches, so check that first
 120          */
 121         if (attr->max_entries > 1UL << 31)
 122                 return ERR_PTR(-E2BIG);
 123
 124         n_buckets = roundup_pow_of_two(attr->max_entries);
 125
 126         cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
 127         err = bpf_map_charge_init(&mem, cost + attr->max_entries *
 128                            (sizeof(struct stack_map_bucket) + (u64)value_size));
 129         if (err)
 130                 return ERR_PTR(err);
 131
 132         smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
 133         if (!smap) {
 134                 bpf_map_charge_finish(&mem);
 135                 return ERR_PTR(-ENOMEM);
 136         }
 137
 138         bpf_map_init_from_attr(&smap->map, attr);
 139         smap->map.value_size = value_size;
 140         smap->n_buckets = n_buckets;
 141
 142         err = get_callchain_buffers(sysctl_perf_event_max_stack);
 143         if (err)
 144                 goto free_charge;
 145
 146         err = prealloc_elems_and_freelist(smap);
 147         if (err)
 148                 goto put_buffers;
 149
 150         bpf_map_charge_move(&smap->map.memory, &mem);
 151
 152         return &smap->map;
 153
 154 put_buffers:
 155         put_callchain_buffers();
 156 free_charge:
 157         bpf_map_charge_finish(&mem);
 158         bpf_map_area_free(smap);
 159         return ERR_PTR(err);
 160 }
 161
 162 #define BPF_BUILD_ID 3
 163 /*
 164  * Parse build id from the note segment. This logic can be shared between
 165  * 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are
 166  * identical.
 167  */
 168 static inline int stack_map_parse_build_id(void *page_addr,
 169                                            unsigned char *build_id,
 170                                            void *note_start,
 171                                            Elf32_Word note_size)
 172 {
 173         Elf32_Word note_offs = 0, new_offs;
 174
 175         /* check for overflow */
 176         if (note_start < page_addr || note_start + note_size < note_start)
 177                 return -EINVAL;
 178
 179         /* only supports note that fits in the first page */
 180         if (note_start + note_size > page_addr + PAGE_SIZE)
 181                 return -EINVAL;
 182
 183         while (note_offs + sizeof(Elf32_Nhdr) < note_size) {
 184                 Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs);
 185
 186                 if (nhdr->n_type == BPF_BUILD_ID &&
 187                     nhdr->n_namesz == sizeof("GNU") &&
 188                     nhdr->n_descsz > 0 &&
 189                     nhdr->n_descsz <= BPF_BUILD_ID_SIZE) {
 190                         memcpy(build_id,
 191                                note_start + note_offs +
 192                                ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr),
 193                                nhdr->n_descsz);
 194                         memset(build_id + nhdr->n_descsz, 0,
 195                                BPF_BUILD_ID_SIZE - nhdr->n_descsz);
 196                         return 0;
 197                 }
 198                 new_offs = note_offs + sizeof(Elf32_Nhdr) +
 199                         ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4);
 200                 if (new_offs <= note_offs)  /* overflow */
 201                         break;
 202                 note_offs = new_offs;
 203         }
 204         return -EINVAL;
 205 }
 206
 207 /* Parse build ID from 32-bit ELF */
 208 static int stack_map_get_build_id_32(void *page_addr,
 209                                      unsigned char *build_id)
 210 {
 211         Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr;
 212         Elf32_Phdr *phdr;
 213         int i;
 214
 215         /* only supports phdr that fits in one page */
 216         if (ehdr->e_phnum >
 217             (PAGE_SIZE - sizeof(Elf32_Ehdr)) / sizeof(Elf32_Phdr))
 218                 return -EINVAL;
 219
 220         phdr = (Elf32_Phdr *)(page_addr + sizeof(Elf32_Ehdr));
 221
 222         for (i = 0; i < ehdr->e_phnum; ++i) {
 223                 if (phdr[i].p_type == PT_NOTE &&
 224                     !stack_map_parse_build_id(page_addr, build_id,
 225                                               page_addr + phdr[i].p_offset,
 226                                               phdr[i].p_filesz))
 227                         return 0;
 228         }
 229         return -EINVAL;
 230 }
 231
 232 /* Parse build ID from 64-bit ELF */
 233 static int stack_map_get_build_id_64(void *page_addr,
 234                                      unsigned char *build_id)
 235 {
 236         Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr;
 237         Elf64_Phdr *phdr;
 238         int i;
 239
 240         /* only supports phdr that fits in one page */
 241         if (ehdr->e_phnum >
 242             (PAGE_SIZE - sizeof(Elf64_Ehdr)) / sizeof(Elf64_Phdr))
 243                 return -EINVAL;
 244
 245         phdr = (Elf64_Phdr *)(page_addr + sizeof(Elf64_Ehdr));
 246
 247         for (i = 0; i < ehdr->e_phnum; ++i) {
 248                 if (phdr[i].p_type == PT_NOTE &&
 249                     !stack_map_parse_build_id(page_addr, build_id,
 250                                               page_addr + phdr[i].p_offset,
 251                                               phdr[i].p_filesz))
 252                         return 0;
 253         }
 254         return -EINVAL;
 255 }
 256
 257 /* Parse build ID of ELF file mapped to vma */
 258 static int stack_map_get_build_id(struct vm_area_struct *vma,
 259                                   unsigned char *build_id)
 260 {
 261         Elf32_Ehdr *ehdr;
 262         struct page *page;
 263         void *page_addr;
 264         int ret;
 265
 266         /* only works for page backed storage  */
 267         if (!vma->vm_file)
 268                 return -EINVAL;
 269
 270         page = find_get_page(vma->vm_file->f_mapping, 0);
 271         if (!page)
 272                 return -EFAULT; /* page not mapped */
 273
 274         ret = -EINVAL;
 275         page_addr = kmap_atomic(page);
 276         ehdr = (Elf32_Ehdr *)page_addr;
 277
 278         /* compare magic x7f "ELF" */
 279         if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0)
 280                 goto out;
 281
 282         /* only support executable file and shared object file */
 283         if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN)
 284                 goto out;
 285
 286         if (ehdr->e_ident[EI_CLASS] == ELFCLASS32)
 287                 ret = stack_map_get_build_id_32(page_addr, build_id);
 288         else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64)
 289                 ret = stack_map_get_build_id_64(page_addr, build_id);
 290 out:
 291         kunmap_atomic(page_addr);
 292         put_page(page);
 293         return ret;
 294 }
 295
 296 static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
 297                                           u64 *ips, u32 trace_nr, bool user)
 298 {
 299         int i;
 300         struct vm_area_struct *vma;
 301         bool irq_work_busy = false;
 302         struct stack_map_irq_work *work = NULL;
 303
 304         if (irqs_disabled()) {
 305                 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
 306                         work = this_cpu_ptr(&up_read_work);
 307                         if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY) {
 308                                 /* cannot queue more up_read, fallback */
 309                                 irq_work_busy = true;
 310                         }
 311                 } else {
 312                         /*
 313                          * PREEMPT_RT does not allow to trylock mmap sem in
 314                          * interrupt disabled context. Force the fallback code.
 315                          */
 316                         irq_work_busy = true;
 317                 }
 318         }
 319
 320         /*
 321          * We cannot do up_read() when the irq is disabled, because of
 322          * risk to deadlock with rq_lock. To do build_id lookup when the
 323          * irqs are disabled, we need to run up_read() in irq_work. We use
 324          * a percpu variable to do the irq_work. If the irq_work is
 325          * already used by another lookup, we fall back to report ips.
 326          *
 327          * Same fallback is used for kernel stack (!user) on a stackmap
 328          * with build_id.
 329          */
 330         if (!user || !current || !current->mm || irq_work_busy ||
 331             !mmap_read_trylock_non_owner(current->mm)) {
 332                 /* cannot access current->mm, fall back to ips */
 333                 for (i = 0; i < trace_nr; i++) {
 334                         id_offs[i].status = BPF_STACK_BUILD_ID_IP;
 335                         id_offs[i].ip = ips[i];
 336                         memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE);
 337                 }
 338                 return;
 339         }
 340
 341         for (i = 0; i < trace_nr; i++) {
 342                 vma = find_vma(current->mm, ips[i]);
 343                 if (!vma || stack_map_get_build_id(vma, id_offs[i].build_id)) {
 344                         /* per entry fall back to ips */
 345                         id_offs[i].status = BPF_STACK_BUILD_ID_IP;
 346                         id_offs[i].ip = ips[i];
 347                         memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE);
 348                         continue;
 349                 }
 350                 id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i]
 351                         - vma->vm_start;
 352                 id_offs[i].status = BPF_STACK_BUILD_ID_VALID;
 353         }
 354
 355         if (!work) {
 356                 mmap_read_unlock_non_owner(current->mm);
 357         } else {
 358                 work->mm = current->mm;
 359                 irq_work_queue(&work->irq_work);
 360         }
 361 }
 362
 363 static struct perf_callchain_entry *
 364 get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
 365 {
 366 #ifdef CONFIG_STACKTRACE
 367         struct perf_callchain_entry *entry;
 368         int rctx;
 369
 370         entry = get_callchain_entry(&rctx);
 371
 372         if (!entry)
 373                 return NULL;
 374
 375         entry->nr = stack_trace_save_tsk(task, (unsigned long *)entry->ip,
 376                                          max_depth, 0);
 377
 378         /* stack_trace_save_tsk() works on unsigned long array, while
 379          * perf_callchain_entry uses u64 array. For 32-bit systems, it is
 380          * necessary to fix this mismatch.
 381          */
 382         if (__BITS_PER_LONG != 64) {
 383                 unsigned long *from = (unsigned long *) entry->ip;
 384                 u64 *to = entry->ip;
 385                 int i;
 386
 387                 /* copy data from the end to avoid using extra buffer */
 388                 for (i = entry->nr - 1; i >= 0; i--)
 389                         to[i] = (u64)(from[i]);
 390         }
 391
 392         put_callchain_entry(rctx);
 393
 394         return entry;
 395 #else /* CONFIG_STACKTRACE */
 396         return NULL;
 397 #endif
 398 }
 399
 400 static long __bpf_get_stackid(struct bpf_map *map,
 401                               struct perf_callchain_entry *trace, u64 flags)
 402 {
 403         struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
 404         struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
 405         u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 406         u32 hash, id, trace_nr, trace_len;
 407         bool user = flags & BPF_F_USER_STACK;
 408         u64 *ips;
 409         bool hash_matches;
 410
 411         if (trace->nr <= skip)
 412                 /* skipping more than usable stack trace */
 413                 return -EFAULT;
 414
 415         trace_nr = trace->nr - skip;
 416         trace_len = trace_nr * sizeof(u64);
 417         ips = trace->ip + skip;
 418         hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
 419         id = hash & (smap->n_buckets - 1);
 420         bucket = READ_ONCE(smap->buckets[id]);
 421
 422         hash_matches = bucket && bucket->hash == hash;
 423         /* fast cmp */
 424         if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
 425                 return id;
 426
 427         if (stack_map_use_build_id(map)) {
 428                 /* for build_id+offset, pop a bucket before slow cmp */
 429                 new_bucket = (struct stack_map_bucket *)
 430                         pcpu_freelist_pop(&smap->freelist);
 431                 if (unlikely(!new_bucket))
 432                         return -ENOMEM;
 433                 new_bucket->nr = trace_nr;
 434                 stack_map_get_build_id_offset(
 435                         (struct bpf_stack_build_id *)new_bucket->data,
 436                         ips, trace_nr, user);
 437                 trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
 438                 if (hash_matches && bucket->nr == trace_nr &&
 439                     memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
 440                         pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
 441                         return id;
 442                 }
 443                 if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
 444                         pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
 445                         return -EEXIST;
 446                 }
 447         } else {
 448                 if (hash_matches && bucket->nr == trace_nr &&
 449                     memcmp(bucket->data, ips, trace_len) == 0)
 450                         return id;
 451                 if (bucket && !(flags & BPF_F_REUSE_STACKID))
 452                         return -EEXIST;
 453
 454                 new_bucket = (struct stack_map_bucket *)
 455                         pcpu_freelist_pop(&smap->freelist);
 456                 if (unlikely(!new_bucket))
 457                         return -ENOMEM;
 458                 memcpy(new_bucket->data, ips, trace_len);
 459         }
 460
 461         new_bucket->hash = hash;
 462         new_bucket->nr = trace_nr;
 463
 464         old_bucket = xchg(&smap->buckets[id], new_bucket);
 465         if (old_bucket)
 466                 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
 467         return id;
 468 }
 469
 470 BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
 471            u64, flags)
 472 {
 473         u32 max_depth = map->value_size / stack_map_data_size(map);
 474         u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 475         bool user = flags & BPF_F_USER_STACK;
 476         struct perf_callchain_entry *trace;
 477         bool kernel = !user;
 478
 479         if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 480                                BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
 481                 return -EINVAL;
 482
 483         max_depth += skip;
 484         if (max_depth > sysctl_perf_event_max_stack)
 485                 max_depth = sysctl_perf_event_max_stack;
 486
 487         trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
 488                                    false, false);
 489
 490         if (unlikely(!trace))
 491                 /* couldn't fetch the stack trace */
 492                 return -EFAULT;
 493
 494         return __bpf_get_stackid(map, trace, flags);
 495 }
 496
 497 const struct bpf_func_proto bpf_get_stackid_proto = {
 498         .func           = bpf_get_stackid,
 499         .gpl_only       = true,
 500         .ret_type       = RET_INTEGER,
 501         .arg1_type      = ARG_PTR_TO_CTX,
 502         .arg2_type      = ARG_CONST_MAP_PTR,
 503         .arg3_type      = ARG_ANYTHING,
 504 };
 505
 506 static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
 507 {
 508         __u64 nr_kernel = 0;
 509
 510         while (nr_kernel < trace->nr) {
 511                 if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
 512                         break;
 513                 nr_kernel++;
 514         }
 515         return nr_kernel;
 516 }
 517
 518 BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
 519            struct bpf_map *, map, u64, flags)
 520 {
 521         struct perf_event *event = ctx->event;
 522         struct perf_callchain_entry *trace;
 523         bool kernel, user;
 524         __u64 nr_kernel;
 525         int ret;
 526
 527         /* perf_sample_data doesn't have callchain, use bpf_get_stackid */
 528         if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY))
 529                 return bpf_get_stackid((unsigned long)(ctx->regs),
 530                                        (unsigned long) map, flags, 0, 0);
 531
 532         if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 533                                BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
 534                 return -EINVAL;
 535
 536         user = flags & BPF_F_USER_STACK;
 537         kernel = !user;
 538
 539         trace = ctx->data->callchain;
 540         if (unlikely(!trace))
 541                 return -EFAULT;
 542
 543         nr_kernel = count_kernel_ip(trace);
 544
 545         if (kernel) {
 546                 __u64 nr = trace->nr;
 547
 548                 trace->nr = nr_kernel;
 549                 ret = __bpf_get_stackid(map, trace, flags);
 550
 551                 /* restore nr */
 552                 trace->nr = nr;
 553         } else { /* user */
 554                 u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
 555
 556                 skip += nr_kernel;
 557                 if (skip > BPF_F_SKIP_FIELD_MASK)
 558                         return -EFAULT;
 559
 560                 flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
 561                 ret = __bpf_get_stackid(map, trace, flags);
 562         }
 563         return ret;
 564 }
 565
 566 const struct bpf_func_proto bpf_get_stackid_proto_pe = {
 567         .func           = bpf_get_stackid_pe,
 568         .gpl_only       = false,
 569         .ret_type       = RET_INTEGER,
 570         .arg1_type      = ARG_PTR_TO_CTX,
 571         .arg2_type      = ARG_CONST_MAP_PTR,
 572         .arg3_type      = ARG_ANYTHING,
 573 };
 574
 575 static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 576                             struct perf_callchain_entry *trace_in,
 577                             void *buf, u32 size, u64 flags)
 578 {
 579         u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
 580         bool user_build_id = flags & BPF_F_USER_BUILD_ID;
 581         bool crosstask = task && task != current;
 582         u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 583         bool user = flags & BPF_F_USER_STACK;
 584         struct perf_callchain_entry *trace;
 585         bool kernel = !user;
 586         int err = -EINVAL;
 587         u64 *ips;
 588
 589         if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 590                                BPF_F_USER_BUILD_ID)))
 591                 goto clear;
 592         if (kernel && user_build_id)
 593                 goto clear;
 594
 595         elem_size = (user && user_build_id) ? sizeof(struct bpf_stack_build_id)
 596                                             : sizeof(u64);
 597         if (unlikely(size % elem_size))
 598                 goto clear;
 599
 600         /* cannot get valid user stack for task without user_mode regs */
 601         if (task && user && !user_mode(regs))
 602                 goto err_fault;
 603
 604         /* get_perf_callchain does not support crosstask user stack walking
 605          * but returns an empty stack instead of NULL.
 606          */
 607         if (crosstask && user) {
 608                 err = -EOPNOTSUPP;
 609                 goto clear;
 610         }
 611
 612         num_elem = size / elem_size;
 613         max_depth = num_elem + skip;
 614         if (sysctl_perf_event_max_stack < max_depth)
 615                 max_depth = sysctl_perf_event_max_stack;
 616
 617         if (trace_in)
 618                 trace = trace_in;
 619         else if (kernel && task)
 620                 trace = get_callchain_entry_for_task(task, max_depth);
 621         else
 622                 trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
 623                                            crosstask, false);
 624         if (unlikely(!trace))
 625                 goto err_fault;
 626
 627         if (trace->nr < skip)
 628                 goto err_fault;
 629
 630         trace_nr = trace->nr - skip;
 631         trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
 632         copy_len = trace_nr * elem_size;
 633
 634         ips = trace->ip + skip;
 635         if (user && user_build_id)
 636                 stack_map_get_build_id_offset(buf, ips, trace_nr, user);
 637         else
 638                 memcpy(buf, ips, copy_len);
 639
 640         if (size > copy_len)
 641                 memset(buf + copy_len, 0, size - copy_len);
 642         return copy_len;
 643
 644 err_fault:
 645         err = -EFAULT;
 646 clear:
 647         memset(buf, 0, size);
 648         return err;
 649 }
 650
 651 BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
 652            u64, flags)
 653 {
 654         return __bpf_get_stack(regs, NULL, NULL, buf, size, flags);
 655 }
 656
 657 const struct bpf_func_proto bpf_get_stack_proto = {
 658         .func           = bpf_get_stack,
 659         .gpl_only       = true,
 660         .ret_type       = RET_INTEGER,
 661         .arg1_type      = ARG_PTR_TO_CTX,
 662         .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
 663         .arg3_type      = ARG_CONST_SIZE_OR_ZERO,
 664         .arg4_type      = ARG_ANYTHING,
 665 };
 666
 667 BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf,
 668            u32, size, u64, flags)
 669 {
 670         struct pt_regs *regs;
 671         long res = -EINVAL;
 672
 673         if (!try_get_task_stack(task))
 674                 return -EFAULT;
 675
 676         regs = task_pt_regs(task);
 677         if (regs)
 678                 res = __bpf_get_stack(regs, task, NULL, buf, size, flags);
 679         put_task_stack(task);
 680
 681         return res;
 682 }
 683
 684 BTF_ID_LIST_SINGLE(bpf_get_task_stack_btf_ids, struct, task_struct)
 685
 686 const struct bpf_func_proto bpf_get_task_stack_proto = {
 687         .func           = bpf_get_task_stack,
 688         .gpl_only       = false,
 689         .ret_type       = RET_INTEGER,
 690         .arg1_type      = ARG_PTR_TO_BTF_ID,
 691         .arg1_btf_id    = &bpf_get_task_stack_btf_ids[0],
 692         .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
 693         .arg3_type      = ARG_CONST_SIZE_OR_ZERO,
 694         .arg4_type      = ARG_ANYTHING,
 695 };
 696
 697 BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
 698            void *, buf, u32, size, u64, flags)
 699 {
 700         struct pt_regs *regs = (struct pt_regs *)(ctx->regs);
 701         struct perf_event *event = ctx->event;
 702         struct perf_callchain_entry *trace;
 703         bool kernel, user;
 704         int err = -EINVAL;
 705         __u64 nr_kernel;
 706
 707         if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY))
 708                 return __bpf_get_stack(regs, NULL, NULL, buf, size, flags);
 709
 710         if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
 711                                BPF_F_USER_BUILD_ID)))
 712                 goto clear;
 713
 714         user = flags & BPF_F_USER_STACK;
 715         kernel = !user;
 716
 717         err = -EFAULT;
 718         trace = ctx->data->callchain;
 719         if (unlikely(!trace))
 720                 goto clear;
 721
 722         nr_kernel = count_kernel_ip(trace);
 723
 724         if (kernel) {
 725                 __u64 nr = trace->nr;
 726
 727                 trace->nr = nr_kernel;
 728                 err = __bpf_get_stack(regs, NULL, trace, buf, size, flags);
 729
 730                 /* restore nr */
 731                 trace->nr = nr;
 732         } else { /* user */
 733                 u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
 734
 735                 skip += nr_kernel;
 736                 if (skip > BPF_F_SKIP_FIELD_MASK)
 737                         goto clear;
 738
 739                 flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
 740                 err = __bpf_get_stack(regs, NULL, trace, buf, size, flags);
 741         }
 742         return err;
 743
 744 clear:
 745         memset(buf, 0, size);
 746         return err;
 747
 748 }
 749
 750 const struct bpf_func_proto bpf_get_stack_proto_pe = {
 751         .func           = bpf_get_stack_pe,
 752         .gpl_only       = true,
 753         .ret_type       = RET_INTEGER,
 754         .arg1_type      = ARG_PTR_TO_CTX,
 755         .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
 756         .arg3_type      = ARG_CONST_SIZE_OR_ZERO,
 757         .arg4_type      = ARG_ANYTHING,
 758 };
 759
 760 /* Called from eBPF program */
 761 static void *stack_map_lookup_elem(struct bpf_map *map, void *key)
 762 {
 763         return ERR_PTR(-EOPNOTSUPP);
 764 }
 765
 766 /* Called from syscall */
 767 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 768 {
 769         struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
 770         struct stack_map_bucket *bucket, *old_bucket;
 771         u32 id = *(u32 *)key, trace_len;
 772
 773         if (unlikely(id >= smap->n_buckets))
 774                 return -ENOENT;
 775
 776         bucket = xchg(&smap->buckets[id], NULL);
 777         if (!bucket)
 778                 return -ENOENT;
 779
 780         trace_len = bucket->nr * stack_map_data_size(map);
 781         memcpy(value, bucket->data, trace_len);
 782         memset(value + trace_len, 0, map->value_size - trace_len);
 783
 784         old_bucket = xchg(&smap->buckets[id], bucket);
 785         if (old_bucket)
 786                 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
 787         return 0;
 788 }
 789
 790 static int stack_map_get_next_key(struct bpf_map *map, void *key,
 791                                   void *next_key)
 792 {
 793         struct bpf_stack_map *smap = container_of(map,
 794                                                   struct bpf_stack_map, map);
 795         u32 id;
 796
 797         WARN_ON_ONCE(!rcu_read_lock_held());
 798
 799         if (!key) {
 800                 id = 0;
 801         } else {
 802                 id = *(u32 *)key;
 803                 if (id >= smap->n_buckets || !smap->buckets[id])
 804                         id = 0;
 805                 else
 806                         id++;
 807         }
 808
 809         while (id < smap->n_buckets && !smap->buckets[id])
 810                 id++;
 811
 812         if (id >= smap->n_buckets)
 813                 return -ENOENT;
 814
 815         *(u32 *)next_key = id;
 816         return 0;
 817 }
 818
 819 static int stack_map_update_elem(struct bpf_map *map, void *key, void *value,
 820                                  u64 map_flags)
 821 {
 822         return -EINVAL;
 823 }
 824
 825 /* Called from syscall or from eBPF program */
 826 static int stack_map_delete_elem(struct bpf_map *map, void *key)
 827 {
 828         struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
 829         struct stack_map_bucket *old_bucket;
 830         u32 id = *(u32 *)key;
 831
 832         if (unlikely(id >= smap->n_buckets))
 833                 return -E2BIG;
 834
 835         old_bucket = xchg(&smap->buckets[id], NULL);
 836         if (old_bucket) {
 837                 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
 838                 return 0;
 839         } else {
 840                 return -ENOENT;
 841         }
 842 }
 843
 844 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
 845 static void stack_map_free(struct bpf_map *map)
 846 {
 847         struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
 848
 849         bpf_map_area_free(smap->elems);
 850         pcpu_freelist_destroy(&smap->freelist);
 851         bpf_map_area_free(smap);
 852         put_callchain_buffers();
 853 }
 854
 855 static int stack_trace_map_btf_id;
 856 const struct bpf_map_ops stack_trace_map_ops = {
 857         .map_meta_equal = bpf_map_meta_equal,
 858         .map_alloc = stack_map_alloc,
 859         .map_free = stack_map_free,
 860         .map_get_next_key = stack_map_get_next_key,
 861         .map_lookup_elem = stack_map_lookup_elem,
 862         .map_update_elem = stack_map_update_elem,
 863         .map_delete_elem = stack_map_delete_elem,
 864         .map_check_btf = map_check_no_btf,
 865         .map_btf_name = "bpf_stack_map",
 866         .map_btf_id = &stack_trace_map_btf_id,
 867 };
 868
 869 static int __init stack_map_init(void)
 870 {
 871         int cpu;
 872         struct stack_map_irq_work *work;
 873
 874         for_each_possible_cpu(cpu) {
 875                 work = per_cpu_ptr(&up_read_work, cpu);
 876                 init_irq_work(&work->irq_work, do_up_read);
 877         }
 878         return 0;
 879 }
 880 subsys_initcall(stack_map_init);