GNU Linux-libre 5.19-rc6-gnu
[releases.git] / tools / perf / util / bpf-event.c
1 // SPDX-License-Identifier: GPL-2.0
2 #include <errno.h>
3 #include <stdlib.h>
4 #include <bpf/bpf.h>
5 #include <bpf/btf.h>
6 #include <bpf/libbpf.h>
7 #include <linux/btf.h>
8 #include <linux/err.h>
9 #include <linux/string.h>
10 #include <internal/lib.h>
11 #include <symbol/kallsyms.h>
12 #include "bpf-event.h"
13 #include "bpf-utils.h"
14 #include "debug.h"
15 #include "dso.h"
16 #include "symbol.h"
17 #include "machine.h"
18 #include "env.h"
19 #include "session.h"
20 #include "map.h"
21 #include "evlist.h"
22 #include "record.h"
23 #include "util/synthetic-events.h"
24
25 #ifndef HAVE_LIBBPF_BTF__LOAD_FROM_KERNEL_BY_ID
26 struct btf *btf__load_from_kernel_by_id(__u32 id)
27 {
28        struct btf *btf;
29 #pragma GCC diagnostic push
30 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
31        int err = btf__get_from_id(id, &btf);
32 #pragma GCC diagnostic pop
33
34        return err ? ERR_PTR(err) : btf;
35 }
36 #endif
37
38 #ifndef HAVE_LIBBPF_BPF_PROG_LOAD
39 int bpf_prog_load(enum bpf_prog_type prog_type,
40                   const char *prog_name __maybe_unused,
41                   const char *license,
42                   const struct bpf_insn *insns, size_t insn_cnt,
43                   const struct bpf_prog_load_opts *opts)
44 {
45 #pragma GCC diagnostic push
46 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
47         return bpf_load_program(prog_type, insns, insn_cnt, license,
48                                 opts->kern_version, opts->log_buf, opts->log_size);
49 #pragma GCC diagnostic pop
50 }
51 #endif
52
53 #ifndef HAVE_LIBBPF_BPF_OBJECT__NEXT_PROGRAM
54 struct bpf_program *
55 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
56 {
57 #pragma GCC diagnostic push
58 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
59         return bpf_program__next(prev, obj);
60 #pragma GCC diagnostic pop
61 }
62 #endif
63
64 #ifndef HAVE_LIBBPF_BPF_OBJECT__NEXT_MAP
65 struct bpf_map *
66 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
67 {
68 #pragma GCC diagnostic push
69 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
70         return bpf_map__next(prev, obj);
71 #pragma GCC diagnostic pop
72 }
73 #endif
74
75 #ifndef HAVE_LIBBPF_BTF__RAW_DATA
76 const void *
77 btf__raw_data(const struct btf *btf_ro, __u32 *size)
78 {
79 #pragma GCC diagnostic push
80 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
81         return btf__get_raw_data(btf_ro, size);
82 #pragma GCC diagnostic pop
83 }
84 #endif
85
86 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len)
87 {
88         int ret = 0;
89         size_t i;
90
91         for (i = 0; i < len; i++)
92                 ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
93         return ret;
94 }
95
96 static int machine__process_bpf_event_load(struct machine *machine,
97                                            union perf_event *event,
98                                            struct perf_sample *sample __maybe_unused)
99 {
100         struct bpf_prog_info_node *info_node;
101         struct perf_env *env = machine->env;
102         struct perf_bpil *info_linear;
103         int id = event->bpf.id;
104         unsigned int i;
105
106         /* perf-record, no need to handle bpf-event */
107         if (env == NULL)
108                 return 0;
109
110         info_node = perf_env__find_bpf_prog_info(env, id);
111         if (!info_node)
112                 return 0;
113         info_linear = info_node->info_linear;
114
115         for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) {
116                 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms);
117                 u64 addr = addrs[i];
118                 struct map *map = maps__find(machine__kernel_maps(machine), addr);
119
120                 if (map) {
121                         map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO;
122                         map->dso->bpf_prog.id = id;
123                         map->dso->bpf_prog.sub_id = i;
124                         map->dso->bpf_prog.env = env;
125                 }
126         }
127         return 0;
128 }
129
130 int machine__process_bpf(struct machine *machine, union perf_event *event,
131                          struct perf_sample *sample)
132 {
133         if (dump_trace)
134                 perf_event__fprintf_bpf(event, stdout);
135
136         switch (event->bpf.type) {
137         case PERF_BPF_EVENT_PROG_LOAD:
138                 return machine__process_bpf_event_load(machine, event, sample);
139
140         case PERF_BPF_EVENT_PROG_UNLOAD:
141                 /*
142                  * Do not free bpf_prog_info and btf of the program here,
143                  * as annotation still need them. They will be freed at
144                  * the end of the session.
145                  */
146                 break;
147         default:
148                 pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
149                 break;
150         }
151         return 0;
152 }
153
154 static int perf_env__fetch_btf(struct perf_env *env,
155                                u32 btf_id,
156                                struct btf *btf)
157 {
158         struct btf_node *node;
159         u32 data_size;
160         const void *data;
161
162         data = btf__raw_data(btf, &data_size);
163
164         node = malloc(data_size + sizeof(struct btf_node));
165         if (!node)
166                 return -1;
167
168         node->id = btf_id;
169         node->data_size = data_size;
170         memcpy(node->data, data, data_size);
171
172         if (!perf_env__insert_btf(env, node)) {
173                 /* Insertion failed because of a duplicate. */
174                 free(node);
175                 return -1;
176         }
177         return 0;
178 }
179
180 static int synthesize_bpf_prog_name(char *buf, int size,
181                                     struct bpf_prog_info *info,
182                                     struct btf *btf,
183                                     u32 sub_id)
184 {
185         u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags);
186         void *func_infos = (void *)(uintptr_t)(info->func_info);
187         u32 sub_prog_cnt = info->nr_jited_ksyms;
188         const struct bpf_func_info *finfo;
189         const char *short_name = NULL;
190         const struct btf_type *t;
191         int name_len;
192
193         name_len = snprintf(buf, size, "bpf_prog_");
194         name_len += snprintf_hex(buf + name_len, size - name_len,
195                                  prog_tags[sub_id], BPF_TAG_SIZE);
196         if (btf) {
197                 finfo = func_infos + sub_id * info->func_info_rec_size;
198                 t = btf__type_by_id(btf, finfo->type_id);
199                 short_name = btf__name_by_offset(btf, t->name_off);
200         } else if (sub_id == 0 && sub_prog_cnt == 1) {
201                 /* no subprog */
202                 if (info->name[0])
203                         short_name = info->name;
204         } else
205                 short_name = "F";
206         if (short_name)
207                 name_len += snprintf(buf + name_len, size - name_len,
208                                      "_%s", short_name);
209         return name_len;
210 }
211
212 /*
213  * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf
214  * program. One PERF_RECORD_BPF_EVENT is generated for the program. And
215  * one PERF_RECORD_KSYMBOL is generated for each sub program.
216  *
217  * Returns:
218  *    0 for success;
219  *   -1 for failures;
220  *   -2 for lack of kernel support.
221  */
222 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session,
223                                                perf_event__handler_t process,
224                                                struct machine *machine,
225                                                int fd,
226                                                union perf_event *event,
227                                                struct record_opts *opts)
228 {
229         struct perf_record_ksymbol *ksymbol_event = &event->ksymbol;
230         struct perf_record_bpf_event *bpf_event = &event->bpf;
231         struct perf_tool *tool = session->tool;
232         struct bpf_prog_info_node *info_node;
233         struct perf_bpil *info_linear;
234         struct bpf_prog_info *info;
235         struct btf *btf = NULL;
236         struct perf_env *env;
237         u32 sub_prog_cnt, i;
238         int err = 0;
239         u64 arrays;
240
241         /*
242          * for perf-record and perf-report use header.env;
243          * otherwise, use global perf_env.
244          */
245         env = session->data ? &session->header.env : &perf_env;
246
247         arrays = 1UL << PERF_BPIL_JITED_KSYMS;
248         arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS;
249         arrays |= 1UL << PERF_BPIL_FUNC_INFO;
250         arrays |= 1UL << PERF_BPIL_PROG_TAGS;
251         arrays |= 1UL << PERF_BPIL_JITED_INSNS;
252         arrays |= 1UL << PERF_BPIL_LINE_INFO;
253         arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO;
254
255         info_linear = get_bpf_prog_info_linear(fd, arrays);
256         if (IS_ERR_OR_NULL(info_linear)) {
257                 info_linear = NULL;
258                 pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
259                 return -1;
260         }
261
262         if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) {
263                 free(info_linear);
264                 pr_debug("%s: the kernel is too old, aborting\n", __func__);
265                 return -2;
266         }
267
268         info = &info_linear->info;
269         if (!info->jited_ksyms) {
270                 free(info_linear);
271                 return -1;
272         }
273
274         /* number of ksyms, func_lengths, and tags should match */
275         sub_prog_cnt = info->nr_jited_ksyms;
276         if (sub_prog_cnt != info->nr_prog_tags ||
277             sub_prog_cnt != info->nr_jited_func_lens) {
278                 free(info_linear);
279                 return -1;
280         }
281
282         /* check BTF func info support */
283         if (info->btf_id && info->nr_func_info && info->func_info_rec_size) {
284                 /* btf func info number should be same as sub_prog_cnt */
285                 if (sub_prog_cnt != info->nr_func_info) {
286                         pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__);
287                         free(info_linear);
288                         return -1;
289                 }
290                 btf = btf__load_from_kernel_by_id(info->btf_id);
291                 if (libbpf_get_error(btf)) {
292                         pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id);
293                         err = -1;
294                         goto out;
295                 }
296                 perf_env__fetch_btf(env, info->btf_id, btf);
297         }
298
299         /* Synthesize PERF_RECORD_KSYMBOL */
300         for (i = 0; i < sub_prog_cnt; i++) {
301                 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
302                 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
303                 int name_len;
304
305                 *ksymbol_event = (struct perf_record_ksymbol) {
306                         .header = {
307                                 .type = PERF_RECORD_KSYMBOL,
308                                 .size = offsetof(struct perf_record_ksymbol, name),
309                         },
310                         .addr = prog_addrs[i],
311                         .len = prog_lens[i],
312                         .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
313                         .flags = 0,
314                 };
315
316                 name_len = synthesize_bpf_prog_name(ksymbol_event->name,
317                                                     KSYM_NAME_LEN, info, btf, i);
318                 ksymbol_event->header.size += PERF_ALIGN(name_len + 1,
319                                                          sizeof(u64));
320
321                 memset((void *)event + event->header.size, 0, machine->id_hdr_size);
322                 event->header.size += machine->id_hdr_size;
323                 err = perf_tool__process_synth_event(tool, event,
324                                                      machine, process);
325         }
326
327         if (!opts->no_bpf_event) {
328                 /* Synthesize PERF_RECORD_BPF_EVENT */
329                 *bpf_event = (struct perf_record_bpf_event) {
330                         .header = {
331                                 .type = PERF_RECORD_BPF_EVENT,
332                                 .size = sizeof(struct perf_record_bpf_event),
333                         },
334                         .type = PERF_BPF_EVENT_PROG_LOAD,
335                         .flags = 0,
336                         .id = info->id,
337                 };
338                 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE);
339                 memset((void *)event + event->header.size, 0, machine->id_hdr_size);
340                 event->header.size += machine->id_hdr_size;
341
342                 /* save bpf_prog_info to env */
343                 info_node = malloc(sizeof(struct bpf_prog_info_node));
344                 if (!info_node) {
345                         err = -1;
346                         goto out;
347                 }
348
349                 info_node->info_linear = info_linear;
350                 perf_env__insert_bpf_prog_info(env, info_node);
351                 info_linear = NULL;
352
353                 /*
354                  * process after saving bpf_prog_info to env, so that
355                  * required information is ready for look up
356                  */
357                 err = perf_tool__process_synth_event(tool, event,
358                                                      machine, process);
359         }
360
361 out:
362         free(info_linear);
363         btf__free(btf);
364         return err ? -1 : 0;
365 }
366
367 struct kallsyms_parse {
368         union perf_event        *event;
369         perf_event__handler_t    process;
370         struct machine          *machine;
371         struct perf_tool        *tool;
372 };
373
374 static int
375 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data)
376 {
377         struct machine *machine = data->machine;
378         union perf_event *event = data->event;
379         struct perf_record_ksymbol *ksymbol;
380         int len;
381
382         ksymbol = &event->ksymbol;
383
384         *ksymbol = (struct perf_record_ksymbol) {
385                 .header = {
386                         .type = PERF_RECORD_KSYMBOL,
387                         .size = offsetof(struct perf_record_ksymbol, name),
388                 },
389                 .addr      = addr,
390                 .len       = page_size,
391                 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
392                 .flags     = 0,
393         };
394
395         len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name);
396         ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64));
397         memset((void *) event + event->header.size, 0, machine->id_hdr_size);
398         event->header.size += machine->id_hdr_size;
399
400         return perf_tool__process_synth_event(data->tool, event, machine,
401                                               data->process);
402 }
403
404 static int
405 kallsyms_process_symbol(void *data, const char *_name,
406                         char type __maybe_unused, u64 start)
407 {
408         char disp[KSYM_NAME_LEN];
409         char *module, *name;
410         unsigned long id;
411         int err = 0;
412
413         module = strchr(_name, '\t');
414         if (!module)
415                 return 0;
416
417         /* We are going after [bpf] module ... */
418         if (strcmp(module + 1, "[bpf]"))
419                 return 0;
420
421         name = memdup(_name, (module - _name) + 1);
422         if (!name)
423                 return -ENOMEM;
424
425         name[module - _name] = 0;
426
427         /* .. and only for trampolines and dispatchers */
428         if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) ||
429             (sscanf(name, "bpf_dispatcher_%s", disp) == 1))
430                 err = process_bpf_image(name, start, data);
431
432         free(name);
433         return err;
434 }
435
436 int perf_event__synthesize_bpf_events(struct perf_session *session,
437                                       perf_event__handler_t process,
438                                       struct machine *machine,
439                                       struct record_opts *opts)
440 {
441         const char *kallsyms_filename = "/proc/kallsyms";
442         struct kallsyms_parse arg;
443         union perf_event *event;
444         __u32 id = 0;
445         int err;
446         int fd;
447
448         event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size);
449         if (!event)
450                 return -1;
451
452         /* Synthesize all the bpf programs in system. */
453         while (true) {
454                 err = bpf_prog_get_next_id(id, &id);
455                 if (err) {
456                         if (errno == ENOENT) {
457                                 err = 0;
458                                 break;
459                         }
460                         pr_debug("%s: can't get next program: %s%s\n",
461                                  __func__, strerror(errno),
462                                  errno == EINVAL ? " -- kernel too old?" : "");
463                         /* don't report error on old kernel or EPERM  */
464                         err = (errno == EINVAL || errno == EPERM) ? 0 : -1;
465                         break;
466                 }
467                 fd = bpf_prog_get_fd_by_id(id);
468                 if (fd < 0) {
469                         pr_debug("%s: failed to get fd for prog_id %u\n",
470                                  __func__, id);
471                         continue;
472                 }
473
474                 err = perf_event__synthesize_one_bpf_prog(session, process,
475                                                           machine, fd,
476                                                           event, opts);
477                 close(fd);
478                 if (err) {
479                         /* do not return error for old kernel */
480                         if (err == -2)
481                                 err = 0;
482                         break;
483                 }
484         }
485
486         /* Synthesize all the bpf images - trampolines/dispatchers. */
487         if (symbol_conf.kallsyms_name != NULL)
488                 kallsyms_filename = symbol_conf.kallsyms_name;
489
490         arg = (struct kallsyms_parse) {
491                 .event   = event,
492                 .process = process,
493                 .machine = machine,
494                 .tool    = session->tool,
495         };
496
497         if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) {
498                 pr_err("%s: failed to synthesize bpf images: %s\n",
499                        __func__, strerror(errno));
500         }
501
502         free(event);
503         return err;
504 }
505
506 static void perf_env__add_bpf_info(struct perf_env *env, u32 id)
507 {
508         struct bpf_prog_info_node *info_node;
509         struct perf_bpil *info_linear;
510         struct btf *btf = NULL;
511         u64 arrays;
512         u32 btf_id;
513         int fd;
514
515         fd = bpf_prog_get_fd_by_id(id);
516         if (fd < 0)
517                 return;
518
519         arrays = 1UL << PERF_BPIL_JITED_KSYMS;
520         arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS;
521         arrays |= 1UL << PERF_BPIL_FUNC_INFO;
522         arrays |= 1UL << PERF_BPIL_PROG_TAGS;
523         arrays |= 1UL << PERF_BPIL_JITED_INSNS;
524         arrays |= 1UL << PERF_BPIL_LINE_INFO;
525         arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO;
526
527         info_linear = get_bpf_prog_info_linear(fd, arrays);
528         if (IS_ERR_OR_NULL(info_linear)) {
529                 pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
530                 goto out;
531         }
532
533         btf_id = info_linear->info.btf_id;
534
535         info_node = malloc(sizeof(struct bpf_prog_info_node));
536         if (info_node) {
537                 info_node->info_linear = info_linear;
538                 perf_env__insert_bpf_prog_info(env, info_node);
539         } else
540                 free(info_linear);
541
542         if (btf_id == 0)
543                 goto out;
544
545         btf = btf__load_from_kernel_by_id(btf_id);
546         if (libbpf_get_error(btf)) {
547                 pr_debug("%s: failed to get BTF of id %u, aborting\n",
548                          __func__, btf_id);
549                 goto out;
550         }
551         perf_env__fetch_btf(env, btf_id, btf);
552
553 out:
554         btf__free(btf);
555         close(fd);
556 }
557
558 static int bpf_event__sb_cb(union perf_event *event, void *data)
559 {
560         struct perf_env *env = data;
561
562         if (event->header.type != PERF_RECORD_BPF_EVENT)
563                 return -1;
564
565         switch (event->bpf.type) {
566         case PERF_BPF_EVENT_PROG_LOAD:
567                 perf_env__add_bpf_info(env, event->bpf.id);
568
569         case PERF_BPF_EVENT_PROG_UNLOAD:
570                 /*
571                  * Do not free bpf_prog_info and btf of the program here,
572                  * as annotation still need them. They will be freed at
573                  * the end of the session.
574                  */
575                 break;
576         default:
577                 pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
578                 break;
579         }
580
581         return 0;
582 }
583
584 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env)
585 {
586         struct perf_event_attr attr = {
587                 .type             = PERF_TYPE_SOFTWARE,
588                 .config           = PERF_COUNT_SW_DUMMY,
589                 .sample_id_all    = 1,
590                 .watermark        = 1,
591                 .bpf_event        = 1,
592                 .size      = sizeof(attr), /* to capture ABI version */
593         };
594
595         /*
596          * Older gcc versions don't support designated initializers, like above,
597          * for unnamed union members, such as the following:
598          */
599         attr.wakeup_watermark = 1;
600
601         return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env);
602 }
603
604 void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info,
605                                     struct perf_env *env,
606                                     FILE *fp)
607 {
608         __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
609         __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
610         char name[KSYM_NAME_LEN];
611         struct btf *btf = NULL;
612         u32 sub_prog_cnt, i;
613
614         sub_prog_cnt = info->nr_jited_ksyms;
615         if (sub_prog_cnt != info->nr_prog_tags ||
616             sub_prog_cnt != info->nr_jited_func_lens)
617                 return;
618
619         if (info->btf_id) {
620                 struct btf_node *node;
621
622                 node = perf_env__find_btf(env, info->btf_id);
623                 if (node)
624                         btf = btf__new((__u8 *)(node->data),
625                                        node->data_size);
626         }
627
628         if (sub_prog_cnt == 1) {
629                 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0);
630                 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n",
631                         info->id, name, prog_addrs[0], prog_lens[0]);
632                 goto out;
633         }
634
635         fprintf(fp, "# bpf_prog_info %u:\n", info->id);
636         for (i = 0; i < sub_prog_cnt; i++) {
637                 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i);
638
639                 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n",
640                         i, name, prog_addrs[i], prog_lens[i]);
641         }
642 out:
643         btf__free(btf);
644 }