1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2018 Facebook */
4 #include <uapi/linux/btf.h>
5 #include <uapi/linux/types.h>
6 #include <linux/seq_file.h>
7 #include <linux/compiler.h>
8 #include <linux/ctype.h>
9 #include <linux/errno.h>
10 #include <linux/slab.h>
11 #include <linux/anon_inodes.h>
12 #include <linux/file.h>
13 #include <linux/uaccess.h>
14 #include <linux/kernel.h>
15 #include <linux/idr.h>
16 #include <linux/sort.h>
17 #include <linux/bpf_verifier.h>
18 #include <linux/btf.h>
20 /* BTF (BPF Type Format) is the meta data format which describes
21 * the data types of BPF program/map. Hence, it basically focus
22 * on the C programming language which the modern BPF is primary
27 * The BTF data is stored under the ".BTF" ELF section
31 * Each 'struct btf_type' object describes a C data type.
32 * Depending on the type it is describing, a 'struct btf_type'
33 * object may be followed by more data. F.e.
34 * To describe an array, 'struct btf_type' is followed by
37 * 'struct btf_type' and any extra data following it are
42 * The BTF type section contains a list of 'struct btf_type' objects.
43 * Each one describes a C type. Recall from the above section
44 * that a 'struct btf_type' object could be immediately followed by extra
45 * data in order to desribe some particular C types.
49 * Each btf_type object is identified by a type_id. The type_id
50 * is implicitly implied by the location of the btf_type object in
51 * the BTF type section. The first one has type_id 1. The second
52 * one has type_id 2...etc. Hence, an earlier btf_type has
55 * A btf_type object may refer to another btf_type object by using
56 * type_id (i.e. the "type" in the "struct btf_type").
58 * NOTE that we cannot assume any reference-order.
59 * A btf_type object can refer to an earlier btf_type object
60 * but it can also refer to a later btf_type object.
62 * For example, to describe "const void *". A btf_type
63 * object describing "const" may refer to another btf_type
64 * object describing "void *". This type-reference is done
65 * by specifying type_id:
67 * [1] CONST (anon) type_id=2
68 * [2] PTR (anon) type_id=0
70 * The above is the btf_verifier debug log:
71 * - Each line started with "[?]" is a btf_type object
72 * - [?] is the type_id of the btf_type object.
73 * - CONST/PTR is the BTF_KIND_XXX
74 * - "(anon)" is the name of the type. It just
75 * happens that CONST and PTR has no name.
76 * - type_id=XXX is the 'u32 type' in btf_type
78 * NOTE: "void" has type_id 0
82 * The BTF string section contains the names used by the type section.
83 * Each string is referred by an "offset" from the beginning of the
86 * Each string is '\0' terminated.
88 * The first character in the string section must be '\0'
89 * which is used to mean 'anonymous'. Some btf_type may not
95 * To verify BTF data, two passes are needed.
99 * The first pass is to collect all btf_type objects to
100 * an array: "btf->types".
102 * Depending on the C type that a btf_type is describing,
103 * a btf_type may be followed by extra data. We don't know
104 * how many btf_type is there, and more importantly we don't
105 * know where each btf_type is located in the type section.
107 * Without knowing the location of each type_id, most verifications
108 * cannot be done. e.g. an earlier btf_type may refer to a later
109 * btf_type (recall the "const void *" above), so we cannot
110 * check this type-reference in the first pass.
112 * In the first pass, it still does some verifications (e.g.
113 * checking the name is a valid offset to the string section).
117 * The main focus is to resolve a btf_type that is referring
120 * We have to ensure the referring type:
121 * 1) does exist in the BTF (i.e. in btf->types[])
122 * 2) does not cause a loop:
131 * btf_type_needs_resolve() decides if a btf_type needs
134 * The needs_resolve type implements the "resolve()" ops which
135 * essentially does a DFS and detects backedge.
137 * During resolve (or DFS), different C types have different
138 * "RESOLVED" conditions.
140 * When resolving a BTF_KIND_STRUCT, we need to resolve all its
141 * members because a member is always referring to another
142 * type. A struct's member can be treated as "RESOLVED" if
143 * it is referring to a BTF_KIND_PTR. Otherwise, the
144 * following valid C struct would be rejected:
151 * When resolving a BTF_KIND_PTR, it needs to keep resolving if
152 * it is referring to another BTF_KIND_PTR. Otherwise, we cannot
153 * detect a pointer loop, e.g.:
154 * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR +
156 * +-----------------------------------------+
160 #define BITS_PER_U64 (sizeof(u64) * BITS_PER_BYTE)
161 #define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
162 #define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
163 #define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
164 #define BITS_ROUNDUP_BYTES(bits) \
165 (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
167 #define BTF_INFO_MASK 0x0f00ffff
168 #define BTF_INT_MASK 0x0fffffff
169 #define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE)
170 #define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET)
172 /* 16MB for 64k structs and each has 16 members and
173 * a few MB spaces for the string section.
174 * The hard limit is S32_MAX.
176 #define BTF_MAX_SIZE (16 * 1024 * 1024)
178 #define for_each_member(i, struct_type, member) \
179 for (i = 0, member = btf_type_member(struct_type); \
180 i < btf_type_vlen(struct_type); \
183 #define for_each_member_from(i, from, struct_type, member) \
184 for (i = from, member = btf_type_member(struct_type) + from; \
185 i < btf_type_vlen(struct_type); \
188 static DEFINE_IDR(btf_idr);
189 static DEFINE_SPINLOCK(btf_idr_lock);
193 struct btf_type **types;
198 struct btf_header hdr;
207 enum verifier_phase {
212 struct resolve_vertex {
213 const struct btf_type *t;
225 RESOLVE_TBD, /* To Be Determined */
226 RESOLVE_PTR, /* Resolving for Pointer */
227 RESOLVE_STRUCT_OR_ARRAY, /* Resolving for struct/union
232 #define MAX_RESOLVE_DEPTH 32
234 struct btf_sec_info {
239 struct btf_verifier_env {
242 struct resolve_vertex stack[MAX_RESOLVE_DEPTH];
243 struct bpf_verifier_log log;
246 enum verifier_phase phase;
247 enum resolve_mode resolve_mode;
250 static const char * const btf_kind_str[NR_BTF_KINDS] = {
251 [BTF_KIND_UNKN] = "UNKNOWN",
252 [BTF_KIND_INT] = "INT",
253 [BTF_KIND_PTR] = "PTR",
254 [BTF_KIND_ARRAY] = "ARRAY",
255 [BTF_KIND_STRUCT] = "STRUCT",
256 [BTF_KIND_UNION] = "UNION",
257 [BTF_KIND_ENUM] = "ENUM",
258 [BTF_KIND_FWD] = "FWD",
259 [BTF_KIND_TYPEDEF] = "TYPEDEF",
260 [BTF_KIND_VOLATILE] = "VOLATILE",
261 [BTF_KIND_CONST] = "CONST",
262 [BTF_KIND_RESTRICT] = "RESTRICT",
265 struct btf_kind_operations {
266 s32 (*check_meta)(struct btf_verifier_env *env,
267 const struct btf_type *t,
269 int (*resolve)(struct btf_verifier_env *env,
270 const struct resolve_vertex *v);
271 int (*check_member)(struct btf_verifier_env *env,
272 const struct btf_type *struct_type,
273 const struct btf_member *member,
274 const struct btf_type *member_type);
275 void (*log_details)(struct btf_verifier_env *env,
276 const struct btf_type *t);
277 void (*seq_show)(const struct btf *btf, const struct btf_type *t,
278 u32 type_id, void *data, u8 bits_offsets,
282 static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
283 static struct btf_type btf_void;
285 static bool btf_type_is_modifier(const struct btf_type *t)
287 /* Some of them is not strictly a C modifier
288 * but they are grouped into the same bucket
290 * A type (t) that refers to another
291 * type through t->type AND its size cannot
292 * be determined without following the t->type.
294 * ptr does not fall into this bucket
295 * because its size is always sizeof(void *).
297 switch (BTF_INFO_KIND(t->info)) {
298 case BTF_KIND_TYPEDEF:
299 case BTF_KIND_VOLATILE:
301 case BTF_KIND_RESTRICT:
308 static bool btf_type_is_void(const struct btf_type *t)
310 /* void => no type and size info.
311 * Hence, FWD is also treated as void.
313 return t == &btf_void || BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
316 static bool btf_type_is_void_or_null(const struct btf_type *t)
318 return !t || btf_type_is_void(t);
321 /* union is only a special case of struct:
322 * all its offsetof(member) == 0
324 static bool btf_type_is_struct(const struct btf_type *t)
326 u8 kind = BTF_INFO_KIND(t->info);
328 return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;
331 static bool btf_type_is_array(const struct btf_type *t)
333 return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
336 static bool btf_type_is_ptr(const struct btf_type *t)
338 return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
341 static bool btf_type_is_int(const struct btf_type *t)
343 return BTF_INFO_KIND(t->info) == BTF_KIND_INT;
346 /* What types need to be resolved?
348 * btf_type_is_modifier() is an obvious one.
350 * btf_type_is_struct() because its member refers to
351 * another type (through member->type).
353 * btf_type_is_array() because its element (array->type)
354 * refers to another type. Array can be thought of a
355 * special case of struct while array just has the same
356 * member-type repeated by array->nelems of times.
358 static bool btf_type_needs_resolve(const struct btf_type *t)
360 return btf_type_is_modifier(t) ||
361 btf_type_is_ptr(t) ||
362 btf_type_is_struct(t) ||
363 btf_type_is_array(t);
366 /* t->size can be used */
367 static bool btf_type_has_size(const struct btf_type *t)
369 switch (BTF_INFO_KIND(t->info)) {
371 case BTF_KIND_STRUCT:
380 static const char *btf_int_encoding_str(u8 encoding)
384 else if (encoding == BTF_INT_SIGNED)
386 else if (encoding == BTF_INT_CHAR)
388 else if (encoding == BTF_INT_BOOL)
394 static u16 btf_type_vlen(const struct btf_type *t)
396 return BTF_INFO_VLEN(t->info);
399 static u32 btf_type_int(const struct btf_type *t)
401 return *(u32 *)(t + 1);
404 static const struct btf_array *btf_type_array(const struct btf_type *t)
406 return (const struct btf_array *)(t + 1);
409 static const struct btf_member *btf_type_member(const struct btf_type *t)
411 return (const struct btf_member *)(t + 1);
414 static const struct btf_enum *btf_type_enum(const struct btf_type *t)
416 return (const struct btf_enum *)(t + 1);
419 static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
421 return kind_ops[BTF_INFO_KIND(t->info)];
424 static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
426 return BTF_STR_OFFSET_VALID(offset) &&
427 offset < btf->hdr.str_len;
430 /* Only C-style identifier is permitted. This can be relaxed if
433 static bool btf_name_valid_identifier(const struct btf *btf, u32 offset)
435 /* offset must be valid */
436 const char *src = &btf->strings[offset];
437 const char *src_limit;
439 if (!isalpha(*src) && *src != '_')
442 /* set a limit on identifier length */
443 src_limit = src + KSYM_NAME_LEN;
445 while (*src && src < src_limit) {
446 if (!isalnum(*src) && *src != '_')
454 static const char *btf_name_by_offset(const struct btf *btf, u32 offset)
458 else if (offset < btf->hdr.str_len)
459 return &btf->strings[offset];
461 return "(invalid-name-offset)";
464 static const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
466 if (type_id > btf->nr_types)
469 return btf->types[type_id];
473 * Regular int is not a bit field and it must be either
476 static bool btf_type_int_is_regular(const struct btf_type *t)
478 u8 nr_bits, nr_bytes;
481 int_data = btf_type_int(t);
482 nr_bits = BTF_INT_BITS(int_data);
483 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
484 if (BITS_PER_BYTE_MASKED(nr_bits) ||
485 BTF_INT_OFFSET(int_data) ||
486 (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) &&
487 nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64))) {
494 __printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
495 const char *fmt, ...)
500 bpf_verifier_vlog(log, fmt, args);
504 __printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,
505 const char *fmt, ...)
507 struct bpf_verifier_log *log = &env->log;
510 if (!bpf_verifier_log_needed(log))
514 bpf_verifier_vlog(log, fmt, args);
518 __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
519 const struct btf_type *t,
521 const char *fmt, ...)
523 struct bpf_verifier_log *log = &env->log;
524 u8 kind = BTF_INFO_KIND(t->info);
525 struct btf *btf = env->btf;
528 if (!bpf_verifier_log_needed(log))
531 __btf_verifier_log(log, "[%u] %s %s%s",
534 btf_name_by_offset(btf, t->name_off),
535 log_details ? " " : "");
538 btf_type_ops(t)->log_details(env, t);
541 __btf_verifier_log(log, " ");
543 bpf_verifier_vlog(log, fmt, args);
547 __btf_verifier_log(log, "\n");
550 #define btf_verifier_log_type(env, t, ...) \
551 __btf_verifier_log_type((env), (t), true, __VA_ARGS__)
552 #define btf_verifier_log_basic(env, t, ...) \
553 __btf_verifier_log_type((env), (t), false, __VA_ARGS__)
556 static void btf_verifier_log_member(struct btf_verifier_env *env,
557 const struct btf_type *struct_type,
558 const struct btf_member *member,
559 const char *fmt, ...)
561 struct bpf_verifier_log *log = &env->log;
562 struct btf *btf = env->btf;
565 if (!bpf_verifier_log_needed(log))
568 /* The CHECK_META phase already did a btf dump.
570 * If member is logged again, it must hit an error in
571 * parsing this member. It is useful to print out which
572 * struct this member belongs to.
574 if (env->phase != CHECK_META)
575 btf_verifier_log_type(env, struct_type, NULL);
577 __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u",
578 btf_name_by_offset(btf, member->name_off),
579 member->type, member->offset);
582 __btf_verifier_log(log, " ");
584 bpf_verifier_vlog(log, fmt, args);
588 __btf_verifier_log(log, "\n");
591 static void btf_verifier_log_hdr(struct btf_verifier_env *env,
594 struct bpf_verifier_log *log = &env->log;
595 const struct btf *btf = env->btf;
596 const struct btf_header *hdr;
598 if (!bpf_verifier_log_needed(log))
602 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
603 __btf_verifier_log(log, "version: %u\n", hdr->version);
604 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags);
605 __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len);
606 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off);
607 __btf_verifier_log(log, "type_len: %u\n", hdr->type_len);
608 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
609 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
610 __btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size);
613 static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
615 struct btf *btf = env->btf;
617 /* < 2 because +1 for btf_void which is always in btf->types[0].
618 * btf_void is not accounted in btf->nr_types because btf_void
619 * does not come from the BTF file.
621 if (btf->types_size - btf->nr_types < 2) {
622 /* Expand 'types' array */
624 struct btf_type **new_types;
625 u32 expand_by, new_size;
627 if (btf->types_size == BTF_MAX_TYPE) {
628 btf_verifier_log(env, "Exceeded max num of types");
632 expand_by = max_t(u32, btf->types_size >> 2, 16);
633 new_size = min_t(u32, BTF_MAX_TYPE,
634 btf->types_size + expand_by);
636 new_types = kvcalloc(new_size, sizeof(*new_types),
637 GFP_KERNEL | __GFP_NOWARN);
641 if (btf->nr_types == 0)
642 new_types[0] = &btf_void;
644 memcpy(new_types, btf->types,
645 sizeof(*btf->types) * (btf->nr_types + 1));
648 btf->types = new_types;
649 btf->types_size = new_size;
652 btf->types[++(btf->nr_types)] = t;
657 static int btf_alloc_id(struct btf *btf)
661 idr_preload(GFP_KERNEL);
662 spin_lock_bh(&btf_idr_lock);
663 id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC);
666 spin_unlock_bh(&btf_idr_lock);
669 if (WARN_ON_ONCE(!id))
672 return id > 0 ? 0 : id;
675 static void btf_free_id(struct btf *btf)
680 * In map-in-map, calling map_delete_elem() on outer
681 * map will call bpf_map_put on the inner map.
682 * It will then eventually call btf_free_id()
683 * on the inner map. Some of the map_delete_elem()
684 * implementation may have irq disabled, so
685 * we need to use the _irqsave() version instead
686 * of the _bh() version.
688 spin_lock_irqsave(&btf_idr_lock, flags);
689 idr_remove(&btf_idr, btf->id);
690 spin_unlock_irqrestore(&btf_idr_lock, flags);
693 static void btf_free(struct btf *btf)
696 kvfree(btf->resolved_sizes);
697 kvfree(btf->resolved_ids);
702 static void btf_free_rcu(struct rcu_head *rcu)
704 struct btf *btf = container_of(rcu, struct btf, rcu);
709 void btf_put(struct btf *btf)
711 if (btf && refcount_dec_and_test(&btf->refcnt)) {
713 call_rcu(&btf->rcu, btf_free_rcu);
717 static int env_resolve_init(struct btf_verifier_env *env)
719 struct btf *btf = env->btf;
720 u32 nr_types = btf->nr_types;
721 u32 *resolved_sizes = NULL;
722 u32 *resolved_ids = NULL;
723 u8 *visit_states = NULL;
725 /* +1 for btf_void */
726 resolved_sizes = kvcalloc(nr_types + 1, sizeof(*resolved_sizes),
727 GFP_KERNEL | __GFP_NOWARN);
731 resolved_ids = kvcalloc(nr_types + 1, sizeof(*resolved_ids),
732 GFP_KERNEL | __GFP_NOWARN);
736 visit_states = kvcalloc(nr_types + 1, sizeof(*visit_states),
737 GFP_KERNEL | __GFP_NOWARN);
741 btf->resolved_sizes = resolved_sizes;
742 btf->resolved_ids = resolved_ids;
743 env->visit_states = visit_states;
748 kvfree(resolved_sizes);
749 kvfree(resolved_ids);
750 kvfree(visit_states);
754 static void btf_verifier_env_free(struct btf_verifier_env *env)
756 kvfree(env->visit_states);
760 static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
761 const struct btf_type *next_type)
763 switch (env->resolve_mode) {
765 /* int, enum or void is a sink */
766 return !btf_type_needs_resolve(next_type);
768 /* int, enum, void, struct or array is a sink for ptr */
769 return !btf_type_is_modifier(next_type) &&
770 !btf_type_is_ptr(next_type);
771 case RESOLVE_STRUCT_OR_ARRAY:
772 /* int, enum, void or ptr is a sink for struct and array */
773 return !btf_type_is_modifier(next_type) &&
774 !btf_type_is_array(next_type) &&
775 !btf_type_is_struct(next_type);
781 static bool env_type_is_resolved(const struct btf_verifier_env *env,
784 return env->visit_states[type_id] == RESOLVED;
787 static int env_stack_push(struct btf_verifier_env *env,
788 const struct btf_type *t, u32 type_id)
790 struct resolve_vertex *v;
792 if (env->top_stack == MAX_RESOLVE_DEPTH)
795 if (env->visit_states[type_id] != NOT_VISITED)
798 env->visit_states[type_id] = VISITED;
800 v = &env->stack[env->top_stack++];
802 v->type_id = type_id;
805 if (env->resolve_mode == RESOLVE_TBD) {
806 if (btf_type_is_ptr(t))
807 env->resolve_mode = RESOLVE_PTR;
808 else if (btf_type_is_struct(t) || btf_type_is_array(t))
809 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;
815 static void env_stack_set_next_member(struct btf_verifier_env *env,
818 env->stack[env->top_stack - 1].next_member = next_member;
821 static void env_stack_pop_resolved(struct btf_verifier_env *env,
822 u32 resolved_type_id,
825 u32 type_id = env->stack[--(env->top_stack)].type_id;
826 struct btf *btf = env->btf;
828 btf->resolved_sizes[type_id] = resolved_size;
829 btf->resolved_ids[type_id] = resolved_type_id;
830 env->visit_states[type_id] = RESOLVED;
833 static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)
835 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;
838 /* The input param "type_id" must point to a needs_resolve type */
839 static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
842 *type_id = btf->resolved_ids[*type_id];
843 return btf_type_by_id(btf, *type_id);
846 const struct btf_type *btf_type_id_size(const struct btf *btf,
847 u32 *type_id, u32 *ret_size)
849 const struct btf_type *size_type;
850 u32 size_type_id = *type_id;
853 size_type = btf_type_by_id(btf, size_type_id);
854 if (btf_type_is_void_or_null(size_type))
857 if (btf_type_has_size(size_type)) {
858 size = size_type->size;
859 } else if (btf_type_is_array(size_type)) {
860 size = btf->resolved_sizes[size_type_id];
861 } else if (btf_type_is_ptr(size_type)) {
862 size = sizeof(void *);
864 if (WARN_ON_ONCE(!btf_type_is_modifier(size_type)))
867 size = btf->resolved_sizes[size_type_id];
868 size_type_id = btf->resolved_ids[size_type_id];
869 size_type = btf_type_by_id(btf, size_type_id);
870 if (btf_type_is_void(size_type))
874 *type_id = size_type_id;
881 static int btf_df_check_member(struct btf_verifier_env *env,
882 const struct btf_type *struct_type,
883 const struct btf_member *member,
884 const struct btf_type *member_type)
886 btf_verifier_log_basic(env, struct_type,
887 "Unsupported check_member");
891 static int btf_df_resolve(struct btf_verifier_env *env,
892 const struct resolve_vertex *v)
894 btf_verifier_log_basic(env, v->t, "Unsupported resolve");
898 static void btf_df_seq_show(const struct btf *btf, const struct btf_type *t,
899 u32 type_id, void *data, u8 bits_offsets,
902 seq_printf(m, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
905 static int btf_int_check_member(struct btf_verifier_env *env,
906 const struct btf_type *struct_type,
907 const struct btf_member *member,
908 const struct btf_type *member_type)
910 u32 int_data = btf_type_int(member_type);
911 u32 struct_bits_off = member->offset;
912 u32 struct_size = struct_type->size;
916 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) {
917 btf_verifier_log_member(env, struct_type, member,
918 "bits_offset exceeds U32_MAX");
922 struct_bits_off += BTF_INT_OFFSET(int_data);
923 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
924 nr_copy_bits = BTF_INT_BITS(int_data) +
925 BITS_PER_BYTE_MASKED(struct_bits_off);
927 if (nr_copy_bits > BITS_PER_U64) {
928 btf_verifier_log_member(env, struct_type, member,
929 "nr_copy_bits exceeds 64");
933 if (struct_size < bytes_offset ||
934 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
935 btf_verifier_log_member(env, struct_type, member,
936 "Member exceeds struct_size");
943 static s32 btf_int_check_meta(struct btf_verifier_env *env,
944 const struct btf_type *t,
947 u32 int_data, nr_bits, meta_needed = sizeof(int_data);
950 if (meta_left < meta_needed) {
951 btf_verifier_log_basic(env, t,
952 "meta_left:%u meta_needed:%u",
953 meta_left, meta_needed);
957 if (btf_type_vlen(t)) {
958 btf_verifier_log_type(env, t, "vlen != 0");
962 int_data = btf_type_int(t);
963 if (int_data & ~BTF_INT_MASK) {
964 btf_verifier_log_basic(env, t, "Invalid int_data:%x",
969 nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data);
971 if (nr_bits > BITS_PER_U64) {
972 btf_verifier_log_type(env, t, "nr_bits exceeds %zu",
977 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) {
978 btf_verifier_log_type(env, t, "nr_bits exceeds type_size");
983 * Only one of the encoding bits is allowed and it
984 * should be sufficient for the pretty print purpose (i.e. decoding).
985 * Multiple bits can be allowed later if it is found
986 * to be insufficient.
988 encoding = BTF_INT_ENCODING(int_data);
990 encoding != BTF_INT_SIGNED &&
991 encoding != BTF_INT_CHAR &&
992 encoding != BTF_INT_BOOL) {
993 btf_verifier_log_type(env, t, "Unsupported encoding");
997 btf_verifier_log_type(env, t, NULL);
1002 static void btf_int_log(struct btf_verifier_env *env,
1003 const struct btf_type *t)
1005 int int_data = btf_type_int(t);
1007 btf_verifier_log(env,
1008 "size=%u bits_offset=%u nr_bits=%u encoding=%s",
1009 t->size, BTF_INT_OFFSET(int_data),
1010 BTF_INT_BITS(int_data),
1011 btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
1014 static void btf_int_bits_seq_show(const struct btf *btf,
1015 const struct btf_type *t,
1016 void *data, u8 bits_offset,
1019 u16 left_shift_bits, right_shift_bits;
1020 u32 int_data = btf_type_int(t);
1021 u8 nr_bits = BTF_INT_BITS(int_data);
1022 u8 total_bits_offset;
1028 * bits_offset is at most 7.
1029 * BTF_INT_OFFSET() cannot exceed 64 bits.
1031 total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
1032 data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
1033 bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
1034 nr_copy_bits = nr_bits + bits_offset;
1035 nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
1038 memcpy(&print_num, data, nr_copy_bytes);
1040 #ifdef __BIG_ENDIAN_BITFIELD
1041 left_shift_bits = bits_offset;
1043 left_shift_bits = BITS_PER_U64 - nr_copy_bits;
1045 right_shift_bits = BITS_PER_U64 - nr_bits;
1047 print_num <<= left_shift_bits;
1048 print_num >>= right_shift_bits;
1050 seq_printf(m, "0x%llx", print_num);
1053 static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t,
1054 u32 type_id, void *data, u8 bits_offset,
1057 u32 int_data = btf_type_int(t);
1058 u8 encoding = BTF_INT_ENCODING(int_data);
1059 bool sign = encoding & BTF_INT_SIGNED;
1060 u8 nr_bits = BTF_INT_BITS(int_data);
1062 if (bits_offset || BTF_INT_OFFSET(int_data) ||
1063 BITS_PER_BYTE_MASKED(nr_bits)) {
1064 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
1071 seq_printf(m, "%lld", *(s64 *)data);
1073 seq_printf(m, "%llu", *(u64 *)data);
1077 seq_printf(m, "%d", *(s32 *)data);
1079 seq_printf(m, "%u", *(u32 *)data);
1083 seq_printf(m, "%d", *(s16 *)data);
1085 seq_printf(m, "%u", *(u16 *)data);
1089 seq_printf(m, "%d", *(s8 *)data);
1091 seq_printf(m, "%u", *(u8 *)data);
1094 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
1098 static const struct btf_kind_operations int_ops = {
1099 .check_meta = btf_int_check_meta,
1100 .resolve = btf_df_resolve,
1101 .check_member = btf_int_check_member,
1102 .log_details = btf_int_log,
1103 .seq_show = btf_int_seq_show,
1106 static int btf_modifier_check_member(struct btf_verifier_env *env,
1107 const struct btf_type *struct_type,
1108 const struct btf_member *member,
1109 const struct btf_type *member_type)
1111 const struct btf_type *resolved_type;
1112 u32 resolved_type_id = member->type;
1113 struct btf_member resolved_member;
1114 struct btf *btf = env->btf;
1116 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
1117 if (!resolved_type) {
1118 btf_verifier_log_member(env, struct_type, member,
1123 resolved_member = *member;
1124 resolved_member.type = resolved_type_id;
1126 return btf_type_ops(resolved_type)->check_member(env, struct_type,
1131 static int btf_ptr_check_member(struct btf_verifier_env *env,
1132 const struct btf_type *struct_type,
1133 const struct btf_member *member,
1134 const struct btf_type *member_type)
1136 u32 struct_size, struct_bits_off, bytes_offset;
1138 struct_size = struct_type->size;
1139 struct_bits_off = member->offset;
1140 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1142 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1143 btf_verifier_log_member(env, struct_type, member,
1144 "Member is not byte aligned");
1148 if (struct_size - bytes_offset < sizeof(void *)) {
1149 btf_verifier_log_member(env, struct_type, member,
1150 "Member exceeds struct_size");
1157 static int btf_ref_type_check_meta(struct btf_verifier_env *env,
1158 const struct btf_type *t,
1161 if (btf_type_vlen(t)) {
1162 btf_verifier_log_type(env, t, "vlen != 0");
1166 if (!BTF_TYPE_ID_VALID(t->type)) {
1167 btf_verifier_log_type(env, t, "Invalid type_id");
1171 /* typedef type must have a valid name, and other ref types,
1172 * volatile, const, restrict, should have a null name.
1174 if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) {
1176 !btf_name_valid_identifier(env->btf, t->name_off)) {
1177 btf_verifier_log_type(env, t, "Invalid name");
1182 btf_verifier_log_type(env, t, "Invalid name");
1187 btf_verifier_log_type(env, t, NULL);
1192 static int btf_modifier_resolve(struct btf_verifier_env *env,
1193 const struct resolve_vertex *v)
1195 const struct btf_type *t = v->t;
1196 const struct btf_type *next_type;
1197 u32 next_type_id = t->type;
1198 struct btf *btf = env->btf;
1199 u32 next_type_size = 0;
1201 next_type = btf_type_by_id(btf, next_type_id);
1203 btf_verifier_log_type(env, v->t, "Invalid type_id");
1207 /* "typedef void new_void", "const void"...etc */
1208 if (btf_type_is_void(next_type))
1211 if (!env_type_is_resolve_sink(env, next_type) &&
1212 !env_type_is_resolved(env, next_type_id))
1213 return env_stack_push(env, next_type, next_type_id);
1215 /* Figure out the resolved next_type_id with size.
1216 * They will be stored in the current modifier's
1217 * resolved_ids and resolved_sizes such that it can
1218 * save us a few type-following when we use it later (e.g. in
1221 if (!btf_type_id_size(btf, &next_type_id, &next_type_size) &&
1222 !btf_type_is_void(btf_type_id_resolve(btf, &next_type_id))) {
1223 btf_verifier_log_type(env, v->t, "Invalid type_id");
1228 env_stack_pop_resolved(env, next_type_id, next_type_size);
1233 static int btf_ptr_resolve(struct btf_verifier_env *env,
1234 const struct resolve_vertex *v)
1236 const struct btf_type *next_type;
1237 const struct btf_type *t = v->t;
1238 u32 next_type_id = t->type;
1239 struct btf *btf = env->btf;
1240 u32 next_type_size = 0;
1242 next_type = btf_type_by_id(btf, next_type_id);
1244 btf_verifier_log_type(env, v->t, "Invalid type_id");
1249 if (btf_type_is_void(next_type))
1252 if (!env_type_is_resolve_sink(env, next_type) &&
1253 !env_type_is_resolved(env, next_type_id))
1254 return env_stack_push(env, next_type, next_type_id);
1256 /* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY,
1257 * the modifier may have stopped resolving when it was resolved
1258 * to a ptr (last-resolved-ptr).
1260 * We now need to continue from the last-resolved-ptr to
1261 * ensure the last-resolved-ptr will not referring back to
1262 * the currenct ptr (t).
1264 if (btf_type_is_modifier(next_type)) {
1265 const struct btf_type *resolved_type;
1266 u32 resolved_type_id;
1268 resolved_type_id = next_type_id;
1269 resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
1271 if (btf_type_is_ptr(resolved_type) &&
1272 !env_type_is_resolve_sink(env, resolved_type) &&
1273 !env_type_is_resolved(env, resolved_type_id))
1274 return env_stack_push(env, resolved_type,
1278 if (!btf_type_id_size(btf, &next_type_id, &next_type_size) &&
1279 !btf_type_is_void(btf_type_id_resolve(btf, &next_type_id))) {
1280 btf_verifier_log_type(env, v->t, "Invalid type_id");
1285 env_stack_pop_resolved(env, next_type_id, 0);
1290 static void btf_modifier_seq_show(const struct btf *btf,
1291 const struct btf_type *t,
1292 u32 type_id, void *data,
1293 u8 bits_offset, struct seq_file *m)
1295 t = btf_type_id_resolve(btf, &type_id);
1297 btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);
1300 static void btf_ptr_seq_show(const struct btf *btf, const struct btf_type *t,
1301 u32 type_id, void *data, u8 bits_offset,
1304 /* It is a hashed value */
1305 seq_printf(m, "%p", *(void **)data);
1308 static void btf_ref_type_log(struct btf_verifier_env *env,
1309 const struct btf_type *t)
1311 btf_verifier_log(env, "type_id=%u", t->type);
1314 static struct btf_kind_operations modifier_ops = {
1315 .check_meta = btf_ref_type_check_meta,
1316 .resolve = btf_modifier_resolve,
1317 .check_member = btf_modifier_check_member,
1318 .log_details = btf_ref_type_log,
1319 .seq_show = btf_modifier_seq_show,
1322 static struct btf_kind_operations ptr_ops = {
1323 .check_meta = btf_ref_type_check_meta,
1324 .resolve = btf_ptr_resolve,
1325 .check_member = btf_ptr_check_member,
1326 .log_details = btf_ref_type_log,
1327 .seq_show = btf_ptr_seq_show,
1330 static s32 btf_fwd_check_meta(struct btf_verifier_env *env,
1331 const struct btf_type *t,
1334 if (btf_type_vlen(t)) {
1335 btf_verifier_log_type(env, t, "vlen != 0");
1340 btf_verifier_log_type(env, t, "type != 0");
1344 /* fwd type must have a valid name */
1346 !btf_name_valid_identifier(env->btf, t->name_off)) {
1347 btf_verifier_log_type(env, t, "Invalid name");
1351 btf_verifier_log_type(env, t, NULL);
1356 static struct btf_kind_operations fwd_ops = {
1357 .check_meta = btf_fwd_check_meta,
1358 .resolve = btf_df_resolve,
1359 .check_member = btf_df_check_member,
1360 .log_details = btf_ref_type_log,
1361 .seq_show = btf_df_seq_show,
1364 static int btf_array_check_member(struct btf_verifier_env *env,
1365 const struct btf_type *struct_type,
1366 const struct btf_member *member,
1367 const struct btf_type *member_type)
1369 u32 struct_bits_off = member->offset;
1370 u32 struct_size, bytes_offset;
1371 u32 array_type_id, array_size;
1372 struct btf *btf = env->btf;
1374 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1375 btf_verifier_log_member(env, struct_type, member,
1376 "Member is not byte aligned");
1380 array_type_id = member->type;
1381 btf_type_id_size(btf, &array_type_id, &array_size);
1382 struct_size = struct_type->size;
1383 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1384 if (struct_size - bytes_offset < array_size) {
1385 btf_verifier_log_member(env, struct_type, member,
1386 "Member exceeds struct_size");
1393 static s32 btf_array_check_meta(struct btf_verifier_env *env,
1394 const struct btf_type *t,
1397 const struct btf_array *array = btf_type_array(t);
1398 u32 meta_needed = sizeof(*array);
1400 if (meta_left < meta_needed) {
1401 btf_verifier_log_basic(env, t,
1402 "meta_left:%u meta_needed:%u",
1403 meta_left, meta_needed);
1407 /* array type should not have a name */
1409 btf_verifier_log_type(env, t, "Invalid name");
1413 if (btf_type_vlen(t)) {
1414 btf_verifier_log_type(env, t, "vlen != 0");
1419 btf_verifier_log_type(env, t, "size != 0");
1423 /* Array elem type and index type cannot be in type void,
1424 * so !array->type and !array->index_type are not allowed.
1426 if (!array->type || !BTF_TYPE_ID_VALID(array->type)) {
1427 btf_verifier_log_type(env, t, "Invalid elem");
1431 if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) {
1432 btf_verifier_log_type(env, t, "Invalid index");
1436 btf_verifier_log_type(env, t, NULL);
1441 static int btf_array_resolve(struct btf_verifier_env *env,
1442 const struct resolve_vertex *v)
1444 const struct btf_array *array = btf_type_array(v->t);
1445 const struct btf_type *elem_type, *index_type;
1446 u32 elem_type_id, index_type_id;
1447 struct btf *btf = env->btf;
1450 /* Check array->index_type */
1451 index_type_id = array->index_type;
1452 index_type = btf_type_by_id(btf, index_type_id);
1453 if (btf_type_is_void_or_null(index_type)) {
1454 btf_verifier_log_type(env, v->t, "Invalid index");
1458 if (!env_type_is_resolve_sink(env, index_type) &&
1459 !env_type_is_resolved(env, index_type_id))
1460 return env_stack_push(env, index_type, index_type_id);
1462 index_type = btf_type_id_size(btf, &index_type_id, NULL);
1463 if (!index_type || !btf_type_is_int(index_type) ||
1464 !btf_type_int_is_regular(index_type)) {
1465 btf_verifier_log_type(env, v->t, "Invalid index");
1469 /* Check array->type */
1470 elem_type_id = array->type;
1471 elem_type = btf_type_by_id(btf, elem_type_id);
1472 if (btf_type_is_void_or_null(elem_type)) {
1473 btf_verifier_log_type(env, v->t,
1478 if (!env_type_is_resolve_sink(env, elem_type) &&
1479 !env_type_is_resolved(env, elem_type_id))
1480 return env_stack_push(env, elem_type, elem_type_id);
1482 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1484 btf_verifier_log_type(env, v->t, "Invalid elem");
1488 if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) {
1489 btf_verifier_log_type(env, v->t, "Invalid array of int");
1493 if (array->nelems && elem_size > U32_MAX / array->nelems) {
1494 btf_verifier_log_type(env, v->t,
1495 "Array size overflows U32_MAX");
1499 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);
1504 static void btf_array_log(struct btf_verifier_env *env,
1505 const struct btf_type *t)
1507 const struct btf_array *array = btf_type_array(t);
1509 btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",
1510 array->type, array->index_type, array->nelems);
1513 static void btf_array_seq_show(const struct btf *btf, const struct btf_type *t,
1514 u32 type_id, void *data, u8 bits_offset,
1517 const struct btf_array *array = btf_type_array(t);
1518 const struct btf_kind_operations *elem_ops;
1519 const struct btf_type *elem_type;
1520 u32 i, elem_size, elem_type_id;
1522 elem_type_id = array->type;
1523 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1524 elem_ops = btf_type_ops(elem_type);
1526 for (i = 0; i < array->nelems; i++) {
1530 elem_ops->seq_show(btf, elem_type, elem_type_id, data,
1537 static struct btf_kind_operations array_ops = {
1538 .check_meta = btf_array_check_meta,
1539 .resolve = btf_array_resolve,
1540 .check_member = btf_array_check_member,
1541 .log_details = btf_array_log,
1542 .seq_show = btf_array_seq_show,
1545 static int btf_struct_check_member(struct btf_verifier_env *env,
1546 const struct btf_type *struct_type,
1547 const struct btf_member *member,
1548 const struct btf_type *member_type)
1550 u32 struct_bits_off = member->offset;
1551 u32 struct_size, bytes_offset;
1553 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1554 btf_verifier_log_member(env, struct_type, member,
1555 "Member is not byte aligned");
1559 struct_size = struct_type->size;
1560 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1561 if (struct_size - bytes_offset < member_type->size) {
1562 btf_verifier_log_member(env, struct_type, member,
1563 "Member exceeds struct_size");
1570 static s32 btf_struct_check_meta(struct btf_verifier_env *env,
1571 const struct btf_type *t,
1574 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION;
1575 const struct btf_member *member;
1576 u32 meta_needed, last_offset;
1577 struct btf *btf = env->btf;
1578 u32 struct_size = t->size;
1581 meta_needed = btf_type_vlen(t) * sizeof(*member);
1582 if (meta_left < meta_needed) {
1583 btf_verifier_log_basic(env, t,
1584 "meta_left:%u meta_needed:%u",
1585 meta_left, meta_needed);
1589 /* struct type either no name or a valid one */
1591 !btf_name_valid_identifier(env->btf, t->name_off)) {
1592 btf_verifier_log_type(env, t, "Invalid name");
1596 btf_verifier_log_type(env, t, NULL);
1599 for_each_member(i, t, member) {
1600 if (!btf_name_offset_valid(btf, member->name_off)) {
1601 btf_verifier_log_member(env, t, member,
1602 "Invalid member name_offset:%u",
1607 /* struct member either no name or a valid one */
1608 if (member->name_off &&
1609 !btf_name_valid_identifier(btf, member->name_off)) {
1610 btf_verifier_log_member(env, t, member, "Invalid name");
1613 /* A member cannot be in type void */
1614 if (!member->type || !BTF_TYPE_ID_VALID(member->type)) {
1615 btf_verifier_log_member(env, t, member,
1620 if (is_union && member->offset) {
1621 btf_verifier_log_member(env, t, member,
1622 "Invalid member bits_offset");
1627 * ">" instead of ">=" because the last member could be
1630 if (last_offset > member->offset) {
1631 btf_verifier_log_member(env, t, member,
1632 "Invalid member bits_offset");
1636 if (BITS_ROUNDUP_BYTES(member->offset) > struct_size) {
1637 btf_verifier_log_member(env, t, member,
1638 "Memmber bits_offset exceeds its struct size");
1642 btf_verifier_log_member(env, t, member, NULL);
1643 last_offset = member->offset;
1649 static int btf_struct_resolve(struct btf_verifier_env *env,
1650 const struct resolve_vertex *v)
1652 const struct btf_member *member;
1656 /* Before continue resolving the next_member,
1657 * ensure the last member is indeed resolved to a
1658 * type with size info.
1660 if (v->next_member) {
1661 const struct btf_type *last_member_type;
1662 const struct btf_member *last_member;
1663 u16 last_member_type_id;
1665 last_member = btf_type_member(v->t) + v->next_member - 1;
1666 last_member_type_id = last_member->type;
1667 if (WARN_ON_ONCE(!env_type_is_resolved(env,
1668 last_member_type_id)))
1671 last_member_type = btf_type_by_id(env->btf,
1672 last_member_type_id);
1673 err = btf_type_ops(last_member_type)->check_member(env, v->t,
1680 for_each_member_from(i, v->next_member, v->t, member) {
1681 u32 member_type_id = member->type;
1682 const struct btf_type *member_type = btf_type_by_id(env->btf,
1685 if (btf_type_is_void_or_null(member_type)) {
1686 btf_verifier_log_member(env, v->t, member,
1691 if (!env_type_is_resolve_sink(env, member_type) &&
1692 !env_type_is_resolved(env, member_type_id)) {
1693 env_stack_set_next_member(env, i + 1);
1694 return env_stack_push(env, member_type, member_type_id);
1697 err = btf_type_ops(member_type)->check_member(env, v->t,
1704 env_stack_pop_resolved(env, 0, 0);
1709 static void btf_struct_log(struct btf_verifier_env *env,
1710 const struct btf_type *t)
1712 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
1715 static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t,
1716 u32 type_id, void *data, u8 bits_offset,
1719 const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "|" : ",";
1720 const struct btf_member *member;
1724 for_each_member(i, t, member) {
1725 const struct btf_type *member_type = btf_type_by_id(btf,
1727 u32 member_offset = member->offset;
1728 u32 bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
1729 u8 bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
1730 const struct btf_kind_operations *ops;
1735 ops = btf_type_ops(member_type);
1736 ops->seq_show(btf, member_type, member->type,
1737 data + bytes_offset, bits8_offset, m);
1742 static struct btf_kind_operations struct_ops = {
1743 .check_meta = btf_struct_check_meta,
1744 .resolve = btf_struct_resolve,
1745 .check_member = btf_struct_check_member,
1746 .log_details = btf_struct_log,
1747 .seq_show = btf_struct_seq_show,
1750 static int btf_enum_check_member(struct btf_verifier_env *env,
1751 const struct btf_type *struct_type,
1752 const struct btf_member *member,
1753 const struct btf_type *member_type)
1755 u32 struct_bits_off = member->offset;
1756 u32 struct_size, bytes_offset;
1758 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1759 btf_verifier_log_member(env, struct_type, member,
1760 "Member is not byte aligned");
1764 struct_size = struct_type->size;
1765 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1766 if (struct_size - bytes_offset < member_type->size) {
1767 btf_verifier_log_member(env, struct_type, member,
1768 "Member exceeds struct_size");
1775 static s32 btf_enum_check_meta(struct btf_verifier_env *env,
1776 const struct btf_type *t,
1779 const struct btf_enum *enums = btf_type_enum(t);
1780 struct btf *btf = env->btf;
1784 nr_enums = btf_type_vlen(t);
1785 meta_needed = nr_enums * sizeof(*enums);
1787 if (meta_left < meta_needed) {
1788 btf_verifier_log_basic(env, t,
1789 "meta_left:%u meta_needed:%u",
1790 meta_left, meta_needed);
1794 if (t->size != sizeof(int)) {
1795 btf_verifier_log_type(env, t, "Expected size:%zu",
1800 /* enum type either no name or a valid one */
1802 !btf_name_valid_identifier(env->btf, t->name_off)) {
1803 btf_verifier_log_type(env, t, "Invalid name");
1807 btf_verifier_log_type(env, t, NULL);
1809 for (i = 0; i < nr_enums; i++) {
1810 if (!btf_name_offset_valid(btf, enums[i].name_off)) {
1811 btf_verifier_log(env, "\tInvalid name_offset:%u",
1816 /* enum member must have a valid name */
1817 if (!enums[i].name_off ||
1818 !btf_name_valid_identifier(btf, enums[i].name_off)) {
1819 btf_verifier_log_type(env, t, "Invalid name");
1824 btf_verifier_log(env, "\t%s val=%d\n",
1825 btf_name_by_offset(btf, enums[i].name_off),
1832 static void btf_enum_log(struct btf_verifier_env *env,
1833 const struct btf_type *t)
1835 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
1838 static void btf_enum_seq_show(const struct btf *btf, const struct btf_type *t,
1839 u32 type_id, void *data, u8 bits_offset,
1842 const struct btf_enum *enums = btf_type_enum(t);
1843 u32 i, nr_enums = btf_type_vlen(t);
1844 int v = *(int *)data;
1846 for (i = 0; i < nr_enums; i++) {
1847 if (v == enums[i].val) {
1849 btf_name_by_offset(btf, enums[i].name_off));
1854 seq_printf(m, "%d", v);
1857 static struct btf_kind_operations enum_ops = {
1858 .check_meta = btf_enum_check_meta,
1859 .resolve = btf_df_resolve,
1860 .check_member = btf_enum_check_member,
1861 .log_details = btf_enum_log,
1862 .seq_show = btf_enum_seq_show,
1865 static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
1866 [BTF_KIND_INT] = &int_ops,
1867 [BTF_KIND_PTR] = &ptr_ops,
1868 [BTF_KIND_ARRAY] = &array_ops,
1869 [BTF_KIND_STRUCT] = &struct_ops,
1870 [BTF_KIND_UNION] = &struct_ops,
1871 [BTF_KIND_ENUM] = &enum_ops,
1872 [BTF_KIND_FWD] = &fwd_ops,
1873 [BTF_KIND_TYPEDEF] = &modifier_ops,
1874 [BTF_KIND_VOLATILE] = &modifier_ops,
1875 [BTF_KIND_CONST] = &modifier_ops,
1876 [BTF_KIND_RESTRICT] = &modifier_ops,
1879 static s32 btf_check_meta(struct btf_verifier_env *env,
1880 const struct btf_type *t,
1883 u32 saved_meta_left = meta_left;
1886 if (meta_left < sizeof(*t)) {
1887 btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",
1888 env->log_type_id, meta_left, sizeof(*t));
1891 meta_left -= sizeof(*t);
1893 if (t->info & ~BTF_INFO_MASK) {
1894 btf_verifier_log(env, "[%u] Invalid btf_info:%x",
1895 env->log_type_id, t->info);
1899 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX ||
1900 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
1901 btf_verifier_log(env, "[%u] Invalid kind:%u",
1902 env->log_type_id, BTF_INFO_KIND(t->info));
1906 if (!btf_name_offset_valid(env->btf, t->name_off)) {
1907 btf_verifier_log(env, "[%u] Invalid name_offset:%u",
1908 env->log_type_id, t->name_off);
1912 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);
1913 if (var_meta_size < 0)
1914 return var_meta_size;
1916 meta_left -= var_meta_size;
1918 return saved_meta_left - meta_left;
1921 static int btf_check_all_metas(struct btf_verifier_env *env)
1923 struct btf *btf = env->btf;
1924 struct btf_header *hdr;
1928 cur = btf->nohdr_data + hdr->type_off;
1929 end = cur + hdr->type_len;
1931 env->log_type_id = 1;
1933 struct btf_type *t = cur;
1936 meta_size = btf_check_meta(env, t, end - cur);
1940 btf_add_type(env, t);
1948 static int btf_resolve(struct btf_verifier_env *env,
1949 const struct btf_type *t, u32 type_id)
1951 const struct resolve_vertex *v;
1954 env->resolve_mode = RESOLVE_TBD;
1955 env_stack_push(env, t, type_id);
1956 while (!err && (v = env_stack_peak(env))) {
1957 env->log_type_id = v->type_id;
1958 err = btf_type_ops(v->t)->resolve(env, v);
1961 env->log_type_id = type_id;
1963 btf_verifier_log_type(env, t,
1964 "Exceeded max resolving depth:%u",
1966 else if (err == -EEXIST)
1967 btf_verifier_log_type(env, t, "Loop detected");
1972 static bool btf_resolve_valid(struct btf_verifier_env *env,
1973 const struct btf_type *t,
1976 struct btf *btf = env->btf;
1978 if (!env_type_is_resolved(env, type_id))
1981 if (btf_type_is_struct(t))
1982 return !btf->resolved_ids[type_id] &&
1983 !btf->resolved_sizes[type_id];
1985 if (btf_type_is_modifier(t) || btf_type_is_ptr(t)) {
1986 t = btf_type_id_resolve(btf, &type_id);
1987 return t && !btf_type_is_modifier(t);
1990 if (btf_type_is_array(t)) {
1991 const struct btf_array *array = btf_type_array(t);
1992 const struct btf_type *elem_type;
1993 u32 elem_type_id = array->type;
1996 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1997 return elem_type && !btf_type_is_modifier(elem_type) &&
1998 (array->nelems * elem_size ==
1999 btf->resolved_sizes[type_id]);
2005 static int btf_check_all_types(struct btf_verifier_env *env)
2007 struct btf *btf = env->btf;
2011 err = env_resolve_init(env);
2016 for (type_id = 1; type_id <= btf->nr_types; type_id++) {
2017 const struct btf_type *t = btf_type_by_id(btf, type_id);
2019 env->log_type_id = type_id;
2020 if (btf_type_needs_resolve(t) &&
2021 !env_type_is_resolved(env, type_id)) {
2022 err = btf_resolve(env, t, type_id);
2027 if (btf_type_needs_resolve(t) &&
2028 !btf_resolve_valid(env, t, type_id)) {
2029 btf_verifier_log_type(env, t, "Invalid resolve state");
2037 static int btf_parse_type_sec(struct btf_verifier_env *env)
2039 const struct btf_header *hdr = &env->btf->hdr;
2042 /* Type section must align to 4 bytes */
2043 if (hdr->type_off & (sizeof(u32) - 1)) {
2044 btf_verifier_log(env, "Unaligned type_off");
2048 if (!hdr->type_len) {
2049 btf_verifier_log(env, "No type found");
2053 err = btf_check_all_metas(env);
2057 return btf_check_all_types(env);
2060 static int btf_parse_str_sec(struct btf_verifier_env *env)
2062 const struct btf_header *hdr;
2063 struct btf *btf = env->btf;
2064 const char *start, *end;
2067 start = btf->nohdr_data + hdr->str_off;
2068 end = start + hdr->str_len;
2070 if (end != btf->data + btf->data_size) {
2071 btf_verifier_log(env, "String section is not at the end");
2075 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
2076 start[0] || end[-1]) {
2077 btf_verifier_log(env, "Invalid string section");
2081 btf->strings = start;
2086 static const size_t btf_sec_info_offset[] = {
2087 offsetof(struct btf_header, type_off),
2088 offsetof(struct btf_header, str_off),
2091 static int btf_sec_info_cmp(const void *a, const void *b)
2093 const struct btf_sec_info *x = a;
2094 const struct btf_sec_info *y = b;
2096 return (int)(x->off - y->off) ? : (int)(x->len - y->len);
2099 static int btf_check_sec_info(struct btf_verifier_env *env,
2102 struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)];
2103 u32 total, expected_total, i;
2104 const struct btf_header *hdr;
2105 const struct btf *btf;
2110 /* Populate the secs from hdr */
2111 for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++)
2112 secs[i] = *(struct btf_sec_info *)((void *)hdr +
2113 btf_sec_info_offset[i]);
2115 sort(secs, ARRAY_SIZE(btf_sec_info_offset),
2116 sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL);
2118 /* Check for gaps and overlap among sections */
2120 expected_total = btf_data_size - hdr->hdr_len;
2121 for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) {
2122 if (expected_total < secs[i].off) {
2123 btf_verifier_log(env, "Invalid section offset");
2126 if (total < secs[i].off) {
2128 btf_verifier_log(env, "Unsupported section found");
2131 if (total > secs[i].off) {
2132 btf_verifier_log(env, "Section overlap found");
2135 if (expected_total - total < secs[i].len) {
2136 btf_verifier_log(env,
2137 "Total section length too long");
2140 total += secs[i].len;
2143 /* There is data other than hdr and known sections */
2144 if (expected_total != total) {
2145 btf_verifier_log(env, "Unsupported section found");
2152 static int btf_parse_hdr(struct btf_verifier_env *env)
2154 u32 hdr_len, hdr_copy, btf_data_size;
2155 const struct btf_header *hdr;
2160 btf_data_size = btf->data_size;
2163 offsetof(struct btf_header, hdr_len) + sizeof(hdr->hdr_len)) {
2164 btf_verifier_log(env, "hdr_len not found");
2169 hdr_len = hdr->hdr_len;
2170 if (btf_data_size < hdr_len) {
2171 btf_verifier_log(env, "btf_header not found");
2175 /* Ensure the unsupported header fields are zero */
2176 if (hdr_len > sizeof(btf->hdr)) {
2177 u8 *expected_zero = btf->data + sizeof(btf->hdr);
2178 u8 *end = btf->data + hdr_len;
2180 for (; expected_zero < end; expected_zero++) {
2181 if (*expected_zero) {
2182 btf_verifier_log(env, "Unsupported btf_header");
2188 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr));
2189 memcpy(&btf->hdr, btf->data, hdr_copy);
2193 if (hdr->hdr_len != hdr_len)
2196 btf_verifier_log_hdr(env, btf_data_size);
2198 if (hdr->magic != BTF_MAGIC) {
2199 btf_verifier_log(env, "Invalid magic");
2203 if (hdr->version != BTF_VERSION) {
2204 btf_verifier_log(env, "Unsupported version");
2209 btf_verifier_log(env, "Unsupported flags");
2213 if (btf_data_size == hdr->hdr_len) {
2214 btf_verifier_log(env, "No data");
2218 err = btf_check_sec_info(env, btf_data_size);
2225 static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size,
2226 u32 log_level, char __user *log_ubuf, u32 log_size)
2228 struct btf_verifier_env *env = NULL;
2229 struct bpf_verifier_log *log;
2230 struct btf *btf = NULL;
2234 if (btf_data_size > BTF_MAX_SIZE)
2235 return ERR_PTR(-E2BIG);
2237 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
2239 return ERR_PTR(-ENOMEM);
2242 if (log_level || log_ubuf || log_size) {
2243 /* user requested verbose verifier output
2244 * and supplied buffer to store the verification trace
2246 log->level = log_level;
2247 log->ubuf = log_ubuf;
2248 log->len_total = log_size;
2250 /* log attributes have to be sane */
2251 if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
2252 !log->level || !log->ubuf) {
2258 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
2265 data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
2272 btf->data_size = btf_data_size;
2274 if (copy_from_user(data, btf_data, btf_data_size)) {
2279 err = btf_parse_hdr(env);
2283 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
2285 err = btf_parse_str_sec(env);
2289 err = btf_parse_type_sec(env);
2293 if (log->level && bpf_verifier_log_full(log)) {
2298 btf_verifier_env_free(env);
2299 refcount_set(&btf->refcnt, 1);
2303 btf_verifier_env_free(env);
2306 return ERR_PTR(err);
2309 void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
2312 const struct btf_type *t = btf_type_by_id(btf, type_id);
2314 btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m);
2317 static int btf_release(struct inode *inode, struct file *filp)
2319 btf_put(filp->private_data);
2323 const struct file_operations btf_fops = {
2324 .release = btf_release,
2327 static int __btf_new_fd(struct btf *btf)
2329 return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC);
2332 int btf_new_fd(const union bpf_attr *attr)
2337 btf = btf_parse(u64_to_user_ptr(attr->btf),
2338 attr->btf_size, attr->btf_log_level,
2339 u64_to_user_ptr(attr->btf_log_buf),
2340 attr->btf_log_size);
2342 return PTR_ERR(btf);
2344 ret = btf_alloc_id(btf);
2351 * The BTF ID is published to the userspace.
2352 * All BTF free must go through call_rcu() from
2353 * now on (i.e. free by calling btf_put()).
2356 ret = __btf_new_fd(btf);
2363 struct btf *btf_get_by_fd(int fd)
2371 return ERR_PTR(-EBADF);
2373 if (f.file->f_op != &btf_fops) {
2375 return ERR_PTR(-EINVAL);
2378 btf = f.file->private_data;
2379 refcount_inc(&btf->refcnt);
2385 int btf_get_info_by_fd(const struct btf *btf,
2386 const union bpf_attr *attr,
2387 union bpf_attr __user *uattr)
2389 struct bpf_btf_info __user *uinfo;
2390 struct bpf_btf_info info;
2391 u32 info_copy, btf_copy;
2395 uinfo = u64_to_user_ptr(attr->info.info);
2396 uinfo_len = attr->info.info_len;
2398 info_copy = min_t(u32, uinfo_len, sizeof(info));
2399 memset(&info, 0, sizeof(info));
2400 if (copy_from_user(&info, uinfo, info_copy))
2404 ubtf = u64_to_user_ptr(info.btf);
2405 btf_copy = min_t(u32, btf->data_size, info.btf_size);
2406 if (copy_to_user(ubtf, btf->data, btf_copy))
2408 info.btf_size = btf->data_size;
2410 if (copy_to_user(uinfo, &info, info_copy) ||
2411 put_user(info_copy, &uattr->info.info_len))
2417 int btf_get_fd_by_id(u32 id)
2423 btf = idr_find(&btf_idr, id);
2424 if (!btf || !refcount_inc_not_zero(&btf->refcnt))
2425 btf = ERR_PTR(-ENOENT);
2429 return PTR_ERR(btf);
2431 fd = __btf_new_fd(btf);
2438 u32 btf_id(const struct btf *btf)