1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2019 Intel Corporation. */
4 #include <linux/hash.h>
6 #include <linux/filter.h>
8 /* The BPF dispatcher is a multiway branch code generator. The
9 * dispatcher is a mechanism to avoid the performance penalty of an
10 * indirect call, which is expensive when retpolines are enabled. A
11 * dispatch client registers a BPF program into the dispatcher, and if
12 * there is available room in the dispatcher a direct call to the BPF
13 * program will be generated. All calls to the BPF programs called via
14 * the dispatcher will then be a direct call, instead of an
15 * indirect. The dispatcher hijacks a trampoline function it via the
16 * __fentry__ of the trampoline. The trampoline function has the
17 * following signature:
19 * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
20 * unsigned int (*bpf_func)(const void *,
21 * const struct bpf_insn *));
24 static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
25 struct bpf_dispatcher *d, struct bpf_prog *prog)
29 for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
30 if (prog == d->progs[i].prog)
36 static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
37 struct bpf_dispatcher *d)
39 return bpf_dispatcher_find_prog(d, NULL);
42 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
43 struct bpf_prog *prog)
45 struct bpf_dispatcher_prog *entry;
50 entry = bpf_dispatcher_find_prog(d, prog);
52 refcount_inc(&entry->users);
56 entry = bpf_dispatcher_find_free(d);
62 refcount_set(&entry->users, 1);
67 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
68 struct bpf_prog *prog)
70 struct bpf_dispatcher_prog *entry;
75 entry = bpf_dispatcher_find_prog(d, prog);
79 if (refcount_dec_and_test(&entry->users)) {
88 int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
93 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)
95 s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
98 for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
100 *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
102 return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs);
105 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
111 if (!prev_num_progs) {
115 old = d->image + d->image_off;
116 noff = d->image_off ^ (PAGE_SIZE / 2);
119 new = d->num_progs ? d->image + noff : NULL;
121 if (bpf_dispatcher_prepare(d, new))
125 err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new);
132 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
135 bool changed = false;
141 mutex_lock(&d->mutex);
143 d->image = bpf_jit_alloc_exec_page();
146 bpf_image_ksym_add(d->image, &d->ksym);
149 prev_num_progs = d->num_progs;
150 changed |= bpf_dispatcher_remove_prog(d, from);
151 changed |= bpf_dispatcher_add_prog(d, to);
156 bpf_dispatcher_update(d, prev_num_progs);
158 mutex_unlock(&d->mutex);