1 // SPDX-License-Identifier: GPL-2.0
3 * Common functionality for RV32 and RV64 BPF JIT compilers
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
10 #include <linux/filter.h>
11 #include <linux/memory.h>
12 #include <asm/patch.h>
15 /* Number of iterations to try until offsets converge. */
16 #define NR_JIT_ITERATIONS 32
18 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
20 const struct bpf_prog *prog = ctx->prog;
23 for (i = 0; i < prog->len; i++) {
24 const struct bpf_insn *insn = &prog->insnsi[i];
27 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
28 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
32 offset[i] = ctx->ninsns;
39 bool bpf_jit_needs_zext(void)
44 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
46 unsigned int prog_size = 0, extable_size = 0;
47 bool tmp_blinded = false, extra_pass = false;
48 struct bpf_prog *tmp, *orig_prog = prog;
49 int pass = 0, prev_ninsns = 0, i;
50 struct rv_jit_data *jit_data;
51 struct rv_jit_context *ctx;
53 if (!prog->jit_requested)
56 tmp = bpf_jit_blind_constants(prog);
64 jit_data = prog->aux->jit_data;
66 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
71 prog->aux->jit_data = jit_data;
78 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
83 ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
89 if (build_body(ctx, extra_pass, NULL)) {
94 for (i = 0; i < prog->len; i++) {
96 ctx->offset[i] = prev_ninsns;
99 for (i = 0; i < NR_JIT_ITERATIONS; i++) {
103 bpf_jit_build_prologue(ctx);
104 ctx->prologue_len = ctx->ninsns;
106 if (build_body(ctx, extra_pass, ctx->offset)) {
111 ctx->epilogue_offset = ctx->ninsns;
112 bpf_jit_build_epilogue(ctx);
114 if (ctx->ninsns == prev_ninsns) {
115 if (jit_data->header)
117 /* obtain the actual image size */
118 extable_size = prog->aux->num_exentries *
119 sizeof(struct exception_table_entry);
120 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
122 jit_data->ro_header =
123 bpf_jit_binary_pack_alloc(prog_size + extable_size,
124 &jit_data->ro_image, sizeof(u32),
125 &jit_data->header, &jit_data->image,
127 if (!jit_data->ro_header) {
133 * Use the image(RW) for writing the JITed instructions. But also save
134 * the ro_image(RX) for calculating the offsets in the image. The RW
135 * image will be later copied to the RX image from where the program
136 * will run. The bpf_jit_binary_pack_finalize() will do this copy in the
139 ctx->ro_insns = (u16 *)jit_data->ro_image;
140 ctx->insns = (u16 *)jit_data->image;
142 * Now, when the image is allocated, the image can
143 * potentially shrink more (auipc/jalr -> jal).
146 prev_ninsns = ctx->ninsns;
149 if (i == NR_JIT_ITERATIONS) {
150 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
156 prog->aux->extable = (void *)ctx->ro_insns + prog_size;
163 bpf_jit_build_prologue(ctx);
164 if (build_body(ctx, extra_pass, NULL)) {
168 bpf_jit_build_epilogue(ctx);
170 if (bpf_jit_enable > 1)
171 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
173 prog->bpf_func = (void *)ctx->ro_insns;
175 prog->jited_len = prog_size;
177 if (!prog->is_func || extra_pass) {
178 if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,
179 jit_data->header))) {
180 /* ro_header has been freed */
181 jit_data->ro_header = NULL;
186 * The instructions have now been copied to the ROX region from
187 * where they will execute.
188 * Write any modified data cache blocks out to memory and
189 * invalidate the corresponding blocks in the instruction cache.
191 bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
192 for (i = 0; i < prog->len; i++)
193 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
194 bpf_prog_fill_jited_linfo(prog, ctx->offset);
198 prog->aux->jit_data = NULL;
203 bpf_jit_prog_release_other(prog, prog == orig_prog ?
208 if (jit_data->header) {
209 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
210 sizeof(jit_data->header->size));
211 bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header);
216 u64 bpf_jit_alloc_exec_limit(void)
218 return BPF_JIT_REGION_SIZE;
221 void *bpf_jit_alloc_exec(unsigned long size)
223 return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
224 BPF_JIT_REGION_END, GFP_KERNEL,
225 PAGE_KERNEL, 0, NUMA_NO_NODE,
226 __builtin_return_address(0));
229 void bpf_jit_free_exec(void *addr)
234 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
238 mutex_lock(&text_mutex);
239 ret = patch_text_nosync(dst, src, len);
240 mutex_unlock(&text_mutex);
243 return ERR_PTR(-EINVAL);
248 int bpf_arch_text_invalidate(void *dst, size_t len)
252 mutex_lock(&text_mutex);
253 ret = patch_text_set_nosync(dst, 0, len);
254 mutex_unlock(&text_mutex);
259 void bpf_jit_free(struct bpf_prog *prog)
262 struct rv_jit_data *jit_data = prog->aux->jit_data;
263 struct bpf_binary_header *hdr;
266 * If we fail the final pass of JIT (from jit_subprogs),
267 * the program may not be finalized yet. Call finalize here
271 bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header);
274 hdr = bpf_jit_binary_pack_hdr(prog);
275 bpf_jit_binary_pack_free(hdr, NULL);
276 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
279 bpf_prog_unlock_free(prog);