arch/riscv/net/bpf_jit_core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common functionality for RV32 and RV64 BPF JIT compilers
   4  *
   5  * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
   6  *
   7  */
   8
   9 #include <linux/bpf.h>
  10 #include <linux/filter.h>
  11 #include <linux/memory.h>
  12 #include <asm/patch.h>
  13 #include "bpf_jit.h"
  14
  15 /* Number of iterations to try until offsets converge. */
  16 #define NR_JIT_ITERATIONS       32
  17
  18 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
  19 {
  20         const struct bpf_prog *prog = ctx->prog;
  21         int i;
  22
  23         for (i = 0; i < prog->len; i++) {
  24                 const struct bpf_insn *insn = &prog->insnsi[i];
  25                 int ret;
  26
  27                 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
  28                 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
  29                 if (ret > 0)
  30                         i++;
  31                 if (offset)
  32                         offset[i] = ctx->ninsns;
  33                 if (ret < 0)
  34                         return ret;
  35         }
  36         return 0;
  37 }
  38
  39 bool bpf_jit_needs_zext(void)
  40 {
  41         return true;
  42 }
  43
  44 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
  45 {
  46         unsigned int prog_size = 0, extable_size = 0;
  47         bool tmp_blinded = false, extra_pass = false;
  48         struct bpf_prog *tmp, *orig_prog = prog;
  49         int pass = 0, prev_ninsns = 0, i;
  50         struct rv_jit_data *jit_data;
  51         struct rv_jit_context *ctx;
  52
  53         if (!prog->jit_requested)
  54                 return orig_prog;
  55
  56         tmp = bpf_jit_blind_constants(prog);
  57         if (IS_ERR(tmp))
  58                 return orig_prog;
  59         if (tmp != prog) {
  60                 tmp_blinded = true;
  61                 prog = tmp;
  62         }
  63
  64         jit_data = prog->aux->jit_data;
  65         if (!jit_data) {
  66                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
  67                 if (!jit_data) {
  68                         prog = orig_prog;
  69                         goto out;
  70                 }
  71                 prog->aux->jit_data = jit_data;
  72         }
  73
  74         ctx = &jit_data->ctx;
  75
  76         if (ctx->offset) {
  77                 extra_pass = true;
  78                 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
  79                 goto skip_init_ctx;
  80         }
  81
  82         ctx->prog = prog;
  83         ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
  84         if (!ctx->offset) {
  85                 prog = orig_prog;
  86                 goto out_offset;
  87         }
  88
  89         if (build_body(ctx, extra_pass, NULL)) {
  90                 prog = orig_prog;
  91                 goto out_offset;
  92         }
  93
  94         for (i = 0; i < prog->len; i++) {
  95                 prev_ninsns += 32;
  96                 ctx->offset[i] = prev_ninsns;
  97         }
  98
  99         for (i = 0; i < NR_JIT_ITERATIONS; i++) {
 100                 pass++;
 101                 ctx->ninsns = 0;
 102
 103                 bpf_jit_build_prologue(ctx);
 104                 ctx->prologue_len = ctx->ninsns;
 105
 106                 if (build_body(ctx, extra_pass, ctx->offset)) {
 107                         prog = orig_prog;
 108                         goto out_offset;
 109                 }
 110
 111                 ctx->epilogue_offset = ctx->ninsns;
 112                 bpf_jit_build_epilogue(ctx);
 113
 114                 if (ctx->ninsns == prev_ninsns) {
 115                         if (jit_data->header)
 116                                 break;
 117                         /* obtain the actual image size */
 118                         extable_size = prog->aux->num_exentries *
 119                                 sizeof(struct exception_table_entry);
 120                         prog_size = sizeof(*ctx->insns) * ctx->ninsns;
 121
 122                         jit_data->ro_header =
 123                                 bpf_jit_binary_pack_alloc(prog_size + extable_size,
 124                                                           &jit_data->ro_image, sizeof(u32),
 125                                                           &jit_data->header, &jit_data->image,
 126                                                           bpf_fill_ill_insns);
 127                         if (!jit_data->ro_header) {
 128                                 prog = orig_prog;
 129                                 goto out_offset;
 130                         }
 131
 132                         /*
 133                          * Use the image(RW) for writing the JITed instructions. But also save
 134                          * the ro_image(RX) for calculating the offsets in the image. The RW
 135                          * image will be later copied to the RX image from where the program
 136                          * will run. The bpf_jit_binary_pack_finalize() will do this copy in the
 137                          * final step.
 138                          */
 139                         ctx->ro_insns = (u16 *)jit_data->ro_image;
 140                         ctx->insns = (u16 *)jit_data->image;
 141                         /*
 142                          * Now, when the image is allocated, the image can
 143                          * potentially shrink more (auipc/jalr -> jal).
 144                          */
 145                 }
 146                 prev_ninsns = ctx->ninsns;
 147         }
 148
 149         if (i == NR_JIT_ITERATIONS) {
 150                 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
 151                 prog = orig_prog;
 152                 goto out_free_hdr;
 153         }
 154
 155         if (extable_size)
 156                 prog->aux->extable = (void *)ctx->ro_insns + prog_size;
 157
 158 skip_init_ctx:
 159         pass++;
 160         ctx->ninsns = 0;
 161         ctx->nexentries = 0;
 162
 163         bpf_jit_build_prologue(ctx);
 164         if (build_body(ctx, extra_pass, NULL)) {
 165                 prog = orig_prog;
 166                 goto out_free_hdr;
 167         }
 168         bpf_jit_build_epilogue(ctx);
 169
 170         if (bpf_jit_enable > 1)
 171                 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
 172
 173         prog->bpf_func = (void *)ctx->ro_insns;
 174         prog->jited = 1;
 175         prog->jited_len = prog_size;
 176
 177         if (!prog->is_func || extra_pass) {
 178                 if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,
 179                                                          jit_data->header))) {
 180                         /* ro_header has been freed */
 181                         jit_data->ro_header = NULL;
 182                         prog = orig_prog;
 183                         goto out_offset;
 184                 }
 185                 /*
 186                  * The instructions have now been copied to the ROX region from
 187                  * where they will execute.
 188                  * Write any modified data cache blocks out to memory and
 189                  * invalidate the corresponding blocks in the instruction cache.
 190                  */
 191                 bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
 192                 for (i = 0; i < prog->len; i++)
 193                         ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
 194                 bpf_prog_fill_jited_linfo(prog, ctx->offset);
 195 out_offset:
 196                 kfree(ctx->offset);
 197                 kfree(jit_data);
 198                 prog->aux->jit_data = NULL;
 199         }
 200 out:
 201
 202         if (tmp_blinded)
 203                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
 204                                            tmp : orig_prog);
 205         return prog;
 206
 207 out_free_hdr:
 208         if (jit_data->header) {
 209                 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
 210                                    sizeof(jit_data->header->size));
 211                 bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header);
 212         }
 213         goto out_offset;
 214 }
 215
 216 u64 bpf_jit_alloc_exec_limit(void)
 217 {
 218         return BPF_JIT_REGION_SIZE;
 219 }
 220
 221 void *bpf_jit_alloc_exec(unsigned long size)
 222 {
 223         return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
 224                                     BPF_JIT_REGION_END, GFP_KERNEL,
 225                                     PAGE_KERNEL, 0, NUMA_NO_NODE,
 226                                     __builtin_return_address(0));
 227 }
 228
 229 void bpf_jit_free_exec(void *addr)
 230 {
 231         return vfree(addr);
 232 }
 233
 234 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
 235 {
 236         int ret;
 237
 238         mutex_lock(&text_mutex);
 239         ret = patch_text_nosync(dst, src, len);
 240         mutex_unlock(&text_mutex);
 241
 242         if (ret)
 243                 return ERR_PTR(-EINVAL);
 244
 245         return dst;
 246 }
 247
 248 int bpf_arch_text_invalidate(void *dst, size_t len)
 249 {
 250         int ret;
 251
 252         mutex_lock(&text_mutex);
 253         ret = patch_text_set_nosync(dst, 0, len);
 254         mutex_unlock(&text_mutex);
 255
 256         return ret;
 257 }
 258
 259 void bpf_jit_free(struct bpf_prog *prog)
 260 {
 261         if (prog->jited) {
 262                 struct rv_jit_data *jit_data = prog->aux->jit_data;
 263                 struct bpf_binary_header *hdr;
 264
 265                 /*
 266                  * If we fail the final pass of JIT (from jit_subprogs),
 267                  * the program may not be finalized yet. Call finalize here
 268                  * before freeing it.
 269                  */
 270                 if (jit_data) {
 271                         bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header);
 272                         kfree(jit_data);
 273                 }
 274                 hdr = bpf_jit_binary_pack_hdr(prog);
 275                 bpf_jit_binary_pack_free(hdr, NULL);
 276                 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
 277         }
 278
 279         bpf_prog_unlock_free(prog);
 280 }