arch/x86/net/bpf_jit_comp.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * bpf_jit_comp.c: BPF JIT compiler
   4  *
   5  * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
   6  * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   7  */
   8 #include <linux/netdevice.h>
   9 #include <linux/filter.h>
  10 #include <linux/if_vlan.h>
  11 #include <linux/bpf.h>
  12
  13 #include <asm/set_memory.h>
  14 #include <asm/nospec-branch.h>
  15
  16 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
  17 {
  18         if (len == 1)
  19                 *ptr = bytes;
  20         else if (len == 2)
  21                 *(u16 *)ptr = bytes;
  22         else {
  23                 *(u32 *)ptr = bytes;
  24                 barrier();
  25         }
  26         return ptr + len;
  27 }
  28
  29 #define EMIT(bytes, len) \
  30         do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
  31
  32 #define EMIT1(b1)               EMIT(b1, 1)
  33 #define EMIT2(b1, b2)           EMIT((b1) + ((b2) << 8), 2)
  34 #define EMIT3(b1, b2, b3)       EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
  35 #define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
  36
  37 #define EMIT1_off32(b1, off) \
  38         do { EMIT1(b1); EMIT(off, 4); } while (0)
  39 #define EMIT2_off32(b1, b2, off) \
  40         do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
  41 #define EMIT3_off32(b1, b2, b3, off) \
  42         do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
  43 #define EMIT4_off32(b1, b2, b3, b4, off) \
  44         do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
  45
  46 static bool is_imm8(int value)
  47 {
  48         return value <= 127 && value >= -128;
  49 }
  50
  51 static bool is_simm32(s64 value)
  52 {
  53         return value == (s64)(s32)value;
  54 }
  55
  56 static bool is_uimm32(u64 value)
  57 {
  58         return value == (u64)(u32)value;
  59 }
  60
  61 /* mov dst, src */
  62 #define EMIT_mov(DST, SRC)                                                               \
  63         do {                                                                             \
  64                 if (DST != SRC)                                                          \
  65                         EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
  66         } while (0)
  67
  68 static int bpf_size_to_x86_bytes(int bpf_size)
  69 {
  70         if (bpf_size == BPF_W)
  71                 return 4;
  72         else if (bpf_size == BPF_H)
  73                 return 2;
  74         else if (bpf_size == BPF_B)
  75                 return 1;
  76         else if (bpf_size == BPF_DW)
  77                 return 4; /* imm32 */
  78         else
  79                 return 0;
  80 }
  81
  82 /*
  83  * List of x86 cond jumps opcodes (. + s8)
  84  * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
  85  */
  86 #define X86_JB  0x72
  87 #define X86_JAE 0x73
  88 #define X86_JE  0x74
  89 #define X86_JNE 0x75
  90 #define X86_JBE 0x76
  91 #define X86_JA  0x77
  92 #define X86_JL  0x7C
  93 #define X86_JGE 0x7D
  94 #define X86_JLE 0x7E
  95 #define X86_JG  0x7F
  96
  97 /* Pick a register outside of BPF range for JIT internal work */
  98 #define AUX_REG (MAX_BPF_JIT_REG + 1)
  99
 100 /*
 101  * The following table maps BPF registers to x86-64 registers.
 102  *
 103  * x86-64 register R12 is unused, since if used as base address
 104  * register in load/store instructions, it always needs an
 105  * extra byte of encoding and is callee saved.
 106  *
 107  * Also x86-64 register R9 is unused. x86-64 register R10 is
 108  * used for blinding (if enabled).
 109  */
 110 static const int reg2hex[] = {
 111         [BPF_REG_0] = 0,  /* RAX */
 112         [BPF_REG_1] = 7,  /* RDI */
 113         [BPF_REG_2] = 6,  /* RSI */
 114         [BPF_REG_3] = 2,  /* RDX */
 115         [BPF_REG_4] = 1,  /* RCX */
 116         [BPF_REG_5] = 0,  /* R8  */
 117         [BPF_REG_6] = 3,  /* RBX callee saved */
 118         [BPF_REG_7] = 5,  /* R13 callee saved */
 119         [BPF_REG_8] = 6,  /* R14 callee saved */
 120         [BPF_REG_9] = 7,  /* R15 callee saved */
 121         [BPF_REG_FP] = 5, /* RBP readonly */
 122         [BPF_REG_AX] = 2, /* R10 temp register */
 123         [AUX_REG] = 3,    /* R11 temp register */
 124 };
 125
 126 /*
 127  * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15
 128  * which need extra byte of encoding.
 129  * rax,rcx,...,rbp have simpler encoding
 130  */
 131 static bool is_ereg(u32 reg)
 132 {
 133         return (1 << reg) & (BIT(BPF_REG_5) |
 134                              BIT(AUX_REG) |
 135                              BIT(BPF_REG_7) |
 136                              BIT(BPF_REG_8) |
 137                              BIT(BPF_REG_9) |
 138                              BIT(BPF_REG_AX));
 139 }
 140
 141 /*
 142  * is_ereg_8l() == true if BPF register 'reg' is mapped to access x86-64
 143  * lower 8-bit registers dil,sil,bpl,spl,r8b..r15b, which need extra byte
 144  * of encoding. al,cl,dl,bl have simpler encoding.
 145  */
 146 static bool is_ereg_8l(u32 reg)
 147 {
 148         return is_ereg(reg) ||
 149             (1 << reg) & (BIT(BPF_REG_1) |
 150                           BIT(BPF_REG_2) |
 151                           BIT(BPF_REG_FP));
 152 }
 153
 154 static bool is_axreg(u32 reg)
 155 {
 156         return reg == BPF_REG_0;
 157 }
 158
 159 /* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */
 160 static u8 add_1mod(u8 byte, u32 reg)
 161 {
 162         if (is_ereg(reg))
 163                 byte |= 1;
 164         return byte;
 165 }
 166
 167 static u8 add_2mod(u8 byte, u32 r1, u32 r2)
 168 {
 169         if (is_ereg(r1))
 170                 byte |= 1;
 171         if (is_ereg(r2))
 172                 byte |= 4;
 173         return byte;
 174 }
 175
 176 /* Encode 'dst_reg' register into x86-64 opcode 'byte' */
 177 static u8 add_1reg(u8 byte, u32 dst_reg)
 178 {
 179         return byte + reg2hex[dst_reg];
 180 }
 181
 182 /* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */
 183 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
 184 {
 185         return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);
 186 }
 187
 188 static void jit_fill_hole(void *area, unsigned int size)
 189 {
 190         /* Fill whole space with INT3 instructions */
 191         memset(area, 0xcc, size);
 192 }
 193
 194 struct jit_context {
 195         int cleanup_addr; /* Epilogue code offset */
 196 };
 197
 198 /* Maximum number of bytes emitted while JITing one eBPF insn */
 199 #define BPF_MAX_INSN_SIZE       128
 200 #define BPF_INSN_SAFETY         64
 201
 202 #define PROLOGUE_SIZE           20
 203
 204 /*
 205  * Emit x86-64 prologue code for BPF program and check its size.
 206  * bpf_tail_call helper will skip it while jumping into another program
 207  */
 208 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
 209 {
 210         u8 *prog = *pprog;
 211         int cnt = 0;
 212
 213         EMIT1(0x55);             /* push rbp */
 214         EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
 215         /* sub rsp, rounded_stack_depth */
 216         EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));
 217         EMIT1(0x53);             /* push rbx */
 218         EMIT2(0x41, 0x55);       /* push r13 */
 219         EMIT2(0x41, 0x56);       /* push r14 */
 220         EMIT2(0x41, 0x57);       /* push r15 */
 221         if (!ebpf_from_cbpf) {
 222                 /* zero init tail_call_cnt */
 223                 EMIT2(0x6a, 0x00);
 224                 BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
 225         }
 226         *pprog = prog;
 227 }
 228
 229 /*
 230  * Generate the following code:
 231  *
 232  * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
 233  *   if (index >= array->map.max_entries)
 234  *     goto out;
 235  *   if (++tail_call_cnt > MAX_TAIL_CALL_CNT)
 236  *     goto out;
 237  *   prog = array->ptrs[index];
 238  *   if (prog == NULL)
 239  *     goto out;
 240  *   goto *(prog->bpf_func + prologue_size);
 241  * out:
 242  */
 243 static void emit_bpf_tail_call(u8 **pprog)
 244 {
 245         u8 *prog = *pprog;
 246         int label1, label2, label3;
 247         int cnt = 0;
 248
 249         /*
 250          * rdi - pointer to ctx
 251          * rsi - pointer to bpf_array
 252          * rdx - index in bpf_array
 253          */
 254
 255         /*
 256          * if (index >= array->map.max_entries)
 257          *      goto out;
 258          */
 259         EMIT2(0x89, 0xD2);                        /* mov edx, edx */
 260         EMIT3(0x39, 0x56,                         /* cmp dword ptr [rsi + 16], edx */
 261               offsetof(struct bpf_array, map.max_entries));
 262 #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */
 263         EMIT2(X86_JBE, OFFSET1);                  /* jbe out */
 264         label1 = cnt;
 265
 266         /*
 267          * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
 268          *      goto out;
 269          */
 270         EMIT2_off32(0x8B, 0x85, -36 - MAX_BPF_STACK); /* mov eax, dword ptr [rbp - 548] */
 271         EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT);     /* cmp eax, MAX_TAIL_CALL_CNT */
 272 #define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
 273         EMIT2(X86_JA, OFFSET2);                   /* ja out */
 274         label2 = cnt;
 275         EMIT3(0x83, 0xC0, 0x01);                  /* add eax, 1 */
 276         EMIT2_off32(0x89, 0x85, -36 - MAX_BPF_STACK); /* mov dword ptr [rbp -548], eax */
 277
 278         /* prog = array->ptrs[index]; */
 279         EMIT4_off32(0x48, 0x8B, 0x84, 0xD6,       /* mov rax, [rsi + rdx * 8 + offsetof(...)] */
 280                     offsetof(struct bpf_array, ptrs));
 281
 282         /*
 283          * if (prog == NULL)
 284          *      goto out;
 285          */
 286         EMIT3(0x48, 0x85, 0xC0);                  /* test rax,rax */
 287 #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE)
 288         EMIT2(X86_JE, OFFSET3);                   /* je out */
 289         label3 = cnt;
 290
 291         /* goto *(prog->bpf_func + prologue_size); */
 292         EMIT4(0x48, 0x8B, 0x40,                   /* mov rax, qword ptr [rax + 32] */
 293               offsetof(struct bpf_prog, bpf_func));
 294         EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE);   /* add rax, prologue_size */
 295
 296         /*
 297          * Wow we're ready to jump into next BPF program
 298          * rdi == ctx (1st arg)
 299          * rax == prog->bpf_func + prologue_size
 300          */
 301         RETPOLINE_RAX_BPF_JIT();
 302
 303         /* out: */
 304         BUILD_BUG_ON(cnt - label1 != OFFSET1);
 305         BUILD_BUG_ON(cnt - label2 != OFFSET2);
 306         BUILD_BUG_ON(cnt - label3 != OFFSET3);
 307         *pprog = prog;
 308 }
 309
 310 static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
 311                            u32 dst_reg, const u32 imm32)
 312 {
 313         u8 *prog = *pprog;
 314         u8 b1, b2, b3;
 315         int cnt = 0;
 316
 317         /*
 318          * Optimization: if imm32 is positive, use 'mov %eax, imm32'
 319          * (which zero-extends imm32) to save 2 bytes.
 320          */
 321         if (sign_propagate && (s32)imm32 < 0) {
 322                 /* 'mov %rax, imm32' sign extends imm32 */
 323                 b1 = add_1mod(0x48, dst_reg);
 324                 b2 = 0xC7;
 325                 b3 = 0xC0;
 326                 EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32);
 327                 goto done;
 328         }
 329
 330         /*
 331          * Optimization: if imm32 is zero, use 'xor %eax, %eax'
 332          * to save 3 bytes.
 333          */
 334         if (imm32 == 0) {
 335                 if (is_ereg(dst_reg))
 336                         EMIT1(add_2mod(0x40, dst_reg, dst_reg));
 337                 b2 = 0x31; /* xor */
 338                 b3 = 0xC0;
 339                 EMIT2(b2, add_2reg(b3, dst_reg, dst_reg));
 340                 goto done;
 341         }
 342
 343         /* mov %eax, imm32 */
 344         if (is_ereg(dst_reg))
 345                 EMIT1(add_1mod(0x40, dst_reg));
 346         EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);
 347 done:
 348         *pprog = prog;
 349 }
 350
 351 static void emit_mov_imm64(u8 **pprog, u32 dst_reg,
 352                            const u32 imm32_hi, const u32 imm32_lo)
 353 {
 354         u8 *prog = *pprog;
 355         int cnt = 0;
 356
 357         if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) {
 358                 /*
 359                  * For emitting plain u32, where sign bit must not be
 360                  * propagated LLVM tends to load imm64 over mov32
 361                  * directly, so save couple of bytes by just doing
 362                  * 'mov %eax, imm32' instead.
 363                  */
 364                 emit_mov_imm32(&prog, false, dst_reg, imm32_lo);
 365         } else {
 366                 /* movabsq %rax, imm64 */
 367                 EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));
 368                 EMIT(imm32_lo, 4);
 369                 EMIT(imm32_hi, 4);
 370         }
 371
 372         *pprog = prog;
 373 }
 374
 375 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)
 376 {
 377         u8 *prog = *pprog;
 378         int cnt = 0;
 379
 380         if (is64) {
 381                 /* mov dst, src */
 382                 EMIT_mov(dst_reg, src_reg);
 383         } else {
 384                 /* mov32 dst, src */
 385                 if (is_ereg(dst_reg) || is_ereg(src_reg))
 386                         EMIT1(add_2mod(0x40, dst_reg, src_reg));
 387                 EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));
 388         }
 389
 390         *pprog = prog;
 391 }
 392
 393 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 394                   int oldproglen, struct jit_context *ctx)
 395 {
 396         struct bpf_insn *insn = bpf_prog->insnsi;
 397         int insn_cnt = bpf_prog->len;
 398         bool seen_exit = false;
 399         u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
 400         int i, cnt = 0;
 401         int proglen = 0;
 402         u8 *prog = temp;
 403
 404         emit_prologue(&prog, bpf_prog->aux->stack_depth,
 405                       bpf_prog_was_classic(bpf_prog));
 406         addrs[0] = prog - temp;
 407
 408         for (i = 1; i <= insn_cnt; i++, insn++) {
 409                 const s32 imm32 = insn->imm;
 410                 u32 dst_reg = insn->dst_reg;
 411                 u32 src_reg = insn->src_reg;
 412                 u8 b2 = 0, b3 = 0;
 413                 s64 jmp_offset;
 414                 u8 jmp_cond;
 415                 int ilen;
 416                 u8 *func;
 417
 418                 switch (insn->code) {
 419                         /* ALU */
 420                 case BPF_ALU | BPF_ADD | BPF_X:
 421                 case BPF_ALU | BPF_SUB | BPF_X:
 422                 case BPF_ALU | BPF_AND | BPF_X:
 423                 case BPF_ALU | BPF_OR | BPF_X:
 424                 case BPF_ALU | BPF_XOR | BPF_X:
 425                 case BPF_ALU64 | BPF_ADD | BPF_X:
 426                 case BPF_ALU64 | BPF_SUB | BPF_X:
 427                 case BPF_ALU64 | BPF_AND | BPF_X:
 428                 case BPF_ALU64 | BPF_OR | BPF_X:
 429                 case BPF_ALU64 | BPF_XOR | BPF_X:
 430                         switch (BPF_OP(insn->code)) {
 431                         case BPF_ADD: b2 = 0x01; break;
 432                         case BPF_SUB: b2 = 0x29; break;
 433                         case BPF_AND: b2 = 0x21; break;
 434                         case BPF_OR: b2 = 0x09; break;
 435                         case BPF_XOR: b2 = 0x31; break;
 436                         }
 437                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 438                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
 439                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
 440                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
 441                         EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));
 442                         break;
 443
 444                 case BPF_ALU64 | BPF_MOV | BPF_X:
 445                 case BPF_ALU | BPF_MOV | BPF_X:
 446                         emit_mov_reg(&prog,
 447                                      BPF_CLASS(insn->code) == BPF_ALU64,
 448                                      dst_reg, src_reg);
 449                         break;
 450
 451                         /* neg dst */
 452                 case BPF_ALU | BPF_NEG:
 453                 case BPF_ALU64 | BPF_NEG:
 454                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 455                                 EMIT1(add_1mod(0x48, dst_reg));
 456                         else if (is_ereg(dst_reg))
 457                                 EMIT1(add_1mod(0x40, dst_reg));
 458                         EMIT2(0xF7, add_1reg(0xD8, dst_reg));
 459                         break;
 460
 461                 case BPF_ALU | BPF_ADD | BPF_K:
 462                 case BPF_ALU | BPF_SUB | BPF_K:
 463                 case BPF_ALU | BPF_AND | BPF_K:
 464                 case BPF_ALU | BPF_OR | BPF_K:
 465                 case BPF_ALU | BPF_XOR | BPF_K:
 466                 case BPF_ALU64 | BPF_ADD | BPF_K:
 467                 case BPF_ALU64 | BPF_SUB | BPF_K:
 468                 case BPF_ALU64 | BPF_AND | BPF_K:
 469                 case BPF_ALU64 | BPF_OR | BPF_K:
 470                 case BPF_ALU64 | BPF_XOR | BPF_K:
 471                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 472                                 EMIT1(add_1mod(0x48, dst_reg));
 473                         else if (is_ereg(dst_reg))
 474                                 EMIT1(add_1mod(0x40, dst_reg));
 475
 476                         /*
 477                          * b3 holds 'normal' opcode, b2 short form only valid
 478                          * in case dst is eax/rax.
 479                          */
 480                         switch (BPF_OP(insn->code)) {
 481                         case BPF_ADD:
 482                                 b3 = 0xC0;
 483                                 b2 = 0x05;
 484                                 break;
 485                         case BPF_SUB:
 486                                 b3 = 0xE8;
 487                                 b2 = 0x2D;
 488                                 break;
 489                         case BPF_AND:
 490                                 b3 = 0xE0;
 491                                 b2 = 0x25;
 492                                 break;
 493                         case BPF_OR:
 494                                 b3 = 0xC8;
 495                                 b2 = 0x0D;
 496                                 break;
 497                         case BPF_XOR:
 498                                 b3 = 0xF0;
 499                                 b2 = 0x35;
 500                                 break;
 501                         }
 502
 503                         if (is_imm8(imm32))
 504                                 EMIT3(0x83, add_1reg(b3, dst_reg), imm32);
 505                         else if (is_axreg(dst_reg))
 506                                 EMIT1_off32(b2, imm32);
 507                         else
 508                                 EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);
 509                         break;
 510
 511                 case BPF_ALU64 | BPF_MOV | BPF_K:
 512                 case BPF_ALU | BPF_MOV | BPF_K:
 513                         emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64,
 514                                        dst_reg, imm32);
 515                         break;
 516
 517                 case BPF_LD | BPF_IMM | BPF_DW:
 518                         emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm);
 519                         insn++;
 520                         i++;
 521                         break;
 522
 523                         /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
 524                 case BPF_ALU | BPF_MOD | BPF_X:
 525                 case BPF_ALU | BPF_DIV | BPF_X:
 526                 case BPF_ALU | BPF_MOD | BPF_K:
 527                 case BPF_ALU | BPF_DIV | BPF_K:
 528                 case BPF_ALU64 | BPF_MOD | BPF_X:
 529                 case BPF_ALU64 | BPF_DIV | BPF_X:
 530                 case BPF_ALU64 | BPF_MOD | BPF_K:
 531                 case BPF_ALU64 | BPF_DIV | BPF_K:
 532                         EMIT1(0x50); /* push rax */
 533                         EMIT1(0x52); /* push rdx */
 534
 535                         if (BPF_SRC(insn->code) == BPF_X)
 536                                 /* mov r11, src_reg */
 537                                 EMIT_mov(AUX_REG, src_reg);
 538                         else
 539                                 /* mov r11, imm32 */
 540                                 EMIT3_off32(0x49, 0xC7, 0xC3, imm32);
 541
 542                         /* mov rax, dst_reg */
 543                         EMIT_mov(BPF_REG_0, dst_reg);
 544
 545                         /*
 546                          * xor edx, edx
 547                          * equivalent to 'xor rdx, rdx', but one byte less
 548                          */
 549                         EMIT2(0x31, 0xd2);
 550
 551                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 552                                 /* div r11 */
 553                                 EMIT3(0x49, 0xF7, 0xF3);
 554                         else
 555                                 /* div r11d */
 556                                 EMIT3(0x41, 0xF7, 0xF3);
 557
 558                         if (BPF_OP(insn->code) == BPF_MOD)
 559                                 /* mov r11, rdx */
 560                                 EMIT3(0x49, 0x89, 0xD3);
 561                         else
 562                                 /* mov r11, rax */
 563                                 EMIT3(0x49, 0x89, 0xC3);
 564
 565                         EMIT1(0x5A); /* pop rdx */
 566                         EMIT1(0x58); /* pop rax */
 567
 568                         /* mov dst_reg, r11 */
 569                         EMIT_mov(dst_reg, AUX_REG);
 570                         break;
 571
 572                 case BPF_ALU | BPF_MUL | BPF_K:
 573                 case BPF_ALU | BPF_MUL | BPF_X:
 574                 case BPF_ALU64 | BPF_MUL | BPF_K:
 575                 case BPF_ALU64 | BPF_MUL | BPF_X:
 576                 {
 577                         bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
 578
 579                         if (dst_reg != BPF_REG_0)
 580                                 EMIT1(0x50); /* push rax */
 581                         if (dst_reg != BPF_REG_3)
 582                                 EMIT1(0x52); /* push rdx */
 583
 584                         /* mov r11, dst_reg */
 585                         EMIT_mov(AUX_REG, dst_reg);
 586
 587                         if (BPF_SRC(insn->code) == BPF_X)
 588                                 emit_mov_reg(&prog, is64, BPF_REG_0, src_reg);
 589                         else
 590                                 emit_mov_imm32(&prog, is64, BPF_REG_0, imm32);
 591
 592                         if (is64)
 593                                 EMIT1(add_1mod(0x48, AUX_REG));
 594                         else if (is_ereg(AUX_REG))
 595                                 EMIT1(add_1mod(0x40, AUX_REG));
 596                         /* mul(q) r11 */
 597                         EMIT2(0xF7, add_1reg(0xE0, AUX_REG));
 598
 599                         if (dst_reg != BPF_REG_3)
 600                                 EMIT1(0x5A); /* pop rdx */
 601                         if (dst_reg != BPF_REG_0) {
 602                                 /* mov dst_reg, rax */
 603                                 EMIT_mov(dst_reg, BPF_REG_0);
 604                                 EMIT1(0x58); /* pop rax */
 605                         }
 606                         break;
 607                 }
 608                         /* Shifts */
 609                 case BPF_ALU | BPF_LSH | BPF_K:
 610                 case BPF_ALU | BPF_RSH | BPF_K:
 611                 case BPF_ALU | BPF_ARSH | BPF_K:
 612                 case BPF_ALU64 | BPF_LSH | BPF_K:
 613                 case BPF_ALU64 | BPF_RSH | BPF_K:
 614                 case BPF_ALU64 | BPF_ARSH | BPF_K:
 615                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 616                                 EMIT1(add_1mod(0x48, dst_reg));
 617                         else if (is_ereg(dst_reg))
 618                                 EMIT1(add_1mod(0x40, dst_reg));
 619
 620                         switch (BPF_OP(insn->code)) {
 621                         case BPF_LSH: b3 = 0xE0; break;
 622                         case BPF_RSH: b3 = 0xE8; break;
 623                         case BPF_ARSH: b3 = 0xF8; break;
 624                         }
 625
 626                         if (imm32 == 1)
 627                                 EMIT2(0xD1, add_1reg(b3, dst_reg));
 628                         else
 629                                 EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
 630                         break;
 631
 632                 case BPF_ALU | BPF_LSH | BPF_X:
 633                 case BPF_ALU | BPF_RSH | BPF_X:
 634                 case BPF_ALU | BPF_ARSH | BPF_X:
 635                 case BPF_ALU64 | BPF_LSH | BPF_X:
 636                 case BPF_ALU64 | BPF_RSH | BPF_X:
 637                 case BPF_ALU64 | BPF_ARSH | BPF_X:
 638
 639                         /* Check for bad case when dst_reg == rcx */
 640                         if (dst_reg == BPF_REG_4) {
 641                                 /* mov r11, dst_reg */
 642                                 EMIT_mov(AUX_REG, dst_reg);
 643                                 dst_reg = AUX_REG;
 644                         }
 645
 646                         if (src_reg != BPF_REG_4) { /* common case */
 647                                 EMIT1(0x51); /* push rcx */
 648
 649                                 /* mov rcx, src_reg */
 650                                 EMIT_mov(BPF_REG_4, src_reg);
 651                         }
 652
 653                         /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
 654                         if (BPF_CLASS(insn->code) == BPF_ALU64)
 655                                 EMIT1(add_1mod(0x48, dst_reg));
 656                         else if (is_ereg(dst_reg))
 657                                 EMIT1(add_1mod(0x40, dst_reg));
 658
 659                         switch (BPF_OP(insn->code)) {
 660                         case BPF_LSH: b3 = 0xE0; break;
 661                         case BPF_RSH: b3 = 0xE8; break;
 662                         case BPF_ARSH: b3 = 0xF8; break;
 663                         }
 664                         EMIT2(0xD3, add_1reg(b3, dst_reg));
 665
 666                         if (src_reg != BPF_REG_4)
 667                                 EMIT1(0x59); /* pop rcx */
 668
 669                         if (insn->dst_reg == BPF_REG_4)
 670                                 /* mov dst_reg, r11 */
 671                                 EMIT_mov(insn->dst_reg, AUX_REG);
 672                         break;
 673
 674                 case BPF_ALU | BPF_END | BPF_FROM_BE:
 675                         switch (imm32) {
 676                         case 16:
 677                                 /* Emit 'ror %ax, 8' to swap lower 2 bytes */
 678                                 EMIT1(0x66);
 679                                 if (is_ereg(dst_reg))
 680                                         EMIT1(0x41);
 681                                 EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);
 682
 683                                 /* Emit 'movzwl eax, ax' */
 684                                 if (is_ereg(dst_reg))
 685                                         EMIT3(0x45, 0x0F, 0xB7);
 686                                 else
 687                                         EMIT2(0x0F, 0xB7);
 688                                 EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
 689                                 break;
 690                         case 32:
 691                                 /* Emit 'bswap eax' to swap lower 4 bytes */
 692                                 if (is_ereg(dst_reg))
 693                                         EMIT2(0x41, 0x0F);
 694                                 else
 695                                         EMIT1(0x0F);
 696                                 EMIT1(add_1reg(0xC8, dst_reg));
 697                                 break;
 698                         case 64:
 699                                 /* Emit 'bswap rax' to swap 8 bytes */
 700                                 EMIT3(add_1mod(0x48, dst_reg), 0x0F,
 701                                       add_1reg(0xC8, dst_reg));
 702                                 break;
 703                         }
 704                         break;
 705
 706                 case BPF_ALU | BPF_END | BPF_FROM_LE:
 707                         switch (imm32) {
 708                         case 16:
 709                                 /*
 710                                  * Emit 'movzwl eax, ax' to zero extend 16-bit
 711                                  * into 64 bit
 712                                  */
 713                                 if (is_ereg(dst_reg))
 714                                         EMIT3(0x45, 0x0F, 0xB7);
 715                                 else
 716                                         EMIT2(0x0F, 0xB7);
 717                                 EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
 718                                 break;
 719                         case 32:
 720                                 /* Emit 'mov eax, eax' to clear upper 32-bits */
 721                                 if (is_ereg(dst_reg))
 722                                         EMIT1(0x45);
 723                                 EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));
 724                                 break;
 725                         case 64:
 726                                 /* nop */
 727                                 break;
 728                         }
 729                         break;
 730
 731                         /* speculation barrier */
 732                 case BPF_ST | BPF_NOSPEC:
 733                         if (boot_cpu_has(X86_FEATURE_XMM2))
 734                                 /* Emit 'lfence' */
 735                                 EMIT3(0x0F, 0xAE, 0xE8);
 736                         break;
 737
 738                         /* ST: *(u8*)(dst_reg + off) = imm */
 739                 case BPF_ST | BPF_MEM | BPF_B:
 740                         if (is_ereg(dst_reg))
 741                                 EMIT2(0x41, 0xC6);
 742                         else
 743                                 EMIT1(0xC6);
 744                         goto st;
 745                 case BPF_ST | BPF_MEM | BPF_H:
 746                         if (is_ereg(dst_reg))
 747                                 EMIT3(0x66, 0x41, 0xC7);
 748                         else
 749                                 EMIT2(0x66, 0xC7);
 750                         goto st;
 751                 case BPF_ST | BPF_MEM | BPF_W:
 752                         if (is_ereg(dst_reg))
 753                                 EMIT2(0x41, 0xC7);
 754                         else
 755                                 EMIT1(0xC7);
 756                         goto st;
 757                 case BPF_ST | BPF_MEM | BPF_DW:
 758                         EMIT2(add_1mod(0x48, dst_reg), 0xC7);
 759
 760 st:                     if (is_imm8(insn->off))
 761                                 EMIT2(add_1reg(0x40, dst_reg), insn->off);
 762                         else
 763                                 EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);
 764
 765                         EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
 766                         break;
 767
 768                         /* STX: *(u8*)(dst_reg + off) = src_reg */
 769                 case BPF_STX | BPF_MEM | BPF_B:
 770                         /* Emit 'mov byte ptr [rax + off], al' */
 771                         if (is_ereg(dst_reg) || is_ereg_8l(src_reg))
 772                                 /* Add extra byte for eregs or SIL,DIL,BPL in src_reg */
 773                                 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);
 774                         else
 775                                 EMIT1(0x88);
 776                         goto stx;
 777                 case BPF_STX | BPF_MEM | BPF_H:
 778                         if (is_ereg(dst_reg) || is_ereg(src_reg))
 779                                 EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);
 780                         else
 781                                 EMIT2(0x66, 0x89);
 782                         goto stx;
 783                 case BPF_STX | BPF_MEM | BPF_W:
 784                         if (is_ereg(dst_reg) || is_ereg(src_reg))
 785                                 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);
 786                         else
 787                                 EMIT1(0x89);
 788                         goto stx;
 789                 case BPF_STX | BPF_MEM | BPF_DW:
 790                         EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);
 791 stx:                    if (is_imm8(insn->off))
 792                                 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
 793                         else
 794                                 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
 795                                             insn->off);
 796                         break;
 797
 798                         /* LDX: dst_reg = *(u8*)(src_reg + off) */
 799                 case BPF_LDX | BPF_MEM | BPF_B:
 800                         /* Emit 'movzx rax, byte ptr [rax + off]' */
 801                         EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
 802                         goto ldx;
 803                 case BPF_LDX | BPF_MEM | BPF_H:
 804                         /* Emit 'movzx rax, word ptr [rax + off]' */
 805                         EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
 806                         goto ldx;
 807                 case BPF_LDX | BPF_MEM | BPF_W:
 808                         /* Emit 'mov eax, dword ptr [rax+0x14]' */
 809                         if (is_ereg(dst_reg) || is_ereg(src_reg))
 810                                 EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
 811                         else
 812                                 EMIT1(0x8B);
 813                         goto ldx;
 814                 case BPF_LDX | BPF_MEM | BPF_DW:
 815                         /* Emit 'mov rax, qword ptr [rax+0x14]' */
 816                         EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
 817 ldx:                    /*
 818                          * If insn->off == 0 we can save one extra byte, but
 819                          * special case of x86 R13 which always needs an offset
 820                          * is not worth the hassle
 821                          */
 822                         if (is_imm8(insn->off))
 823                                 EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off);
 824                         else
 825                                 EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),
 826                                             insn->off);
 827                         break;
 828
 829                         /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */
 830                 case BPF_STX | BPF_XADD | BPF_W:
 831                         /* Emit 'lock add dword ptr [rax + off], eax' */
 832                         if (is_ereg(dst_reg) || is_ereg(src_reg))
 833                                 EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01);
 834                         else
 835                                 EMIT2(0xF0, 0x01);
 836                         goto xadd;
 837                 case BPF_STX | BPF_XADD | BPF_DW:
 838                         EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01);
 839 xadd:                   if (is_imm8(insn->off))
 840                                 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
 841                         else
 842                                 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
 843                                             insn->off);
 844                         break;
 845
 846                         /* call */
 847                 case BPF_JMP | BPF_CALL:
 848                         func = (u8 *) __bpf_call_base + imm32;
 849                         jmp_offset = func - (image + addrs[i]);
 850                         if (!imm32 || !is_simm32(jmp_offset)) {
 851                                 pr_err("unsupported BPF func %d addr %p image %p\n",
 852                                        imm32, func, image);
 853                                 return -EINVAL;
 854                         }
 855                         EMIT1_off32(0xE8, jmp_offset);
 856                         break;
 857
 858                 case BPF_JMP | BPF_TAIL_CALL:
 859                         emit_bpf_tail_call(&prog);
 860                         break;
 861
 862                         /* cond jump */
 863                 case BPF_JMP | BPF_JEQ | BPF_X:
 864                 case BPF_JMP | BPF_JNE | BPF_X:
 865                 case BPF_JMP | BPF_JGT | BPF_X:
 866                 case BPF_JMP | BPF_JLT | BPF_X:
 867                 case BPF_JMP | BPF_JGE | BPF_X:
 868                 case BPF_JMP | BPF_JLE | BPF_X:
 869                 case BPF_JMP | BPF_JSGT | BPF_X:
 870                 case BPF_JMP | BPF_JSLT | BPF_X:
 871                 case BPF_JMP | BPF_JSGE | BPF_X:
 872                 case BPF_JMP | BPF_JSLE | BPF_X:
 873                 case BPF_JMP32 | BPF_JEQ | BPF_X:
 874                 case BPF_JMP32 | BPF_JNE | BPF_X:
 875                 case BPF_JMP32 | BPF_JGT | BPF_X:
 876                 case BPF_JMP32 | BPF_JLT | BPF_X:
 877                 case BPF_JMP32 | BPF_JGE | BPF_X:
 878                 case BPF_JMP32 | BPF_JLE | BPF_X:
 879                 case BPF_JMP32 | BPF_JSGT | BPF_X:
 880                 case BPF_JMP32 | BPF_JSLT | BPF_X:
 881                 case BPF_JMP32 | BPF_JSGE | BPF_X:
 882                 case BPF_JMP32 | BPF_JSLE | BPF_X:
 883                         /* cmp dst_reg, src_reg */
 884                         if (BPF_CLASS(insn->code) == BPF_JMP)
 885                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
 886                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
 887                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
 888                         EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));
 889                         goto emit_cond_jmp;
 890
 891                 case BPF_JMP | BPF_JSET | BPF_X:
 892                 case BPF_JMP32 | BPF_JSET | BPF_X:
 893                         /* test dst_reg, src_reg */
 894                         if (BPF_CLASS(insn->code) == BPF_JMP)
 895                                 EMIT1(add_2mod(0x48, dst_reg, src_reg));
 896                         else if (is_ereg(dst_reg) || is_ereg(src_reg))
 897                                 EMIT1(add_2mod(0x40, dst_reg, src_reg));
 898                         EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));
 899                         goto emit_cond_jmp;
 900
 901                 case BPF_JMP | BPF_JSET | BPF_K:
 902                 case BPF_JMP32 | BPF_JSET | BPF_K:
 903                         /* test dst_reg, imm32 */
 904                         if (BPF_CLASS(insn->code) == BPF_JMP)
 905                                 EMIT1(add_1mod(0x48, dst_reg));
 906                         else if (is_ereg(dst_reg))
 907                                 EMIT1(add_1mod(0x40, dst_reg));
 908                         EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
 909                         goto emit_cond_jmp;
 910
 911                 case BPF_JMP | BPF_JEQ | BPF_K:
 912                 case BPF_JMP | BPF_JNE | BPF_K:
 913                 case BPF_JMP | BPF_JGT | BPF_K:
 914                 case BPF_JMP | BPF_JLT | BPF_K:
 915                 case BPF_JMP | BPF_JGE | BPF_K:
 916                 case BPF_JMP | BPF_JLE | BPF_K:
 917                 case BPF_JMP | BPF_JSGT | BPF_K:
 918                 case BPF_JMP | BPF_JSLT | BPF_K:
 919                 case BPF_JMP | BPF_JSGE | BPF_K:
 920                 case BPF_JMP | BPF_JSLE | BPF_K:
 921                 case BPF_JMP32 | BPF_JEQ | BPF_K:
 922                 case BPF_JMP32 | BPF_JNE | BPF_K:
 923                 case BPF_JMP32 | BPF_JGT | BPF_K:
 924                 case BPF_JMP32 | BPF_JLT | BPF_K:
 925                 case BPF_JMP32 | BPF_JGE | BPF_K:
 926                 case BPF_JMP32 | BPF_JLE | BPF_K:
 927                 case BPF_JMP32 | BPF_JSGT | BPF_K:
 928                 case BPF_JMP32 | BPF_JSLT | BPF_K:
 929                 case BPF_JMP32 | BPF_JSGE | BPF_K:
 930                 case BPF_JMP32 | BPF_JSLE | BPF_K:
 931                         /* cmp dst_reg, imm8/32 */
 932                         if (BPF_CLASS(insn->code) == BPF_JMP)
 933                                 EMIT1(add_1mod(0x48, dst_reg));
 934                         else if (is_ereg(dst_reg))
 935                                 EMIT1(add_1mod(0x40, dst_reg));
 936
 937                         if (is_imm8(imm32))
 938                                 EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);
 939                         else
 940                                 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32);
 941
 942 emit_cond_jmp:          /* Convert BPF opcode to x86 */
 943                         switch (BPF_OP(insn->code)) {
 944                         case BPF_JEQ:
 945                                 jmp_cond = X86_JE;
 946                                 break;
 947                         case BPF_JSET:
 948                         case BPF_JNE:
 949                                 jmp_cond = X86_JNE;
 950                                 break;
 951                         case BPF_JGT:
 952                                 /* GT is unsigned '>', JA in x86 */
 953                                 jmp_cond = X86_JA;
 954                                 break;
 955                         case BPF_JLT:
 956                                 /* LT is unsigned '<', JB in x86 */
 957                                 jmp_cond = X86_JB;
 958                                 break;
 959                         case BPF_JGE:
 960                                 /* GE is unsigned '>=', JAE in x86 */
 961                                 jmp_cond = X86_JAE;
 962                                 break;
 963                         case BPF_JLE:
 964                                 /* LE is unsigned '<=', JBE in x86 */
 965                                 jmp_cond = X86_JBE;
 966                                 break;
 967                         case BPF_JSGT:
 968                                 /* Signed '>', GT in x86 */
 969                                 jmp_cond = X86_JG;
 970                                 break;
 971                         case BPF_JSLT:
 972                                 /* Signed '<', LT in x86 */
 973                                 jmp_cond = X86_JL;
 974                                 break;
 975                         case BPF_JSGE:
 976                                 /* Signed '>=', GE in x86 */
 977                                 jmp_cond = X86_JGE;
 978                                 break;
 979                         case BPF_JSLE:
 980                                 /* Signed '<=', LE in x86 */
 981                                 jmp_cond = X86_JLE;
 982                                 break;
 983                         default: /* to silence GCC warning */
 984                                 return -EFAULT;
 985                         }
 986                         jmp_offset = addrs[i + insn->off] - addrs[i];
 987                         if (is_imm8(jmp_offset)) {
 988                                 EMIT2(jmp_cond, jmp_offset);
 989                         } else if (is_simm32(jmp_offset)) {
 990                                 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
 991                         } else {
 992                                 pr_err("cond_jmp gen bug %llx\n", jmp_offset);
 993                                 return -EFAULT;
 994                         }
 995
 996                         break;
 997
 998                 case BPF_JMP | BPF_JA:
 999                         if (insn->off == -1)
1000                                 /* -1 jmp instructions will always jump
1001                                  * backwards two bytes. Explicitly handling
1002                                  * this case avoids wasting too many passes
1003                                  * when there are long sequences of replaced
1004                                  * dead code.
1005                                  */
1006                                 jmp_offset = -2;
1007                         else
1008                                 jmp_offset = addrs[i + insn->off] - addrs[i];
1009
1010                         if (!jmp_offset)
1011                                 /* Optimize out nop jumps */
1012                                 break;
1013 emit_jmp:
1014                         if (is_imm8(jmp_offset)) {
1015                                 EMIT2(0xEB, jmp_offset);
1016                         } else if (is_simm32(jmp_offset)) {
1017                                 EMIT1_off32(0xE9, jmp_offset);
1018                         } else {
1019                                 pr_err("jmp gen bug %llx\n", jmp_offset);
1020                                 return -EFAULT;
1021                         }
1022                         break;
1023
1024                 case BPF_JMP | BPF_EXIT:
1025                         if (seen_exit) {
1026                                 jmp_offset = ctx->cleanup_addr - addrs[i];
1027                                 goto emit_jmp;
1028                         }
1029                         seen_exit = true;
1030                         /* Update cleanup_addr */
1031                         ctx->cleanup_addr = proglen;
1032                         if (!bpf_prog_was_classic(bpf_prog))
1033                                 EMIT1(0x5B); /* get rid of tail_call_cnt */
1034                         EMIT2(0x41, 0x5F);   /* pop r15 */
1035                         EMIT2(0x41, 0x5E);   /* pop r14 */
1036                         EMIT2(0x41, 0x5D);   /* pop r13 */
1037                         EMIT1(0x5B);         /* pop rbx */
1038                         EMIT1(0xC9);         /* leave */
1039                         EMIT1(0xC3);         /* ret */
1040                         break;
1041
1042                 default:
1043                         /*
1044                          * By design x86-64 JIT should support all BPF instructions.
1045                          * This error will be seen if new instruction was added
1046                          * to the interpreter, but not to the JIT, or if there is
1047                          * junk in bpf_prog.
1048                          */
1049                         pr_err("bpf_jit: unknown opcode %02x\n", insn->code);
1050                         return -EINVAL;
1051                 }
1052
1053                 ilen = prog - temp;
1054                 if (ilen > BPF_MAX_INSN_SIZE) {
1055                         pr_err("bpf_jit: fatal insn size error\n");
1056                         return -EFAULT;
1057                 }
1058
1059                 if (image) {
1060                         /*
1061                          * When populating the image, assert that:
1062                          *
1063                          *  i) We do not write beyond the allocated space, and
1064                          * ii) addrs[i] did not change from the prior run, in order
1065                          *     to validate assumptions made for computing branch
1066                          *     displacements.
1067                          */
1068                         if (unlikely(proglen + ilen > oldproglen ||
1069                                      proglen + ilen != addrs[i])) {
1070                                 pr_err("bpf_jit: fatal error\n");
1071                                 return -EFAULT;
1072                         }
1073                         memcpy(image + proglen, temp, ilen);
1074                 }
1075                 proglen += ilen;
1076                 addrs[i] = proglen;
1077                 prog = temp;
1078         }
1079         return proglen;
1080 }
1081
1082 struct x64_jit_data {
1083         struct bpf_binary_header *header;
1084         int *addrs;
1085         u8 *image;
1086         int proglen;
1087         struct jit_context ctx;
1088 };
1089
1090 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1091 {
1092         struct bpf_binary_header *header = NULL;
1093         struct bpf_prog *tmp, *orig_prog = prog;
1094         struct x64_jit_data *jit_data;
1095         int proglen, oldproglen = 0;
1096         struct jit_context ctx = {};
1097         bool tmp_blinded = false;
1098         bool extra_pass = false;
1099         u8 *image = NULL;
1100         int *addrs;
1101         int pass;
1102         int i;
1103
1104         if (!prog->jit_requested)
1105                 return orig_prog;
1106
1107         tmp = bpf_jit_blind_constants(prog);
1108         /*
1109          * If blinding was requested and we failed during blinding,
1110          * we must fall back to the interpreter.
1111          */
1112         if (IS_ERR(tmp))
1113                 return orig_prog;
1114         if (tmp != prog) {
1115                 tmp_blinded = true;
1116                 prog = tmp;
1117         }
1118
1119         jit_data = prog->aux->jit_data;
1120         if (!jit_data) {
1121                 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1122                 if (!jit_data) {
1123                         prog = orig_prog;
1124                         goto out;
1125                 }
1126                 prog->aux->jit_data = jit_data;
1127         }
1128         addrs = jit_data->addrs;
1129         if (addrs) {
1130                 ctx = jit_data->ctx;
1131                 oldproglen = jit_data->proglen;
1132                 image = jit_data->image;
1133                 header = jit_data->header;
1134                 extra_pass = true;
1135                 goto skip_init_addrs;
1136         }
1137         addrs = kvmalloc_array(prog->len + 1, sizeof(*addrs), GFP_KERNEL);
1138         if (!addrs) {
1139                 prog = orig_prog;
1140                 goto out_addrs;
1141         }
1142
1143         /*
1144          * Before first pass, make a rough estimation of addrs[]
1145          * each BPF instruction is translated to less than 64 bytes
1146          */
1147         for (proglen = 0, i = 0; i <= prog->len; i++) {
1148                 proglen += 64;
1149                 addrs[i] = proglen;
1150         }
1151         ctx.cleanup_addr = proglen;
1152 skip_init_addrs:
1153
1154         /*
1155          * JITed image shrinks with every pass and the loop iterates
1156          * until the image stops shrinking. Very large BPF programs
1157          * may converge on the last pass. In such case do one more
1158          * pass to emit the final image.
1159          */
1160         for (pass = 0; pass < 20 || image; pass++) {
1161                 proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
1162                 if (proglen <= 0) {
1163 out_image:
1164                         image = NULL;
1165                         if (header)
1166                                 bpf_jit_binary_free(header);
1167                         prog = orig_prog;
1168                         goto out_addrs;
1169                 }
1170                 if (image) {
1171                         if (proglen != oldproglen) {
1172                                 pr_err("bpf_jit: proglen=%d != oldproglen=%d\n",
1173                                        proglen, oldproglen);
1174                                 goto out_image;
1175                         }
1176                         break;
1177                 }
1178                 if (proglen == oldproglen) {
1179                         header = bpf_jit_binary_alloc(proglen, &image,
1180                                                       1, jit_fill_hole);
1181                         if (!header) {
1182                                 prog = orig_prog;
1183                                 goto out_addrs;
1184                         }
1185                 }
1186                 oldproglen = proglen;
1187                 cond_resched();
1188         }
1189
1190         if (bpf_jit_enable > 1)
1191                 bpf_jit_dump(prog->len, proglen, pass + 1, image);
1192
1193         if (image) {
1194                 if (!prog->is_func || extra_pass) {
1195                         bpf_jit_binary_lock_ro(header);
1196                 } else {
1197                         jit_data->addrs = addrs;
1198                         jit_data->ctx = ctx;
1199                         jit_data->proglen = proglen;
1200                         jit_data->image = image;
1201                         jit_data->header = header;
1202                 }
1203                 prog->bpf_func = (void *)image;
1204                 prog->jited = 1;
1205                 prog->jited_len = proglen;
1206         } else {
1207                 prog = orig_prog;
1208         }
1209
1210         if (!image || !prog->is_func || extra_pass) {
1211                 if (image)
1212                         bpf_prog_fill_jited_linfo(prog, addrs + 1);
1213 out_addrs:
1214                 kvfree(addrs);
1215                 kfree(jit_data);
1216                 prog->aux->jit_data = NULL;
1217         }
1218 out:
1219         if (tmp_blinded)
1220                 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1221                                            tmp : orig_prog);
1222         return prog;
1223 }