1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Common functionality for RV32 and RV64 BPF JIT compilers
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
12 #include <linux/bpf.h>
13 #include <linux/filter.h>
14 #include <asm/cacheflush.h>
16 static inline bool rvc_enabled(void)
18 return IS_ENABLED(CONFIG_RISCV_ISA_C);
22 RV_REG_ZERO = 0, /* The constant value 0 */
23 RV_REG_RA = 1, /* Return address */
24 RV_REG_SP = 2, /* Stack pointer */
25 RV_REG_GP = 3, /* Global pointer */
26 RV_REG_TP = 4, /* Thread pointer */
27 RV_REG_T0 = 5, /* Temporaries */
30 RV_REG_FP = 8, /* Saved register/frame pointer */
31 RV_REG_S1 = 9, /* Saved register */
32 RV_REG_A0 = 10, /* Function argument/return values */
33 RV_REG_A1 = 11, /* Function arguments */
40 RV_REG_S2 = 18, /* Saved registers */
50 RV_REG_T3 = 28, /* Temporaries */
56 static inline bool is_creg(u8 reg)
58 return (1 << reg) & (BIT(RV_REG_FP) |
68 struct rv_jit_context {
69 struct bpf_prog *prog;
70 u16 *insns; /* RV insns */
75 int *offset; /* BPF to RV */
81 /* Convert from ninsns to bytes. */
82 static inline int ninsns_rvoff(int ninsns)
88 struct bpf_binary_header *header;
89 struct bpf_binary_header *ro_header;
92 struct rv_jit_context ctx;
95 static inline void bpf_fill_ill_insns(void *area, unsigned int size)
97 memset(area, 0, size);
100 static inline void bpf_flush_icache(void *start, void *end)
102 flush_icache_range((unsigned long)start, (unsigned long)end);
105 /* Emit a 4-byte riscv instruction. */
106 static inline void emit(const u32 insn, struct rv_jit_context *ctx)
109 ctx->insns[ctx->ninsns] = insn;
110 ctx->insns[ctx->ninsns + 1] = (insn >> 16);
116 /* Emit a 2-byte riscv compressed instruction. */
117 static inline void emitc(const u16 insn, struct rv_jit_context *ctx)
119 BUILD_BUG_ON(!rvc_enabled());
122 ctx->insns[ctx->ninsns] = insn;
127 static inline int epilogue_offset(struct rv_jit_context *ctx)
129 int to = ctx->epilogue_offset, from = ctx->ninsns;
131 return ninsns_rvoff(to - from);
134 /* Return -1 or inverted cond. */
135 static inline int invert_bpf_cond(u8 cond)
162 static inline bool is_6b_int(long val)
164 return -(1L << 5) <= val && val < (1L << 5);
167 static inline bool is_7b_uint(unsigned long val)
169 return val < (1UL << 7);
172 static inline bool is_8b_uint(unsigned long val)
174 return val < (1UL << 8);
177 static inline bool is_9b_uint(unsigned long val)
179 return val < (1UL << 9);
182 static inline bool is_10b_int(long val)
184 return -(1L << 9) <= val && val < (1L << 9);
187 static inline bool is_10b_uint(unsigned long val)
189 return val < (1UL << 10);
192 static inline bool is_12b_int(long val)
194 return -(1L << 11) <= val && val < (1L << 11);
197 static inline int is_12b_check(int off, int insn)
199 if (!is_12b_int(off)) {
200 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
207 static inline bool is_13b_int(long val)
209 return -(1L << 12) <= val && val < (1L << 12);
212 static inline bool is_21b_int(long val)
214 return -(1L << 20) <= val && val < (1L << 20);
217 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
221 off++; /* BPF branch is from PC+1, RV is from PC */
222 from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len;
223 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len;
224 return ninsns_rvoff(to - from);
227 /* Instruction formats. */
229 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
232 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
236 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
238 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
242 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
244 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
246 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
247 (imm4_0 << 7) | opcode;
250 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
252 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
253 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
255 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
256 (imm4_1 << 7) | opcode;
259 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
261 return (imm31_12 << 12) | (rd << 7) | opcode;
264 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
268 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
269 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
271 return (imm << 12) | (rd << 7) | opcode;
274 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
275 u8 funct3, u8 rd, u8 opcode)
277 u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
279 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
282 /* RISC-V compressed instruction formats. */
284 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)
286 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;
289 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)
293 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
294 return (funct3 << 13) | (rd << 7) | op | imm;
297 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)
299 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;
302 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)
304 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;
307 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,
310 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
311 (imm_lo << 5) | ((rd & 0x7) << 2) | op;
314 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,
317 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
318 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op;
321 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)
323 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |
324 ((rs2 & 0x7) << 2) | op;
327 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)
331 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
332 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;
335 /* Instructions shared by both RV32 and RV64. */
337 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
339 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
342 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
344 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
347 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
349 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
352 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
354 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
357 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
359 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
362 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
364 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
367 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
369 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
372 static inline u32 rv_lui(u8 rd, u32 imm31_12)
374 return rv_u_insn(imm31_12, rd, 0x37);
377 static inline u32 rv_auipc(u8 rd, u32 imm31_12)
379 return rv_u_insn(imm31_12, rd, 0x17);
382 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
384 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
387 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
389 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
392 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
394 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
397 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
399 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
402 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
404 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
407 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
409 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
412 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
414 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
417 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
419 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
422 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
424 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
427 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
429 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
432 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
434 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
437 static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2)
439 return rv_r_insn(1, rs2, rs1, 4, rd, 0x33);
442 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
444 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
447 static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2)
449 return rv_r_insn(1, rs2, rs1, 6, rd, 0x33);
452 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
454 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
457 static inline u32 rv_jal(u8 rd, u32 imm20_1)
459 return rv_j_insn(imm20_1, rd, 0x6f);
462 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
464 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
467 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
469 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
472 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
474 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
477 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
479 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
482 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
484 return rv_bltu(rs2, rs1, imm12_1);
487 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
489 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
492 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
494 return rv_bgeu(rs2, rs1, imm12_1);
497 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
499 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
502 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
504 return rv_blt(rs2, rs1, imm12_1);
507 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
509 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
512 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
514 return rv_bge(rs2, rs1, imm12_1);
517 static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1)
519 return rv_i_insn(imm11_0, rs1, 0, rd, 0x03);
522 static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1)
524 return rv_i_insn(imm11_0, rs1, 1, rd, 0x03);
527 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
529 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
532 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
534 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
537 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
539 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
542 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
544 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
547 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
549 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
552 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
554 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
557 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
559 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
562 static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
564 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f);
567 static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
569 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f);
572 static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
574 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f);
577 static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
579 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f);
582 static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
584 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f);
587 static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
589 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f);
592 static inline u32 rv_fence(u8 pred, u8 succ)
594 u16 imm11_0 = pred << 4 | succ;
596 return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
599 static inline u32 rv_nop(void)
601 return rv_i_insn(0, 0, 0, 0, 0x13);
604 /* RVC instrutions. */
606 static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
610 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |
611 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);
612 return rv_ciw_insn(0x0, imm, rd, 0x0);
615 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)
619 imm_hi = (imm7 & 0x38) >> 3;
620 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
621 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0);
624 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)
628 imm_hi = (imm7 & 0x38) >> 3;
629 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
630 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0);
633 static inline u16 rvc_addi(u8 rd, u32 imm6)
635 return rv_ci_insn(0, imm6, rd, 0x1);
638 static inline u16 rvc_li(u8 rd, u32 imm6)
640 return rv_ci_insn(0x2, imm6, rd, 0x1);
643 static inline u16 rvc_addi16sp(u32 imm10)
647 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |
648 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);
649 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1);
652 static inline u16 rvc_lui(u8 rd, u32 imm6)
654 return rv_ci_insn(0x3, imm6, rd, 0x1);
657 static inline u16 rvc_srli(u8 rd, u32 imm6)
659 return rv_cb_insn(0x4, imm6, 0, rd, 0x1);
662 static inline u16 rvc_srai(u8 rd, u32 imm6)
664 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1);
667 static inline u16 rvc_andi(u8 rd, u32 imm6)
669 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1);
672 static inline u16 rvc_sub(u8 rd, u8 rs)
674 return rv_ca_insn(0x23, rd, 0, rs, 0x1);
677 static inline u16 rvc_xor(u8 rd, u8 rs)
679 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1);
682 static inline u16 rvc_or(u8 rd, u8 rs)
684 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1);
687 static inline u16 rvc_and(u8 rd, u8 rs)
689 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1);
692 static inline u16 rvc_slli(u8 rd, u32 imm6)
694 return rv_ci_insn(0, imm6, rd, 0x2);
697 static inline u16 rvc_lwsp(u8 rd, u32 imm8)
701 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);
702 return rv_ci_insn(0x2, imm, rd, 0x2);
705 static inline u16 rvc_jr(u8 rs1)
707 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2);
710 static inline u16 rvc_mv(u8 rd, u8 rs)
712 return rv_cr_insn(0x8, rd, rs, 0x2);
715 static inline u16 rvc_jalr(u8 rs1)
717 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2);
720 static inline u16 rvc_add(u8 rd, u8 rs)
722 return rv_cr_insn(0x9, rd, rs, 0x2);
725 static inline u16 rvc_swsp(u32 imm8, u8 rs2)
729 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);
730 return rv_css_insn(0x6, imm, rs2, 0x2);
734 * RV64-only instructions.
736 * These instructions are not available on RV32. Wrap them below a #if to
737 * ensure that the RV32 JIT doesn't emit any of these instructions.
740 #if __riscv_xlen == 64
742 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
744 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
747 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
749 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
752 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
754 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
757 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
759 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
762 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
764 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
767 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
769 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
772 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
774 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
777 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
779 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
782 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
784 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
787 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
789 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
792 static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2)
794 return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b);
797 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
799 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
802 static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2)
804 return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b);
807 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
809 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
812 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
814 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
817 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
819 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
822 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
824 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
827 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
829 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
832 static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
834 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f);
837 static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
839 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f);
842 static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
844 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f);
847 static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
849 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f);
852 static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
854 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f);
857 static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
859 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f);
862 /* RV64-only RVC instructions. */
864 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)
868 imm_hi = (imm8 & 0x38) >> 3;
869 imm_lo = (imm8 & 0xc0) >> 6;
870 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);
873 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)
877 imm_hi = (imm8 & 0x38) >> 3;
878 imm_lo = (imm8 & 0xc0) >> 6;
879 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);
882 static inline u16 rvc_subw(u8 rd, u8 rs)
884 return rv_ca_insn(0x27, rd, 0, rs, 0x1);
887 static inline u16 rvc_addiw(u8 rd, u32 imm6)
889 return rv_ci_insn(0x1, imm6, rd, 0x1);
892 static inline u16 rvc_ldsp(u8 rd, u32 imm9)
896 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);
897 return rv_ci_insn(0x3, imm, rd, 0x2);
900 static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
904 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);
905 return rv_css_insn(0x7, imm, rs2, 0x2);
908 #endif /* __riscv_xlen == 64 */
910 /* Helper functions that emit RVC instructions when possible. */
912 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
914 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)
915 emitc(rvc_jalr(rs), ctx);
916 else if (rvc_enabled() && !rd && rs && !imm)
917 emitc(rvc_jr(rs), ctx);
919 emit(rv_jalr(rd, rs, imm), ctx);
922 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)
924 if (rvc_enabled() && rd && rs)
925 emitc(rvc_mv(rd, rs), ctx);
927 emit(rv_addi(rd, rs, 0), ctx);
930 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
932 if (rvc_enabled() && rd && rd == rs1 && rs2)
933 emitc(rvc_add(rd, rs2), ctx);
935 emit(rv_add(rd, rs1, rs2), ctx);
938 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
940 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf))
941 emitc(rvc_addi16sp(imm), ctx);
942 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) &&
944 emitc(rvc_addi4spn(rd, imm), ctx);
945 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm))
946 emitc(rvc_addi(rd, imm), ctx);
948 emit(rv_addi(rd, rs, imm), ctx);
951 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)
953 if (rvc_enabled() && rd && is_6b_int(imm))
954 emitc(rvc_li(rd, imm), ctx);
956 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx);
959 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)
961 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm)
962 emitc(rvc_lui(rd, imm), ctx);
964 emit(rv_lui(rd, imm), ctx);
967 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
969 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)
970 emitc(rvc_slli(rd, imm), ctx);
972 emit(rv_slli(rd, rs, imm), ctx);
975 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
977 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm))
978 emitc(rvc_andi(rd, imm), ctx);
980 emit(rv_andi(rd, rs, imm), ctx);
983 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
985 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
986 emitc(rvc_srli(rd, imm), ctx);
988 emit(rv_srli(rd, rs, imm), ctx);
991 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
993 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
994 emitc(rvc_srai(rd, imm), ctx);
996 emit(rv_srai(rd, rs, imm), ctx);
999 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1001 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1002 emitc(rvc_sub(rd, rs2), ctx);
1004 emit(rv_sub(rd, rs1, rs2), ctx);
1007 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1009 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1010 emitc(rvc_or(rd, rs2), ctx);
1012 emit(rv_or(rd, rs1, rs2), ctx);
1015 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1017 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1018 emitc(rvc_and(rd, rs2), ctx);
1020 emit(rv_and(rd, rs1, rs2), ctx);
1023 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1025 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1026 emitc(rvc_xor(rd, rs2), ctx);
1028 emit(rv_xor(rd, rs1, rs2), ctx);
1031 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1033 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3))
1034 emitc(rvc_lwsp(rd, off), ctx);
1035 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3))
1036 emitc(rvc_lw(rd, off, rs1), ctx);
1038 emit(rv_lw(rd, off, rs1), ctx);
1041 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1043 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3))
1044 emitc(rvc_swsp(off, rs2), ctx);
1045 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3))
1046 emitc(rvc_sw(rs1, off, rs2), ctx);
1048 emit(rv_sw(rs1, off, rs2), ctx);
1051 /* RV64-only helper functions. */
1052 #if __riscv_xlen == 64
1054 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1056 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))
1057 emitc(rvc_addiw(rd, imm), ctx);
1059 emit(rv_addiw(rd, rs, imm), ctx);
1062 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1064 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))
1065 emitc(rvc_ldsp(rd, off), ctx);
1066 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))
1067 emitc(rvc_ld(rd, off, rs1), ctx);
1069 emit(rv_ld(rd, off, rs1), ctx);
1072 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1074 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))
1075 emitc(rvc_sdsp(off, rs2), ctx);
1076 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))
1077 emitc(rvc_sd(rs1, off, rs2), ctx);
1079 emit(rv_sd(rs1, off, rs2), ctx);
1082 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1084 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1085 emitc(rvc_subw(rd, rs2), ctx);
1087 emit(rv_subw(rd, rs1, rs2), ctx);
1090 #endif /* __riscv_xlen == 64 */
1092 void bpf_jit_build_prologue(struct rv_jit_context *ctx);
1093 void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
1095 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1098 #endif /* _BPF_JIT_H */