Google Git
Sign in
zircon-guest/third_party/linux/refs/heads/master/./arch/loongarch/net/bpf_jit.c
blob: 24913dc7f4e835ebb5630f2f19302dcff5b5b89a [file] [edit]
// SPDX-License-Identifier: GPL-2.0-only
/*
* BPF JIT compiler for LoongArch
*
* Copyright (C) 2022 Loongson Technology Corporation Limited
*/
#include <linux/memory.h>
#include "bpf_jit.h"
#define LOONGARCH_MAX_REG_ARGS 8
#define LOONGARCH_LONG_JUMP_NINSNS 5
#define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
#define LOONGARCH_FENTRY_NINSNS 2
#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4)
#define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
#define REG_TCC LOONGARCH_GPR_A6
#define REG_ARENA LOONGARCH_GPR_S6 /* For storing arena_vm_start */
#define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80)
static const int regmap[] = {
/* return value from in-kernel function, and exit value for eBPF program */
[BPF_REG_0] = LOONGARCH_GPR_A5,
/* arguments from eBPF program to in-kernel function */
[BPF_REG_1] = LOONGARCH_GPR_A0,
[BPF_REG_2] = LOONGARCH_GPR_A1,
[BPF_REG_3] = LOONGARCH_GPR_A2,
[BPF_REG_4] = LOONGARCH_GPR_A3,
[BPF_REG_5] = LOONGARCH_GPR_A4,
/* callee saved registers that in-kernel function will preserve */
[BPF_REG_6] = LOONGARCH_GPR_S0,
[BPF_REG_7] = LOONGARCH_GPR_S1,
[BPF_REG_8] = LOONGARCH_GPR_S2,
[BPF_REG_9] = LOONGARCH_GPR_S3,
/* read-only frame pointer to access stack */
[BPF_REG_FP] = LOONGARCH_GPR_S4,
/* temporary register for blinding constants */
[BPF_REG_AX] = LOONGARCH_GPR_T0,
};
static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
{
const struct bpf_prog *prog = ctx->prog;
const bool is_main_prog = !bpf_is_subprog(prog);
if (is_main_prog) {
/*
* LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT
* if (REG_TCC > T3 )
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
* else
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
* REG_TCC = LOONGARCH_GPR_SP + store_offset
*
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
*
* The purpose of this code is to first push the TCC into stack,
* and then push the address of TCC into stack.
* In cases where bpf2bpf and tailcall are used in combination,
* the value in REG_TCC may be a count or an address,
* these two cases need to be judged and handled separately.
*/
emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
*store_offset -= sizeof(long);
emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4);
/*
* If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count,
* push tcc into stack
*/
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
/* Push the address of TCC into the REG_TCC */
emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
emit_uncond_jmp(ctx, 2);
/*
* If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address,
* push tcc_ptr into stack
*/
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
} else {
*store_offset -= sizeof(long);
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
}
/* Push tcc_ptr into stack */
*store_offset -= sizeof(long);
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
}
/*
* eBPF prog stack layout:
*
* high
* original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
* | $ra |
* +-------------------------+
* | $fp |
* +-------------------------+
* | $s0 |
* +-------------------------+
* | $s1 |
* +-------------------------+
* | $s2 |
* +-------------------------+
* | $s3 |
* +-------------------------+
* | $s4 |
* +-------------------------+
* | $s5 |
* +-------------------------+
* | tcc |
* +-------------------------+
* | tcc_ptr |
* +-------------------------+ <--BPF_REG_FP
* | prog->aux->stack_depth |
* | (optional) |
* current $sp -------------> +-------------------------+
* low
*/
static void build_prologue(struct jit_ctx *ctx)
{
int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
const struct bpf_prog *prog = ctx->prog;
const bool is_main_prog = !bpf_is_subprog(prog);
bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
/* To store ra, fp, s0, s1, s2, s3, s4, s5 */
stack_adjust += sizeof(long) * 8;
/* To store tcc and tcc_ptr */
stack_adjust += sizeof(long) * 2;
if (ctx->arena_vm_start)
stack_adjust += 8;
stack_adjust = round_up(stack_adjust, 16);
stack_adjust += bpf_stack_adjust;
move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA);
/* Reserve space for the move_imm + jirl instruction */
for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
emit_insn(ctx, nop);
/*
* First instruction initializes the tail call count (TCC)
* register to zero. On tail call we skip this instruction,
* and the TCC is passed in REG_TCC from the caller.
*/
if (is_main_prog)
emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
store_offset = stack_adjust - sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
if (ctx->arena_vm_start) {
store_offset -= sizeof(long);
emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset);
}
prepare_bpf_tail_call_cnt(ctx, &store_offset);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
if (bpf_stack_adjust)
emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
ctx->stack_size = stack_adjust;
if (ctx->arena_vm_start)
move_imm(ctx, REG_ARENA, ctx->arena_vm_start, false);
}
static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
{
int stack_adjust = ctx->stack_size;
int load_offset;
load_offset = stack_adjust - sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
if (ctx->arena_vm_start) {
load_offset -= sizeof(long);
emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset);
}
/*
* When push into the stack, follow the order of tcc then tcc_ptr.
* When pop from the stack, first pop tcc_ptr then followed by tcc.
*/
load_offset -= 2 * sizeof(long);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
load_offset += sizeof(long);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
if (!is_tail_call) {
/* Set return value */
emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0);
/* Return to the caller */
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
} else {
/*
* Call the next bpf prog and skip the first instruction
* of TCC initialization.
*/
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 7);
}
}
static void build_epilogue(struct jit_ctx *ctx)
{
__build_epilogue(ctx, false);
}
bool bpf_jit_supports_kfunc_call(void)
{
return true;
}
bool bpf_jit_supports_far_kfunc_call(void)
{
return true;
}
static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
{
int off, tc_ninsn = 0;
int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
u8 a1 = LOONGARCH_GPR_A1;
u8 a2 = LOONGARCH_GPR_A2;
u8 t1 = LOONGARCH_GPR_T1;
u8 t2 = LOONGARCH_GPR_T2;
u8 t3 = LOONGARCH_GPR_T3;
const int idx0 = ctx->idx;
#define cur_offset (ctx->idx - idx0)
#define jmp_offset (tc_ninsn - (cur_offset))
/*
* a0: &ctx
* a1: &array
* a2: index
*
* if (index >= array->map.max_entries)
* goto out;
*/
tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0];
emit_zext_32(ctx, a2, true);
off = offsetof(struct bpf_array, map.max_entries);
emit_insn(ctx, ldwu, t1, a1, off);
/* bgeu $a2, $t1, jmp_offset */
if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
goto toofar;
/*
* if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
* goto out;
*/
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
emit_insn(ctx, ldd, t3, REG_TCC, 0);
emit_insn(ctx, addid, t3, t3, 1);
emit_insn(ctx, std, t3, REG_TCC, 0);
emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0)
goto toofar;
/*
* prog = array->ptrs[index];
* if (!prog)
* goto out;
*/
emit_insn(ctx, alsld, t2, a2, a1, 2);
off = offsetof(struct bpf_array, ptrs);
emit_insn(ctx, ldd, t2, t2, off);
/* beq $t2, $zero, jmp_offset */
if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
goto toofar;
/* goto *(prog->bpf_func + 4); */
off = offsetof(struct bpf_prog, bpf_func);
emit_insn(ctx, ldd, t3, t2, off);
__build_epilogue(ctx, true);
return 0;
toofar:
pr_info_once("tail_call: jump too far\n");
return -1;
#undef cur_offset
#undef jmp_offset
}
static void emit_store_stack_imm64(struct jit_ctx *ctx, int reg, int stack_off, u64 imm64)
{
move_imm(ctx, reg, imm64, false);
emit_insn(ctx, std, reg, LOONGARCH_GPR_FP, stack_off);
}
static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
{
const u8 t1 = LOONGARCH_GPR_T1;
const u8 t2 = LOONGARCH_GPR_T2;
const u8 t3 = LOONGARCH_GPR_T3;
const u8 r0 = regmap[BPF_REG_0];
const u8 src = regmap[insn->src_reg];
const u8 dst = regmap[insn->dst_reg];
const s16 off = insn->off;
const s32 imm = insn->imm;
const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
move_imm(ctx, t1, off, false);
emit_insn(ctx, addd, t1, dst, t1);
move_reg(ctx, t3, src);
switch (imm) {
/* lock *(size *)(dst + off) <op>= src */
case BPF_ADD:
switch (BPF_SIZE(insn->code)) {
case BPF_B:
if (!cpu_has_lam_bh) {
pr_err_once("bpf-jit: amadd.b instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amaddb, t2, t1, src);
break;
case BPF_H:
if (!cpu_has_lam_bh) {
pr_err_once("bpf-jit: amadd.h instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amaddh, t2, t1, src);
break;
case BPF_W:
emit_insn(ctx, amaddw, t2, t1, src);
break;
case BPF_DW:
emit_insn(ctx, amaddd, t2, t1, src);
break;
}
break;
case BPF_AND:
if (isdw)
emit_insn(ctx, amandd, t2, t1, src);
else
emit_insn(ctx, amandw, t2, t1, src);
break;
case BPF_OR:
if (isdw)
emit_insn(ctx, amord, t2, t1, src);
else
emit_insn(ctx, amorw, t2, t1, src);
break;
case BPF_XOR:
if (isdw)
emit_insn(ctx, amxord, t2, t1, src);
else
emit_insn(ctx, amxorw, t2, t1, src);
break;
/* src = atomic_fetch_<op>(dst + off, src) */
case BPF_ADD | BPF_FETCH:
switch (BPF_SIZE(insn->code)) {
case BPF_B:
if (!cpu_has_lam_bh) {
pr_err_once("bpf-jit: amadd.b instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amaddb, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_H:
if (!cpu_has_lam_bh) {
pr_err_once("bpf-jit: amadd.h instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amaddh, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_W:
emit_insn(ctx, amaddw, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_DW:
emit_insn(ctx, amaddd, src, t1, t3);
break;
}
break;
case BPF_AND | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amandd, src, t1, t3);
} else {
emit_insn(ctx, amandw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
case BPF_OR | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amord, src, t1, t3);
} else {
emit_insn(ctx, amorw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
case BPF_XOR | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amxord, src, t1, t3);
} else {
emit_insn(ctx, amxorw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
/* src = atomic_xchg(dst + off, src); */
case BPF_XCHG:
switch (BPF_SIZE(insn->code)) {
case BPF_B:
if (!cpu_has_lam_bh) {
pr_err_once("bpf-jit: amswap.b instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amswapb, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_H:
if (!cpu_has_lam_bh) {
pr_err_once("bpf-jit: amswap.h instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amswaph, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_W:
emit_insn(ctx, amswapw, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_DW:
emit_insn(ctx, amswapd, src, t1, t3);
break;
}
break;
/* r0 = atomic_cmpxchg(dst + off, r0, src); */
case BPF_CMPXCHG:
move_reg(ctx, t2, r0);
if (isdw) {
emit_insn(ctx, lld, r0, t1, 0);
emit_insn(ctx, bne, t2, r0, 4);
move_reg(ctx, t3, src);
emit_insn(ctx, scd, t3, t1, 0);
emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
} else {
emit_insn(ctx, llw, r0, t1, 0);
emit_zext_32(ctx, t2, true);
emit_zext_32(ctx, r0, true);
emit_insn(ctx, bne, t2, r0, 4);
move_reg(ctx, t3, src);
emit_insn(ctx, scw, t3, t1, 0);
emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
emit_zext_32(ctx, r0, true);
}
break;
default:
pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm);
return -EINVAL;
}
return 0;
}
static int emit_atomic_ld_st(const struct bpf_insn *insn, struct jit_ctx *ctx)
{
const u8 t1 = LOONGARCH_GPR_T1;
const u8 src = regmap[insn->src_reg];
const u8 dst = regmap[insn->dst_reg];
const s16 off = insn->off;
const s32 imm = insn->imm;
switch (imm) {
/* dst_reg = load_acquire(src_reg + off16) */
case BPF_LOAD_ACQ:
switch (BPF_SIZE(insn->code)) {
case BPF_B:
if (is_signed_imm12(off)) {
emit_insn(ctx, ldbu, dst, src, off);
} else {
move_imm(ctx, t1, off, false);
emit_insn(ctx, ldxbu, dst, src, t1);
}
break;
case BPF_H:
if (is_signed_imm12(off)) {
emit_insn(ctx, ldhu, dst, src, off);
} else {
move_imm(ctx, t1, off, false);
emit_insn(ctx, ldxhu, dst, src, t1);
}
break;
case BPF_W:
if (is_signed_imm12(off)) {
emit_insn(ctx, ldwu, dst, src, off);
} else {
move_imm(ctx, t1, off, false);
emit_insn(ctx, ldxwu, dst, src, t1);
}
break;
case BPF_DW:
if (is_signed_imm12(off)) {
emit_insn(ctx, ldd, dst, src, off);
} else {
move_imm(ctx, t1, off, false);
emit_insn(ctx, ldxd, dst, src, t1);
}
break;
}
emit_insn(ctx, dbar, 0b10100);
break;
/* store_release(dst_reg + off16, src_reg) */
case BPF_STORE_REL:
emit_insn(ctx, dbar, 0b10010);
switch (BPF_SIZE(insn->code)) {
case BPF_B:
if (is_signed_imm12(off)) {
emit_insn(ctx, stb, src, dst, off);
} else {
move_imm(ctx, t1, off, false);
emit_insn(ctx, stxb, src, dst, t1);
}
break;
case BPF_H:
if (is_signed_imm12(off)) {
emit_insn(ctx, sth, src, dst, off);
} else {
move_imm(ctx, t1, off, false);
emit_insn(ctx, stxh, src, dst, t1);
}
break;
case BPF_W:
if (is_signed_imm12(off)) {
emit_insn(ctx, stw, src, dst, off);
} else {
move_imm(ctx, t1, off, false);
emit_insn(ctx, stxw, src, dst, t1);
}
break;
case BPF_DW:
if (is_signed_imm12(off)) {
emit_insn(ctx, std, src, dst, off);
} else {
move_imm(ctx, t1, off, false);
emit_insn(ctx, stxd, src, dst, t1);
}
break;
}
break;
default:
pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm);
return -EINVAL;
}
return 0;
}
static bool is_signed_bpf_cond(u8 cond)
{
return cond == BPF_JSGT || cond == BPF_JSLT ||
cond == BPF_JSGE || cond == BPF_JSLE;
}
#define BPF_FIXUP_REG_MASK GENMASK(31, 27)
#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
#define REG_DONT_CLEAR_MARKER 0
bool ex_handler_bpf(const struct exception_table_entry *ex,
struct pt_regs *regs)
{
int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
if (dst_reg != REG_DONT_CLEAR_MARKER)
regs->regs[dst_reg] = 0;
regs->csr_era = (unsigned long)&ex->fixup - offset;
return true;
}
/* For accesses to BTF pointers, add an entry to the exception table */
static int add_exception_handler(const struct bpf_insn *insn,
struct jit_ctx *ctx,
int dst_reg)
{
unsigned long pc;
off_t ins_offset, fixup_offset;
struct exception_table_entry *ex;
if (!ctx->image || !ctx->ro_image || !ctx->prog->aux->extable)
return 0;
if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
BPF_MODE(insn->code) != BPF_PROBE_MEM32)
return 0;
if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
return -EINVAL;
ex = &ctx->prog->aux->extable[ctx->num_exentries];
pc = (unsigned long)&ctx->ro_image[ctx->idx - 1];
/*
* This is the relative offset of the instruction that may fault from
* the exception table itself. This will be written to the exception
* table and if this instruction faults, the destination register will
* be set to '0' and the execution will jump to the next instruction.
*/
ins_offset = pc - (long)&ex->insn;
if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN))
return -ERANGE;
/*
* Since the extable follows the program, the fixup offset is always
* negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
* to keep things simple, and put the destination register in the upper
* bits. We don't need to worry about buildtime or runtime sort
* modifying the upper bits because the table is already sorted, and
* isn't part of the main exception table.
*
* The fixup_offset is set to the next instruction from the instruction
* that may fault. The execution will jump to this after handling the fault.
*/
fixup_offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset))
return -ERANGE;
/*
* The offsets above have been calculated using the RO buffer but we
* need to use the R/W buffer for writes. Switch ex to rw buffer for writing.
*/
ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image);
ex->insn = ins_offset;
ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) |
FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
ex->type = EX_TYPE_BPF;
ctx->num_exentries++;
return 0;
}
static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
{
u8 tm = -1;
u64 func_addr;
bool func_addr_fixed, sign_extend;
int i = insn - ctx->prog->insnsi;
int ret, jmp_offset, tcc_ptr_off;
const u8 code = insn->code;
const u8 cond = BPF_OP(code);
const u8 t1 = LOONGARCH_GPR_T1;
const u8 t2 = LOONGARCH_GPR_T2;
const u8 t3 = LOONGARCH_GPR_T3;
u8 src = regmap[insn->src_reg];
u8 dst = regmap[insn->dst_reg];
const s16 off = insn->off;
const s32 imm = insn->imm;
const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
switch (code) {
/* dst = src */
case BPF_ALU | BPF_MOV | BPF_X:
case BPF_ALU64 | BPF_MOV | BPF_X:
if (insn_is_cast_user(insn)) {
move_reg(ctx, t1, src);
emit_zext_32(ctx, t1, true);
move_imm(ctx, dst, (ctx->user_vm_start >> 32) << 32, false);
emit_insn(ctx, beq, t1, LOONGARCH_GPR_ZERO, 1);
emit_insn(ctx, or, t1, dst, t1);
move_reg(ctx, dst, t1);
break;
}
switch (off) {
case 0:
move_reg(ctx, dst, src);
emit_zext_32(ctx, dst, is32);
break;
case 8:
emit_insn(ctx, extwb, dst, src);
emit_zext_32(ctx, dst, is32);
break;
case 16:
emit_insn(ctx, extwh, dst, src);
emit_zext_32(ctx, dst, is32);
break;
case 32:
emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
break;
}
break;
/* dst = imm */
case BPF_ALU | BPF_MOV | BPF_K:
case BPF_ALU64 | BPF_MOV | BPF_K:
move_imm(ctx, dst, imm, is32);
break;
/* dst = dst + src */
case BPF_ALU | BPF_ADD | BPF_X:
case BPF_ALU64 | BPF_ADD | BPF_X:
emit_insn(ctx, addd, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst + imm */
case BPF_ALU | BPF_ADD | BPF_K:
case BPF_ALU64 | BPF_ADD | BPF_K:
if (is_signed_imm12(imm)) {
emit_insn(ctx, addid, dst, dst, imm);
} else {
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, addd, dst, dst, t1);
}
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst - src */
case BPF_ALU | BPF_SUB | BPF_X:
case BPF_ALU64 | BPF_SUB | BPF_X:
emit_insn(ctx, subd, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst - imm */
case BPF_ALU | BPF_SUB | BPF_K:
case BPF_ALU64 | BPF_SUB | BPF_K:
if (is_signed_imm12(-imm)) {
emit_insn(ctx, addid, dst, dst, -imm);
} else {
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, subd, dst, dst, t1);
}
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst * src */
case BPF_ALU | BPF_MUL | BPF_X:
case BPF_ALU64 | BPF_MUL | BPF_X:
emit_insn(ctx, muld, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst * imm */
case BPF_ALU | BPF_MUL | BPF_K:
case BPF_ALU64 | BPF_MUL | BPF_K:
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, muld, dst, dst, t1);
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst / src */
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU64 | BPF_DIV | BPF_X:
if (!off) {
emit_zext_32(ctx, dst, is32);
move_reg(ctx, t1, src);
emit_zext_32(ctx, t1, is32);
emit_insn(ctx, divdu, dst, dst, t1);
emit_zext_32(ctx, dst, is32);
} else {
emit_sext_32(ctx, dst, is32);
move_reg(ctx, t1, src);
emit_sext_32(ctx, t1, is32);
emit_insn(ctx, divd, dst, dst, t1);
emit_sext_32(ctx, dst, is32);
}
break;
/* dst = dst / imm */
case BPF_ALU | BPF_DIV | BPF_K:
case BPF_ALU64 | BPF_DIV | BPF_K:
if (!off) {
move_imm(ctx, t1, imm, is32);
emit_zext_32(ctx, dst, is32);
emit_insn(ctx, divdu, dst, dst, t1);
emit_zext_32(ctx, dst, is32);
} else {
move_imm(ctx, t1, imm, false);
emit_sext_32(ctx, t1, is32);
emit_sext_32(ctx, dst, is32);
emit_insn(ctx, divd, dst, dst, t1);
emit_sext_32(ctx, dst, is32);
}
break;
/* dst = dst % src */
case BPF_ALU | BPF_MOD | BPF_X:
case BPF_ALU64 | BPF_MOD | BPF_X:
if (!off) {
emit_zext_32(ctx, dst, is32);
move_reg(ctx, t1, src);
emit_zext_32(ctx, t1, is32);
emit_insn(ctx, moddu, dst, dst, t1);
emit_zext_32(ctx, dst, is32);
} else {
emit_sext_32(ctx, dst, is32);
move_reg(ctx, t1, src);
emit_sext_32(ctx, t1, is32);
emit_insn(ctx, modd, dst, dst, t1);
emit_sext_32(ctx, dst, is32);
}
break;
/* dst = dst % imm */
case BPF_ALU | BPF_MOD | BPF_K:
case BPF_ALU64 | BPF_MOD | BPF_K:
if (!off) {
move_imm(ctx, t1, imm, is32);
emit_zext_32(ctx, dst, is32);
emit_insn(ctx, moddu, dst, dst, t1);
emit_zext_32(ctx, dst, is32);
} else {
move_imm(ctx, t1, imm, false);
emit_sext_32(ctx, t1, is32);
emit_sext_32(ctx, dst, is32);
emit_insn(ctx, modd, dst, dst, t1);
emit_sext_32(ctx, dst, is32);
}
break;
/* dst = -dst */
case BPF_ALU | BPF_NEG:
case BPF_ALU64 | BPF_NEG:
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst & src */
case BPF_ALU | BPF_AND | BPF_X:
case BPF_ALU64 | BPF_AND | BPF_X:
emit_insn(ctx, and, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst & imm */
case BPF_ALU | BPF_AND | BPF_K:
case BPF_ALU64 | BPF_AND | BPF_K:
if (is_unsigned_imm12(imm)) {
emit_insn(ctx, andi, dst, dst, imm);
} else {
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, and, dst, dst, t1);
}
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst | src */
case BPF_ALU | BPF_OR | BPF_X:
case BPF_ALU64 | BPF_OR | BPF_X:
emit_insn(ctx, or, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst | imm */
case BPF_ALU | BPF_OR | BPF_K:
case BPF_ALU64 | BPF_OR | BPF_K:
if (is_unsigned_imm12(imm)) {
emit_insn(ctx, ori, dst, dst, imm);
} else {
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, or, dst, dst, t1);
}
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst ^ src */
case BPF_ALU | BPF_XOR | BPF_X:
case BPF_ALU64 | BPF_XOR | BPF_X:
emit_insn(ctx, xor, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst ^ imm */
case BPF_ALU | BPF_XOR | BPF_K:
case BPF_ALU64 | BPF_XOR | BPF_K:
if (is_unsigned_imm12(imm)) {
emit_insn(ctx, xori, dst, dst, imm);
} else {
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, xor, dst, dst, t1);
}
emit_zext_32(ctx, dst, is32);
break;
/* dst = dst << src (logical) */
case BPF_ALU | BPF_LSH | BPF_X:
emit_insn(ctx, sllw, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
case BPF_ALU64 | BPF_LSH | BPF_X:
emit_insn(ctx, slld, dst, dst, src);
break;
/* dst = dst << imm (logical) */
case BPF_ALU | BPF_LSH | BPF_K:
emit_insn(ctx, slliw, dst, dst, imm);
emit_zext_32(ctx, dst, is32);
break;
case BPF_ALU64 | BPF_LSH | BPF_K:
emit_insn(ctx, sllid, dst, dst, imm);
break;
/* dst = dst >> src (logical) */
case BPF_ALU | BPF_RSH | BPF_X:
emit_insn(ctx, srlw, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
case BPF_ALU64 | BPF_RSH | BPF_X:
emit_insn(ctx, srld, dst, dst, src);
break;
/* dst = dst >> imm (logical) */
case BPF_ALU | BPF_RSH | BPF_K:
emit_insn(ctx, srliw, dst, dst, imm);
emit_zext_32(ctx, dst, is32);
break;
case BPF_ALU64 | BPF_RSH | BPF_K:
emit_insn(ctx, srlid, dst, dst, imm);
break;
/* dst = dst >> src (arithmetic) */
case BPF_ALU | BPF_ARSH | BPF_X:
emit_insn(ctx, sraw, dst, dst, src);
emit_zext_32(ctx, dst, is32);
break;
case BPF_ALU64 | BPF_ARSH | BPF_X:
emit_insn(ctx, srad, dst, dst, src);
break;
/* dst = dst >> imm (arithmetic) */
case BPF_ALU | BPF_ARSH | BPF_K:
emit_insn(ctx, sraiw, dst, dst, imm);
emit_zext_32(ctx, dst, is32);
break;
case BPF_ALU64 | BPF_ARSH | BPF_K:
emit_insn(ctx, sraid, dst, dst, imm);
break;
/* dst = BSWAP##imm(dst) */
case BPF_ALU | BPF_END | BPF_FROM_LE:
switch (imm) {
case 16:
/* zero-extend 16 bits into 64 bits */
emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
break;
case 32:
/* zero-extend 32 bits into 64 bits */
emit_zext_32(ctx, dst, is32);
break;
case 64:
/* do nothing */
break;
}
break;
case BPF_ALU | BPF_END | BPF_FROM_BE:
case BPF_ALU64 | BPF_END | BPF_FROM_LE:
switch (imm) {
case 16:
emit_insn(ctx, revb2h, dst, dst);
/* zero-extend 16 bits into 64 bits */
emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
break;
case 32:
emit_insn(ctx, revb2w, dst, dst);
/* clear the upper 32 bits */
emit_zext_32(ctx, dst, true);
break;
case 64:
emit_insn(ctx, revbd, dst, dst);
break;
}
break;
/* PC += off if dst cond src */
case BPF_JMP | BPF_JEQ | BPF_X:
case BPF_JMP | BPF_JNE | BPF_X:
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JLT | BPF_X:
case BPF_JMP | BPF_JLE | BPF_X:
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP | BPF_JSGE | BPF_X:
case BPF_JMP | BPF_JSLT | BPF_X:
case BPF_JMP | BPF_JSLE | BPF_X:
case BPF_JMP32 | BPF_JEQ | BPF_X:
case BPF_JMP32 | BPF_JNE | BPF_X:
case BPF_JMP32 | BPF_JGT | BPF_X:
case BPF_JMP32 | BPF_JGE | BPF_X:
case BPF_JMP32 | BPF_JLT | BPF_X:
case BPF_JMP32 | BPF_JLE | BPF_X:
case BPF_JMP32 | BPF_JSGT | BPF_X:
case BPF_JMP32 | BPF_JSGE | BPF_X:
case BPF_JMP32 | BPF_JSLT | BPF_X:
case BPF_JMP32 | BPF_JSLE | BPF_X:
jmp_offset = bpf2la_offset(i, off, ctx);
move_reg(ctx, t1, dst);
move_reg(ctx, t2, src);
if (is_signed_bpf_cond(BPF_OP(code))) {
emit_sext_32(ctx, t1, is32);
emit_sext_32(ctx, t2, is32);
} else {
emit_zext_32(ctx, t1, is32);
emit_zext_32(ctx, t2, is32);
}
if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
goto toofar;
break;
/* PC += off if dst cond imm */
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JLT | BPF_K:
case BPF_JMP | BPF_JLE | BPF_K:
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_K:
case BPF_JMP | BPF_JSLT | BPF_K:
case BPF_JMP | BPF_JSLE | BPF_K:
case BPF_JMP32 | BPF_JEQ | BPF_K:
case BPF_JMP32 | BPF_JNE | BPF_K:
case BPF_JMP32 | BPF_JGT | BPF_K:
case BPF_JMP32 | BPF_JGE | BPF_K:
case BPF_JMP32 | BPF_JLT | BPF_K:
case BPF_JMP32 | BPF_JLE | BPF_K:
case BPF_JMP32 | BPF_JSGT | BPF_K:
case BPF_JMP32 | BPF_JSGE | BPF_K:
case BPF_JMP32 | BPF_JSLT | BPF_K:
case BPF_JMP32 | BPF_JSLE | BPF_K:
jmp_offset = bpf2la_offset(i, off, ctx);
if (imm) {
move_imm(ctx, t1, imm, false);
tm = t1;
} else {
/* If imm is 0, simply use zero register. */
tm = LOONGARCH_GPR_ZERO;
}
move_reg(ctx, t2, dst);
if (is_signed_bpf_cond(BPF_OP(code))) {
emit_sext_32(ctx, tm, is32);
emit_sext_32(ctx, t2, is32);
} else {
emit_zext_32(ctx, tm, is32);
emit_zext_32(ctx, t2, is32);
}
if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
goto toofar;
break;
/* PC += off if dst & src */
case BPF_JMP | BPF_JSET | BPF_X:
case BPF_JMP32 | BPF_JSET | BPF_X:
jmp_offset = bpf2la_offset(i, off, ctx);
emit_insn(ctx, and, t1, dst, src);
emit_zext_32(ctx, t1, is32);
if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
goto toofar;
break;
/* PC += off if dst & imm */
case BPF_JMP | BPF_JSET | BPF_K:
case BPF_JMP32 | BPF_JSET | BPF_K:
jmp_offset = bpf2la_offset(i, off, ctx);
move_imm(ctx, t1, imm, is32);
emit_insn(ctx, and, t1, dst, t1);
emit_zext_32(ctx, t1, is32);
if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
goto toofar;
break;
/* PC += off */
case BPF_JMP | BPF_JA:
case BPF_JMP32 | BPF_JA:
if (BPF_CLASS(code) == BPF_JMP)
jmp_offset = bpf2la_offset(i, off, ctx);
else
jmp_offset = bpf2la_offset(i, imm, ctx);
if (emit_uncond_jmp(ctx, jmp_offset) < 0)
goto toofar;
break;
/* function call */
case BPF_JMP | BPF_CALL:
ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
&func_addr, &func_addr_fixed);
if (ret < 0)
return ret;
if (insn->src_reg == BPF_PSEUDO_CALL) {
tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
}
if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
const struct btf_func_model *m;
int i;
m = bpf_jit_find_kfunc_model(ctx->prog, insn);
if (!m)
return -EINVAL;
for (i = 0; i < m->nr_args; i++) {
u8 reg = regmap[BPF_REG_1 + i];
bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG;
emit_abi_ext(ctx, reg, m->arg_size[i], sign);
}
}
move_addr(ctx, t1, func_addr);
emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0);
if (insn->src_reg != BPF_PSEUDO_CALL)
move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
break;
/* tail call */
case BPF_JMP | BPF_TAIL_CALL:
if (emit_bpf_tail_call(ctx, i) < 0)
return -EINVAL;
break;
/* function return */
case BPF_JMP | BPF_EXIT:
if (i == ctx->prog->len - 1)
break;
jmp_offset = epilogue_offset(ctx);
if (emit_uncond_jmp(ctx, jmp_offset) < 0)
goto toofar;
break;
/* dst = imm64 */
case BPF_LD | BPF_IMM | BPF_DW:
{
const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
if (bpf_pseudo_func(insn))
move_addr(ctx, dst, imm64);
else
move_imm(ctx, dst, imm64, is32);
return 1;
}
/* dst = *(size *)(src + off) */
case BPF_LDX | BPF_MEM | BPF_B:
case BPF_LDX | BPF_MEM | BPF_H:
case BPF_LDX | BPF_MEM | BPF_W:
case BPF_LDX | BPF_MEM | BPF_DW:
case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
case BPF_LDX | BPF_PROBE_MEM | BPF_B:
/* dst_reg = (s64)*(signed size *)(src_reg + off) */
case BPF_LDX | BPF_MEMSX | BPF_B:
case BPF_LDX | BPF_MEMSX | BPF_H:
case BPF_LDX | BPF_MEMSX | BPF_W:
case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
/* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + REG_ARENA + off) */
case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
sign_extend = BPF_MODE(code) == BPF_MEMSX ||
BPF_MODE(code) == BPF_PROBE_MEMSX;
if (BPF_MODE(code) == BPF_PROBE_MEM32) {
emit_insn(ctx, addd, t2, src, REG_ARENA);
src = t2;
}
switch (BPF_SIZE(code)) {
case BPF_B:
if (is_signed_imm12(off)) {
if (sign_extend)
emit_insn(ctx, ldb, dst, src, off);
else
emit_insn(ctx, ldbu, dst, src, off);
} else {
move_imm(ctx, t1, off, is32);
if (sign_extend)
emit_insn(ctx, ldxb, dst, src, t1);
else
emit_insn(ctx, ldxbu, dst, src, t1);
}
break;
case BPF_H:
if (is_signed_imm12(off)) {
if (sign_extend)
emit_insn(ctx, ldh, dst, src, off);
else
emit_insn(ctx, ldhu, dst, src, off);
} else {
move_imm(ctx, t1, off, is32);
if (sign_extend)
emit_insn(ctx, ldxh, dst, src, t1);
else
emit_insn(ctx, ldxhu, dst, src, t1);
}
break;
case BPF_W:
if (is_signed_imm12(off)) {
if (sign_extend)
emit_insn(ctx, ldw, dst, src, off);
else
emit_insn(ctx, ldwu, dst, src, off);
} else {
move_imm(ctx, t1, off, is32);
if (sign_extend)
emit_insn(ctx, ldxw, dst, src, t1);
else
emit_insn(ctx, ldxwu, dst, src, t1);
}
break;
case BPF_DW:
move_imm(ctx, t1, off, is32);
emit_insn(ctx, ldxd, dst, src, t1);
break;
}
ret = add_exception_handler(insn, ctx, dst);
if (ret)
return ret;
break;
/* *(size *)(dst + off) = imm */
case BPF_ST | BPF_MEM | BPF_B:
case BPF_ST | BPF_MEM | BPF_H:
case BPF_ST | BPF_MEM | BPF_W:
case BPF_ST | BPF_MEM | BPF_DW:
/* ST | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = imm */
case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:
if (BPF_MODE(code) == BPF_PROBE_MEM32) {
emit_insn(ctx, addd, t3, dst, REG_ARENA);
dst = t3;
}
switch (BPF_SIZE(code)) {
case BPF_B:
move_imm(ctx, t1, imm, is32);
if (is_signed_imm12(off)) {
emit_insn(ctx, stb, t1, dst, off);
} else {
move_imm(ctx, t2, off, is32);
emit_insn(ctx, stxb, t1, dst, t2);
}
break;
case BPF_H:
move_imm(ctx, t1, imm, is32);
if (is_signed_imm12(off)) {
emit_insn(ctx, sth, t1, dst, off);
} else {
move_imm(ctx, t2, off, is32);
emit_insn(ctx, stxh, t1, dst, t2);
}
break;
case BPF_W:
move_imm(ctx, t1, imm, is32);
if (is_signed_imm12(off)) {
emit_insn(ctx, stw, t1, dst, off);
} else if (is_signed_imm14(off)) {
emit_insn(ctx, stptrw, t1, dst, off);
} else {
move_imm(ctx, t2, off, is32);
emit_insn(ctx, stxw, t1, dst, t2);
}
break;
case BPF_DW:
move_imm(ctx, t1, imm, is32);
if (is_signed_imm12(off)) {
emit_insn(ctx, std, t1, dst, off);
} else if (is_signed_imm14(off)) {
emit_insn(ctx, stptrd, t1, dst, off);
} else {
move_imm(ctx, t2, off, is32);
emit_insn(ctx, stxd, t1, dst, t2);
}
break;
}
ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER);
if (ret)
return ret;
break;
/* *(size *)(dst + off) = src */
case BPF_STX | BPF_MEM | BPF_B:
case BPF_STX | BPF_MEM | BPF_H:
case BPF_STX | BPF_MEM | BPF_W:
case BPF_STX | BPF_MEM | BPF_DW:
/* STX | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = src */
case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
if (BPF_MODE(code) == BPF_PROBE_MEM32) {
emit_insn(ctx, addd, t2, dst, REG_ARENA);
dst = t2;
}
switch (BPF_SIZE(code)) {
case BPF_B:
if (is_signed_imm12(off)) {
emit_insn(ctx, stb, src, dst, off);
} else {
move_imm(ctx, t1, off, is32);
emit_insn(ctx, stxb, src, dst, t1);
}
break;
case BPF_H:
if (is_signed_imm12(off)) {
emit_insn(ctx, sth, src, dst, off);
} else {
move_imm(ctx, t1, off, is32);
emit_insn(ctx, stxh, src, dst, t1);
}
break;
case BPF_W:
if (is_signed_imm12(off)) {
emit_insn(ctx, stw, src, dst, off);
} else if (is_signed_imm14(off)) {
emit_insn(ctx, stptrw, src, dst, off);
} else {
move_imm(ctx, t1, off, is32);
emit_insn(ctx, stxw, src, dst, t1);
}
break;
case BPF_DW:
if (is_signed_imm12(off)) {
emit_insn(ctx, std, src, dst, off);
} else if (is_signed_imm14(off)) {
emit_insn(ctx, stptrd, src, dst, off);
} else {
move_imm(ctx, t1, off, is32);
emit_insn(ctx, stxd, src, dst, t1);
}
break;
}
ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER);
if (ret)
return ret;
break;
/* Atomics */
case BPF_STX | BPF_ATOMIC | BPF_B:
case BPF_STX | BPF_ATOMIC | BPF_H:
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
if (!bpf_atomic_is_load_store(insn))
ret = emit_atomic_rmw(insn, ctx);
else
ret = emit_atomic_ld_st(insn, ctx);
if (ret)
return ret;
break;
/* Speculation barrier */
case BPF_ST | BPF_NOSPEC:
break;
default:
pr_err("bpf_jit: unknown opcode %02x\n", code);
return -EINVAL;
}
return 0;
toofar:
pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
return -E2BIG;
}
static int build_body(struct jit_ctx *ctx, bool extra_pass)
{
int i;
const struct bpf_prog *prog = ctx->prog;
for (i = 0; i < prog->len; i++) {
const struct bpf_insn *insn = &prog->insnsi[i];
int ret;
if (ctx->image == NULL)
ctx->offset[i] = ctx->idx;
ret = build_insn(insn, ctx, extra_pass);
if (ret > 0) {
i++;
if (ctx->image == NULL)
ctx->offset[i] = ctx->idx;
continue;
}
if (ret)
return ret;
}
if (ctx->image == NULL)
ctx->offset[i] = ctx->idx;
return 0;
}
/* Fill space with break instructions */
static void jit_fill_hole(void *area, unsigned int size)
{
u32 *ptr;
/* We are guaranteed to have aligned memory */
for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
*ptr++ = INSN_BREAK;
}
static int validate_code(struct jit_ctx *ctx)
{
int i;
union loongarch_instruction insn;
for (i = 0; i < ctx->idx; i++) {
insn = ctx->image[i];
/* Check INSN_BREAK */
if (insn.word == INSN_BREAK)
return -1;
}
return 0;
}
static int validate_ctx(struct jit_ctx *ctx)
{
if (validate_code(ctx))
return -1;
if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
return -1;
return 0;
}
static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target)
{
if (!target) {
pr_err("bpf_jit: jump target address is error\n");
return -EFAULT;
}
move_imm(ctx, LOONGARCH_GPR_T1, target, false);
emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0);
return 0;
}
static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
{
int i;
struct jit_ctx ctx;
ctx.idx = 0;
ctx.image = (union loongarch_instruction *)insns;
if (!target) {
for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
emit_insn((&ctx), nop);
return 0;
}
return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target);
}
static int emit_call(struct jit_ctx *ctx, u64 addr)
{
return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr);
}
void *bpf_arch_text_copy(void *dst, void *src, size_t len)
{
int ret;
cpus_read_lock();
mutex_lock(&text_mutex);
ret = larch_insn_text_copy(dst, src, len);
mutex_unlock(&text_mutex);
cpus_read_unlock();
return ret ? ERR_PTR(-EINVAL) : dst;
}
int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
enum bpf_text_poke_type new_t, void *old_addr,
void *new_addr)
{
int ret;
bool is_call;
unsigned long size = 0;
unsigned long offset = 0;
void *image = NULL;
char namebuf[KSYM_NAME_LEN];
u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
/* Only poking bpf text is supported. Since kernel function entry
* is set up by ftrace, we rely on ftrace to poke kernel functions.
*/
if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf))
return -ENOTSUPP;
image = ip - offset;
/* zero offset means we're poking bpf prog entry */
if (offset == 0) {
/* skip to the nop instruction in bpf prog entry:
* move t0, ra
* nop
*/
ip = image + LOONGARCH_INSN_SIZE;
}
is_call = old_t == BPF_MOD_CALL;
ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call);
if (ret)
return ret;
if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES))
return -EFAULT;
is_call = new_t == BPF_MOD_CALL;
ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call);
if (ret)
return ret;
cpus_read_lock();
mutex_lock(&text_mutex);
if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES))
ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES);
mutex_unlock(&text_mutex);
cpus_read_unlock();
return ret;
}
int bpf_arch_text_invalidate(void *dst, size_t len)
{
int i;
int ret = 0;
u32 *inst;
inst = kvmalloc(len, GFP_KERNEL);
if (!inst)
return -ENOMEM;
for (i = 0; i < (len / sizeof(u32)); i++)
inst[i] = INSN_BREAK;
cpus_read_lock();
mutex_lock(&text_mutex);
if (larch_insn_text_copy(dst, inst, len))
ret = -EINVAL;
mutex_unlock(&text_mutex);
cpus_read_unlock();
kvfree(inst);
return ret;
}
static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off)
{
int i;
for (i = 0; i < nr_arg_slots; i++) {
if (i < LOONGARCH_MAX_REG_ARGS)
emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
else {
/* Skip slots for T0 and FP of traced function */
emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP,
16 + (i - LOONGARCH_MAX_REG_ARGS) * 8);
emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -args_off);
}
args_off -= 8;
}
}
static void restore_args(struct jit_ctx *ctx, int nr_reg_args, int args_off)
{
int i;
for (i = 0; i < nr_reg_args; i++) {
emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
args_off -= 8;
}
}
static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off, int stk_args_off)
{
int i;
for (i = 0; i < nr_stk_args; i++) {
emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP,
-(args_off - LOONGARCH_MAX_REG_ARGS * 8));
emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -stk_args_off);
args_off -= 8;
stk_args_off -= 8;
}
}
static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
int args_off, int retval_off, int run_ctx_off, bool save_ret)
{
int ret;
u32 *branch;
struct bpf_prog *p = l->link.prog;
int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
if (l->cookie)
emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1,
-run_ctx_off + cookie_off, l->cookie);
else
emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
/* arg1: prog */
move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
/* arg2: &run_ctx */
emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off);
ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p));
if (ret)
return ret;
/* store prog start time */
move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0);
/*
* if (__bpf_prog_enter(prog) == 0)
* goto skip_exec_of_prog;
*/
branch = (u32 *)ctx->image + ctx->idx;
/* nop reserved for conditional jump */
emit_insn(ctx, nop);
/* arg1: &args_off */
emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off);
if (!p->jited)
move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false);
ret = emit_call(ctx, (const u64)p->bpf_func);
if (ret)
return ret;
if (save_ret) {
emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
}
/* update branch with beqz */
if (ctx->image) {
int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch;
*branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset);
}
/* arg1: prog */
move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
/* arg2: prog start time */
move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1);
/* arg3: &run_ctx */
emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p));
return ret;
}
static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
int args_off, int retval_off, int run_ctx_off,
int func_meta_off, bool save_ret, u64 func_meta, int cookie_off)
{
int i, cur_cookie = (cookie_off - args_off) / 8;
for (i = 0; i < tl->nr_links; i++) {
int err;
if (bpf_prog_calls_session_cookie(tl->links[i])) {
u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT);
emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, meta);
cur_cookie--;
}
err = invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, run_ctx_off, save_ret);
if (err)
return err;
}
return 0;
}
void *arch_alloc_bpf_trampoline(unsigned int size)
{
return bpf_prog_pack_alloc(size, jit_fill_hole);
}
void arch_free_bpf_trampoline(void *image, unsigned int size)
{
bpf_prog_pack_free(image, size);
}
int arch_protect_bpf_trampoline(void *image, unsigned int size)
{
return 0;
}
/*
* Sign-extend the register if necessary
*/
static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign)
{
/* ABI requires unsigned char/short to be zero-extended */
if (!sign && (size == 1 || size == 2)) {
if (rd != rj)
move_reg(ctx, rd, rj);
return;
}
switch (size) {
case 1:
emit_insn(ctx, extwb, rd, rj);
break;
case 2:
emit_insn(ctx, extwh, rd, rj);
break;
case 4:
emit_insn(ctx, addiw, rd, rj, 0);
break;
case 8:
if (rd != rj)
move_reg(ctx, rd, rj);
break;
default:
pr_warn("bpf_jit: invalid size %d for sign_extend\n", size);
}
}
static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
const struct btf_func_model *m, struct bpf_tramp_links *tlinks,
void *func_addr, u32 flags)
{
int i, ret, save_ret;
int cookie_cnt, cookie_off;
int stack_size, args_off, stk_args_off, nr_arg_slots = 0;
int retval_off, func_meta_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off;
unsigned long long func_meta;
bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
void *orig_call = func_addr;
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
u32 **branches = NULL;
/*
* FP + 8 [ RA to parent func ] return address to parent
* function
* FP + 0 [ FP of parent func ] frame pointer of parent
* function
* FP - 8 [ T0 to traced func ] return address of traced
* function
* FP - 16 [ FP of traced func ] frame pointer of traced
* function
*
* FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
* BPF_TRAMP_F_RET_FENTRY_RET
* [ arg regN ]
* [ ... ]
* FP - args_off [ arg reg1 ]
*
* FP - func_meta_off [ regs count, etc ]
*
* FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG
*
* [ stack cookie N ]
* [ ... ]
* FP - cookie_off [ stack cookie 1 ]
*
* FP - run_ctx_off [ bpf_tramp_run_ctx ]
*
* FP - sreg_off [ callee saved reg ]
*
* FP - tcc_ptr_off [ tail_call_cnt_ptr ]
*
* [ stack_argN ]
* [ ... ]
* FP - stk_args_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG
*/
if (m->nr_args > MAX_BPF_FUNC_ARGS)
return -ENOTSUPP;
/* Extra registers for struct arguments */
for (i = 0; i < m->nr_args; i++) {
/*
* The struct argument size is at most 16 bytes,
* enforced by the verifier. The struct argument
* may be passed in a pair of registers if its
* size is more than 8 bytes and no more than 16
* bytes.
*/
nr_arg_slots += round_up(m->arg_size[i], 8) / 8;
}
if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
return -ENOTSUPP;
/* Room of trampoline frame to store return address and frame pointer */
stack_size = 16;
save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
if (save_ret)
stack_size += 16; /* Save BPF R0 and A0 */
retval_off = stack_size;
/* Room of trampoline frame to store args */
stack_size += nr_arg_slots * 8;
args_off = stack_size;
/* Room of function metadata, such as regs count */
stack_size += 8;
func_meta_off = stack_size;
/* Room of trampoline frame to store ip address */
if (flags & BPF_TRAMP_F_IP_ARG) {
stack_size += 8;
ip_off = stack_size;
}
cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
/* Room for session cookies */
stack_size += cookie_cnt * 8;
cookie_off = stack_size;
/* Room of trampoline frame to store struct bpf_tramp_run_ctx */
stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
run_ctx_off = stack_size;
stack_size += 8;
sreg_off = stack_size;
/* Room of trampoline frame to store tail_call_cnt_ptr */
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
stack_size += 8;
tcc_ptr_off = stack_size;
}
if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - LOONGARCH_MAX_REG_ARGS > 0))
stack_size += (nr_arg_slots - LOONGARCH_MAX_REG_ARGS) * 8;
stack_size = round_up(stack_size, 16);
/* Room for args on stack must be at the top of stack */
stk_args_off = stack_size;
if (is_struct_ops) {
/*
* For the trampoline called directly, just handle
* the frame of trampoline.
*/
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
} else {
/*
* For the trampoline called from function entry,
* the frame of traced function and the frame of
* trampoline need to be considered.
*/
/* RA and FP for parent function */
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16);
emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16);
/* RA and FP for traced function */
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
}
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
/* callee saved register S1 to pass start time */
emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
/* store ip address of the traced function */
if (flags & BPF_TRAMP_F_IP_ARG)
emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -ip_off, (u64)func_addr);
/* store arg regs count */
func_meta = nr_arg_slots;
emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta);
store_args(ctx, nr_arg_slots, args_off);
if (bpf_fsession_cnt(tlinks)) {
/* clear all session cookies' value */
for (i = 0; i < cookie_cnt; i++)
emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -cookie_off + 8 * i);
/* clear return value to make sure fentry always get 0 */
emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
}
/* To traced function */
/* Ftrace jump skips 2 NOP instructions */
if (is_kernel_text((unsigned long)orig_call) ||
is_module_text_address((unsigned long)orig_call))
orig_call += LOONGARCH_FENTRY_NBYTES;
/* Direct jump skips 5 NOP instructions */
else if (is_bpf_text_address((unsigned long)orig_call))
orig_call += LOONGARCH_BPF_FENTRY_NBYTES;
if (flags & BPF_TRAMP_F_CALL_ORIG) {
move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
ret = emit_call(ctx, (const u64)__bpf_tramp_enter);
if (ret)
return ret;
}
if (fentry->nr_links) {
ret = invoke_bpf(ctx, fentry, args_off, retval_off, run_ctx_off, func_meta_off,
flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off);
if (ret)
return ret;
}
if (fmod_ret->nr_links) {
branches = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL);
if (!branches)
return -ENOMEM;
emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
for (i = 0; i < fmod_ret->nr_links; i++) {
ret = invoke_bpf_prog(ctx, fmod_ret->links[i],
args_off, retval_off, run_ctx_off, true);
if (ret)
goto out;
emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off);
branches[i] = (u32 *)ctx->image + ctx->idx;
emit_insn(ctx, nop);
}
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off);
restore_stk_args(ctx, nr_arg_slots - LOONGARCH_MAX_REG_ARGS, args_off, stk_args_off);
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
ret = emit_call(ctx, (const u64)orig_call);
if (ret)
goto out;
emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
im->ip_after_call = ctx->ro_image + ctx->idx;
/* Reserve space for the move_imm + jirl instruction */
for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
emit_insn(ctx, nop);
}
for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) {
int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i];
*branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset);
}
/* Set "is_return" flag for fsession */
func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
if (bpf_fsession_cnt(tlinks))
emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta);
if (fexit->nr_links) {
ret = invoke_bpf(ctx, fexit, args_off, retval_off, run_ctx_off,
func_meta_off, false, func_meta, cookie_off);
if (ret)
goto out;
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
im->ip_epilogue = ctx->ro_image + ctx->idx;
move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
ret = emit_call(ctx, (const u64)__bpf_tramp_exit);
if (ret)
goto out;
}
if (flags & BPF_TRAMP_F_RESTORE_REGS)
restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off);
if (save_ret) {
emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
if (is_struct_ops)
sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0],
m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG);
else
emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
}
emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
if (is_struct_ops) {
/* trampoline called directly */
emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
} else {
/* trampoline called from function entry */
emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16);
if (flags & BPF_TRAMP_F_SKIP_FRAME) {
/* return to parent function */
move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0);
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0);
} else {
/* return to traced function */
move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA);
move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0);
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0);
}
}
ret = ctx->idx;
out:
kfree(branches);
return ret;
}
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
void *ro_image_end, const struct btf_func_model *m,
u32 flags, struct bpf_tramp_links *tlinks, void *func_addr)
{
int ret, size;
void *image, *tmp;
struct jit_ctx ctx;
size = ro_image_end - ro_image;
image = kvmalloc(size, GFP_KERNEL);
if (!image)
return -ENOMEM;
ctx.image = (union loongarch_instruction *)image;
ctx.ro_image = (union loongarch_instruction *)ro_image;
ctx.idx = 0;
jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image));
ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags);
if (ret < 0)
goto out;
if (validate_code(&ctx) < 0) {
ret = -EINVAL;
goto out;
}
tmp = bpf_arch_text_copy(ro_image, image, size);
if (IS_ERR(tmp)) {
ret = PTR_ERR(tmp);
goto out;
}
out:
kvfree(image);
return ret < 0 ? ret : size;
}
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
struct bpf_tramp_links *tlinks, void *func_addr)
{
int ret;
struct jit_ctx ctx;
struct bpf_tramp_image im;
ctx.image = NULL;
ctx.idx = 0;
ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags);
return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
}
struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
bool extra_pass = false;
u8 *image_ptr, *ro_image_ptr;
int image_size, prog_size, extable_size;
struct jit_ctx ctx;
struct jit_data *jit_data;
struct bpf_binary_header *header;
struct bpf_binary_header *ro_header;
/*
* If BPF JIT was not enabled then we must fall back to
* the interpreter.
*/
if (!prog->jit_requested)
return prog;
jit_data = prog->aux->jit_data;
if (!jit_data) {
jit_data = kzalloc_obj(*jit_data);
if (!jit_data)
return prog;
prog->aux->jit_data = jit_data;
}
if (jit_data->ctx.offset) {
ctx = jit_data->ctx;
ro_header = jit_data->ro_header;
ro_image_ptr = (void *)ctx.ro_image;
header = jit_data->header;
image_ptr = (void *)header + ((void *)ro_image_ptr - (void *)ro_header);
extra_pass = true;
prog_size = sizeof(u32) * ctx.idx;
goto skip_init_ctx;
}
memset(&ctx, 0, sizeof(ctx));
ctx.prog = prog;
ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
if (ctx.offset == NULL)
goto out_offset;
/* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
build_prologue(&ctx);
if (build_body(&ctx, extra_pass))
goto out_offset;
ctx.epilogue_offset = ctx.idx;
build_epilogue(&ctx);
extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
/* Now we know the actual image size.
* As each LoongArch instruction is of length 32bit,
* we are translating number of JITed intructions into
* the size required to store these JITed code.
*/
prog_size = sizeof(u32) * ctx.idx;
image_size = prog_size + extable_size;
/* Now we know the size of the structure to make */
ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),
&header, &image_ptr, jit_fill_hole);
if (!ro_header)
goto out_offset;
/* 2. Now, the actual pass to generate final JIT code */
/*
* Use the image (RW) for writing the JITed instructions. But also save
* the ro_image (RX) for calculating the offsets in the image. The RW
* image will be later copied to the RX image from where the program will
* run. The bpf_jit_binary_pack_finalize() will do this copy in the final
* step.
*/
ctx.image = (union loongarch_instruction *)image_ptr;
ctx.ro_image = (union loongarch_instruction *)ro_image_ptr;
if (extable_size)
prog->aux->extable = (void *)ro_image_ptr + prog_size;
skip_init_ctx:
ctx.idx = 0;
ctx.num_exentries = 0;
build_prologue(&ctx);
if (build_body(&ctx, extra_pass))
goto out_free;
build_epilogue(&ctx);
/* 3. Extra pass to validate JITed code */
if (validate_ctx(&ctx))
goto out_free;
/* And we're done */
if (bpf_jit_enable > 1)
bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
if (!prog->is_func || extra_pass) {
if (extra_pass && ctx.idx != jit_data->ctx.idx) {
pr_err_once("multi-func JIT bug %d != %d\n",
ctx.idx, jit_data->ctx.idx);
goto out_free;
}
if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
/* ro_header and header have been freed */
ro_header = NULL;
header = NULL;
goto out_free;
}
/*
* The instructions have now been copied to the ROX region from
* where they will execute. Now the data cache has to be cleaned
* to the PoU and the I-cache has to be invalidated for the VAs.
*/
bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx);
} else {
jit_data->ctx = ctx;
jit_data->header = header;
jit_data->ro_header = ro_header;
}
prog->jited = 1;
prog->jited_len = prog_size;
prog->bpf_func = (void *)ctx.ro_image;
if (!prog->is_func || extra_pass) {
int i;
/* offset[prog->len] is the size of program */
for (i = 0; i <= prog->len; i++)
ctx.offset[i] *= LOONGARCH_INSN_SIZE;
bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
out_offset:
kvfree(ctx.offset);
kfree(jit_data);
prog->aux->jit_data = NULL;
}
return prog;
out_free:
if (extra_pass) {
prog->bpf_func = NULL;
prog->jited = 0;
prog->jited_len = 0;
}
if (header) {
bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size));
bpf_jit_binary_pack_free(ro_header, header);
}
goto out_offset;
}
void bpf_jit_free(struct bpf_prog *prog)
{
if (prog->jited) {
struct jit_data *jit_data = prog->aux->jit_data;
struct bpf_binary_header *hdr;
/*
* If we fail the final pass of JIT (from jit_subprogs), the
* program may not be finalized yet. Call finalize here before
* freeing it.
*/
if (jit_data) {
bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header);
kfree(jit_data);
}
hdr = bpf_jit_binary_pack_hdr(prog);
bpf_jit_binary_pack_free(hdr, NULL);
WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
}
bpf_prog_unlock_free(prog);
}
bool bpf_jit_bypass_spec_v1(void)
{
return true;
}
bool bpf_jit_bypass_spec_v4(void)
{
return true;
}
bool bpf_jit_supports_arena(void)
{
return true;
}
bool bpf_jit_supports_fsession(void)
{
return true;
}
/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
bool bpf_jit_supports_subprog_tailcalls(void)
{
return true;
}