@@ -642,6 +642,100 @@ data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
return 0;
}
+typedef int
+(*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off,
+ unsigned int size);
+
+static int
+wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off,
+ unsigned int size)
+{
+ u32 idx, dst_byte;
+ enum shf_sc sc;
+ swreg reg;
+ int shf;
+ u8 mask;
+
+ if (WARN_ON_ONCE(src_byte + size > 4 || off % 4 + size > 4))
+ return -EOPNOTSUPP;
+
+ idx = off / 4;
+
+ /* Move the entire word */
+ if (size == 4) {
+ wrp_mov(nfp_prog, reg_lm(0, idx), reg_b(src));
+ return 0;
+ }
+
+ dst_byte = off % 4;
+
+ mask = (1 << size) - 1;
+ mask <<= dst_byte;
+
+ if (WARN_ON_ONCE(mask > 0xf))
+ return -EOPNOTSUPP;
+
+ shf = abs(src_byte - dst_byte) * 8;
+ if (src_byte == dst_byte) {
+ sc = SHF_SC_NONE;
+ } else if (src_byte < dst_byte) {
+ shf = 32 - shf;
+ sc = SHF_SC_L_SHF;
+ } else {
+ sc = SHF_SC_R_SHF;
+ }
+
+ /* ld_field can address fewer indexes, if offset too large do RMW.
+ * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes.
+ */
+ if (idx <= RE_REG_LM_IDX_MAX) {
+ reg = reg_lm(0, idx);
+ } else {
+ reg = imm_a(nfp_prog);
+ wrp_mov(nfp_prog, reg, reg_lm(0, idx));
+ }
+
+ emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf);
+
+ if (idx > RE_REG_LM_IDX_MAX)
+ wrp_mov(nfp_prog, reg_lm(0, idx), reg);
+
+ return 0;
+}
+
+static int
+mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+ unsigned int size, u8 gpr, lmem_step step)
+{
+ s32 off = nfp_prog->stack_depth + meta->insn.off;
+ u32 gpr_byte = 0;
+ int ret;
+
+ while (size) {
+ u32 slice_end;
+ u8 slice_size;
+
+ slice_size = min(size, 4 - gpr_byte);
+ slice_end = min(off + slice_size, round_up(off + 1, 4));
+ slice_size = slice_end - off;
+
+ ret = step(nfp_prog, gpr, gpr_byte, off, slice_size);
+ if (ret)
+ return ret;
+
+ gpr_byte += slice_size;
+ if (gpr_byte >= 4) {
+ gpr_byte -= 4;
+ gpr++;
+ }
+
+ size -= slice_size;
+ off += slice_size;
+ }
+
+ return 0;
+}
+
static void
wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm)
{
@@ -1298,6 +1392,14 @@ mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
meta->insn.src_reg * 2, size);
}
+static int
+mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+ unsigned int size)
+{
+ return mem_op_stack(nfp_prog, meta, size, meta->insn.src_reg * 2,
+ wrp_lmem_store);
+}
+
static int
mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
unsigned int size)
@@ -1305,6 +1407,9 @@ mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
if (meta->ptr.type == PTR_TO_PACKET)
return mem_stx_data(nfp_prog, meta, size);
+ if (meta->ptr.type == PTR_TO_STACK)
+ return mem_stx_stack(nfp_prog, meta, size);
+
return -EOPNOTSUPP;
}
@@ -151,6 +151,7 @@ static inline u8 mbpf_mode(const struct nfp_insn_meta *meta)
* @tgt_done: jump target to get the next packet
* @n_translated: number of successfully translated instructions (for errors)
* @error: error code if something went wrong
+ * @stack_depth: max stack depth from the verifier
* @insns: list of BPF instruction wrappers (struct nfp_insn_meta)
*/
struct nfp_prog {
@@ -171,6 +172,8 @@ struct nfp_prog {
unsigned int n_translated;
int error;
+ unsigned int stack_depth;
+
struct list_head insns;
};
@@ -146,6 +146,7 @@ nfp_net_bpf_offload_prepare(struct nfp_net *nn,
{
unsigned int code_sz = max_instr * sizeof(u64);
enum nfp_bpf_action_type act;
+ unsigned int stack_size;
u16 start_off, done_off;
unsigned int max_mtu;
int ret;
@@ -167,6 +168,19 @@ nfp_net_bpf_offload_prepare(struct nfp_net *nn,
start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
done_off = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
+ if (cls_bpf->prog->aux->stack_depth > 64) {
+ nn_info(nn, "large stack not supported: program %dB > 64B\n",
+ cls_bpf->prog->aux->stack_depth);
+ return -EOPNOTSUPP;
+ }
+
+ stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
+ if (cls_bpf->prog->aux->stack_depth > stack_size) {
+ nn_info(nn, "stack too large: program %dB > FW stack %dB\n",
+ cls_bpf->prog->aux->stack_depth, stack_size);
+ return -EOPNOTSUPP;
+ }
+
*code = dma_zalloc_coherent(nn->dp.dev, code_sz, dma_addr, GFP_KERNEL);
if (!*code)
return -ENOMEM;
@@ -111,18 +111,41 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
return 0;
}
+static int nfp_bpf_check_stack_access(const struct bpf_reg_state *reg)
+{
+ if (!tnum_is_const(reg->var_off)) {
+ pr_info("variable ptr stack access\n");
+ return -EINVAL;
+ }
+
+ if (reg->var_off.value || reg->off) {
+ pr_info("stack access via modified register\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static int
nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
const struct bpf_verifier_env *env, u8 reg_no)
{
const struct bpf_reg_state *reg = &env->cur_state.regs[reg_no];
+ int err;
if (reg->type != PTR_TO_CTX &&
+ reg->type != PTR_TO_STACK &&
reg->type != PTR_TO_PACKET) {
pr_info("unsupported ptr type: %d\n", reg->type);
return -EINVAL;
}
+ if (reg->type == PTR_TO_STACK) {
+ err = nfp_bpf_check_stack_access(reg);
+ if (err)
+ return err;
+ }
+
if (meta->ptr.type != NOT_INIT && meta->ptr.type != reg->type) {
pr_info("ptr type changed for instruction %d -> %d\n",
meta->ptr.type, reg->type);
@@ -143,11 +166,6 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
meta = nfp_bpf_goto_meta(priv->prog, meta, insn_idx, env->prog->len);
priv->meta = meta;
- if (meta->insn.src_reg == BPF_REG_10 ||
- meta->insn.dst_reg == BPF_REG_10) {
- pr_err("stack not yet supported\n");
- return -EINVAL;
- }
if (meta->insn.src_reg >= MAX_BPF_REG ||
meta->insn.dst_reg >= MAX_BPF_REG) {
pr_err("program uses extended registers - jit hardening?\n");
@@ -176,6 +194,8 @@ int nfp_prog_verify(struct nfp_prog *nfp_prog, struct bpf_prog *prog)
struct nfp_bpf_analyzer_priv *priv;
int ret;
+ nfp_prog->stack_depth = prog->aux->stack_depth;
+
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;