10 #include <bpf/libbpf.h> 18 #include <linux/filter.h> 20 #define BPF_REG_SIZE 8 22 #define JMP_TO_ERROR_CODE -1 23 #define JMP_TO_SUCCESS_CODE -2 24 #define JMP_TO_USER_CODE -3 44 if (pos->
pos + 1 >= pos->
end) {
45 pr_err(
"bpf prologue: prologue too long\n");
50 *(pos->
pos)++ = new_insn;
70 int arg_size = type ? atoi(&type[1]) : 64;
103 #define ins(i, p) append_insn((i), (p)) 113 const char *reg,
int target_reg)
118 pr_err(
"bpf: prologue: failed to get register %s\n",
122 ins(BPF_LDX_MEM(BPF_DW, target_reg, ctx_reg, offset), pos);
143 int src_base_addr_reg,
148 if (src_base_addr_reg != BPF_REG_ARG3)
149 ins(BPF_MOV64_REG(BPF_REG_ARG3, src_base_addr_reg), pos);
152 ins(BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG3, offset), pos);
158 if (dst_addr_reg != BPF_REG_ARG1)
159 ins(BPF_MOV64_REG(BPF_REG_ARG1, dst_addr_reg), pos);
162 ins(BPF_EMIT_CALL(BPF_FUNC_probe_read), pos);
187 for (i = 0; i < nargs; i++) {
246 for (i = 0; i < nargs; i++) {
248 const char *reg = arg->
value;
250 int stack_offset = (i + 1) * -8;
252 pr_debug(
"prologue: fetch arg %d, base reg is %s\n",
259 pr_err(
"prologue: failed to get offset of register %s\n",
265 ins(BPF_MOV64_REG(BPF_REG_7, BPF_REG_FP), pos);
267 ins(BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, stack_offset), pos);
275 ins(BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_ARG3,
280 pr_debug(
"prologue: arg %d: offset %ld\n",
285 pr_err(
"prologue: failed to generate probe_read function call\n");
296 ins(BPF_LDX_MEM(BPF_DW, BPF_REG_ARG3,
297 BPF_REG_FP, stack_offset), pos);
302 for (i = 0; i < nargs; i++) {
305 pr_debug(
"prologue: load arg %d, insn_sz is %s\n",
320 struct bpf_insn *success_code,
struct bpf_insn *user_code)
322 struct bpf_insn *
insn;
329 u8
class = BPF_CLASS(insn->code);
332 if (
class != BPF_JMP)
334 opcode = BPF_OP(insn->code);
335 if (opcode == BPF_CALL)
343 target = success_code;
349 pr_err(
"bpf prologue: internal error: relocation failed\n");
353 insn->off = target - (insn + 1);
359 struct bpf_insn *new_prog,
size_t *new_cnt,
362 struct bpf_insn *success_code = NULL;
363 struct bpf_insn *error_code = NULL;
364 struct bpf_insn *user_code = NULL;
366 bool fastpath =
true;
369 if (!new_prog || !new_cnt)
372 if (cnt_space > BPF_MAXINSNS)
373 cnt_space = BPF_MAXINSNS;
375 pos.
begin = new_prog;
376 pos.
end = new_prog + cnt_space;
391 pr_warning(
"bpf: prologue: %d arguments are dropped\n",
397 for (i = 0; i < nargs; i++) {
400 if (args[i].
value[0] ==
'@') {
402 pr_err(
"bpf: prologue: global %s%+ld not support\n",
418 #define OFFSET_MAX ((1LL << 31) - 1) 419 #define OFFSET_MIN ((1LL << 31) * -1) 420 if (ref->
offset > OFFSET_MAX ||
421 ref->
offset < OFFSET_MIN) {
422 pr_err(
"bpf: prologue: offset out of bound: %ld\n",
430 pr_debug(
"prologue: pass validation\n");
442 ins(BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1), &pos);
456 error_code = pos.
pos;
460 for (i = 0; i < nargs; i++)
461 ins(BPF_ALU64_IMM(BPF_MOV,
474 success_code = pos.
pos;
487 ins(BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX), &pos);
static const char * insn_sz_to_str(int insn_sz)
#define BPF_PROLOGUE_START_ARG_REG
static int gen_prologue_fastpath(struct bpf_insn_pos *pos, struct probe_trace_arg *args, int nargs)
static int gen_ldx_reg_from_ctx(struct bpf_insn_pos *pos, int ctx_reg, const char *reg, int target_reg)
static int gen_prologue_slowpath(struct bpf_insn_pos *pos, struct probe_trace_arg *args, int nargs)
int regs_query_register_offset(const char *name)
#define BPF_PROLOGUE_MAX_ARGS
static int append_insn(struct bpf_insn new_insn, struct bpf_insn_pos *pos)
struct probe_trace_arg_ref * next
#define JMP_TO_ERROR_CODE
#define pr_debug(fmt,...)
#define JMP_TO_SUCCESS_CODE
static int pos_get_cnt(struct bpf_insn_pos *pos)
static int gen_read_mem(struct bpf_insn_pos *pos, int src_base_addr_reg, int dst_addr_reg, long offset)
int bpf__gen_prologue(struct probe_trace_arg *args, int nargs, struct bpf_insn *new_prog, size_t *new_cnt, size_t cnt_space)
static int check_pos(struct bpf_insn_pos *pos)
static int prologue_relocate(struct bpf_insn_pos *pos, struct bpf_insn *error_code, struct bpf_insn *success_code, struct bpf_insn *user_code)
#define BPF_PROLOGUE_FETCH_RESULT_REG
#define pr_warning(fmt,...)
static int argtype_to_ldx_size(const char *type)
struct probe_trace_arg_ref * ref