25 #include <linux/compiler.h> 35 #define INTEL_PT_BLK_SIZE 1024 37 #define BIT63 (((uint64_t)1 << 63)) 39 #define INTEL_PT_RETURN 1 42 #define INTEL_PT_MAX_LOOPS 10000 86 #ifdef INTEL_PT_STRICT 87 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 88 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 89 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 90 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 92 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 93 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 94 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 95 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 101 uint64_t *insn_cnt_ptr, uint64_t *
ip, uint64_t to_ip,
102 uint64_t max_insn_cnt,
void *
data);
188 for (i = 0; x != 1; i++)
205 static uint64_t
multdiv(uint64_t t, uint32_t n, uint32_t d)
209 return (t / d) * n + ((t % d) * n) / d;
297 return stack->
blk->
ip[--stack->
pos];
405 switch (packet->
count) {
407 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
411 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
417 if (ip & (uint64_t)0x800000000000ULL)
418 ip |= (uint64_t)0xffff000000000000ULL;
421 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
494 decoder->
buf = buffer.
buf;
495 decoder->
len = buffer.
len;
530 size_t old_len,
len, n;
533 old_len = decoder->
len;
539 decoder->
pos += old_len;
540 return ret < 0 ? ret : -EINVAL;
544 if (n > decoder->
len)
550 if (ret < (
int)old_len) {
554 decoder->
len = old_len;
558 decoder->
next_buf = decoder->
buf + (ret - old_len);
559 decoder->
next_len = decoder->
len - (ret - old_len);
583 const unsigned char *buf = decoder->
buf;
584 size_t len = decoder->
len;
588 pkt_info.
pos = decoder->
pos;
643 uint32_t first_missing_bit = 1U << (16 - mtc_shift);
644 uint32_t mask = ~(first_missing_bit - 1);
646 *last_mtc |= mtc & mask;
647 if (*last_mtc >= mtc) {
648 *last_mtc -= first_missing_bit;
660 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
692 mtc_delta = mtc + 256 - data->
last_mtc;
706 if (timestamp < data->timestamp)
725 if (data->
from_mtc && timestamp < data->timestamp &&
726 data->
timestamp - timestamp < decoder->tsc_slip)
728 if (timestamp < data->timestamp)
729 timestamp += (1ULL << 56);
773 if (data->
cbr && data->
cbr != cbr)
792 if (!data->
cbr && decoder->
cbr) {
804 intel_pt_log(
"Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
813 intel_pt_log(
"Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
816 intel_pt_log(
"Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
817 cyc_to_tsc, pkt_info->
pos);
892 masked_timestamp = timestamp & decoder->
period_mask;
898 masked_timestamp = timestamp & decoder->
period_mask;
904 return decoder->
period_ticks - (timestamp - masked_timestamp);
931 masked_timestamp = timestamp & decoder->
period_mask;
946 uint64_t max_insn_cnt, insn_cnt = 0;
954 err = decoder->
walk_insn(intel_pt_insn, &insn_cnt, &decoder->
ip, ip,
955 max_insn_cnt, decoder->
data);
972 if (ip && decoder->
ip == ip) {
977 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
984 decoder->
ip += intel_pt_insn->
length;
992 intel_pt_insn->
rel) {
1006 decoder->
ip += intel_pt_insn->
length +
1036 goto out_no_progress;
1105 uint64_t
ip,
int err)
1109 ip == decoder->
ip + intel_pt_insn->
length;
1124 if (err == -EAGAIN ||
1165 decoder->
pge =
false;
1178 decoder->
pge =
false;
1199 uint64_t to_ip = decoder->
ip + intel_pt_insn.
length +
1206 decoder->
pge =
false;
1209 decoder->
ip = to_ip;
1214 intel_pt_log_at(
"ERROR: Conditional branch when expecting indirect branch",
1291 decoder->
ip += intel_pt_insn.
length +
1302 decoder->
ip += intel_pt_insn.
length;
1305 decoder->
ip += intel_pt_insn.
length;
1318 unsigned int fup_tx_flags;
1348 if (timestamp < decoder->ref_timestamp) {
1350 timestamp += (1ULL << 56);
1353 timestamp -= (1ULL << 56);
1363 if (timestamp < decoder->timestamp &&
1364 decoder->
timestamp - timestamp < decoder->tsc_slip) {
1369 if (timestamp < decoder->timestamp) {
1371 timestamp += (1ULL << 56);
1427 uint32_t mtc, mtc_delta;
1441 mtc_delta = mtc - decoder->
last_mtc;
1443 mtc_delta = mtc + 256 - decoder->
last_mtc;
1457 if (timestamp < decoder->timestamp)
1480 if (decoder->
cbr == cbr)
1500 else if (decoder->
cbr)
1505 if (timestamp < decoder->timestamp)
1570 decoder->
pge =
true;
1649 decoder->
pge =
false;
1654 decoder->
pge =
true;
1707 bool no_tip =
false;
1733 decoder->
pge =
true;
1772 if (err != -EAGAIN) {
1789 decoder->
pge =
false;
1862 if (!decoder->
pge) {
1985 decoder->
pge =
true;
1987 uint64_t current_ip = decoder->
ip;
2029 decoder->
pge =
false;
2130 decoder->
pge =
false;
2202 const unsigned char *end = decoder->
buf + decoder->
len;
2206 if (i > decoder->
len)
2219 if (rest_psb > decoder->
len ||
2220 memcmp(decoder->
buf, psb + part_psb, rest_psb))
2231 decoder->
pos += decoder->
len;
2242 decoder->
pos -= part_psb;
2254 unsigned char *next;
2259 if (!decoder->
len) {
2276 decoder->
pos += decoder->
len;
2291 decoder->
pge =
false;
2327 est /= decoder->
cbr;
2382 }
while (err == -ENOLINK);
2407 return &decoder->
state;
2423 unsigned char *next;
2427 *len -= next - *buf;
2448 unsigned char *next;
2455 *len -= next - *buf;
2482 p = memrchr(buf, n[0], k);
2485 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2544 const uint64_t halfway = (1ULL << 55);
2550 if (tsc2 - tsc1 < halfway)
2555 if (tsc1 - tsc2 < halfway)
2582 unsigned char *buf_b,
2583 size_t len_b,
bool *consecutive)
2585 uint64_t tsc_a, tsc_b;
2587 size_t len, rem_a, rem_b;
2593 len = len_a - (p - buf_a);
2600 len = len_a - (p - buf_a);
2611 if (!cmp && rem_b >= rem_a) {
2612 *consecutive =
true;
2613 return buf_b + len_b - (rem_b - rem_a);
2620 return buf_b + len_b;
2642 unsigned char *buf_b,
size_t len_b,
2643 bool have_tsc,
bool *consecutive)
2645 unsigned char *found;
2649 return buf_b + len_b;
2665 while (len_b < len_a) {
2673 found = memmem(buf_a, len_a, buf_b, len_a);
2675 *consecutive =
true;
2676 return buf_b + len_a;
static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
#define intel_pt_log(fmt,...)
static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
#define INTEL_PT_STATE_ERR4
int(* walk_insn)(struct intel_pt_insn *intel_pt_insn, uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, uint64_t max_insn_cnt, void *data)
static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
bool(* pgd_ip)(uint64_t ip, void *data)
static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
static void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
uint64_t fup_mwait_payload
static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, size_t *rem)
static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, uint64_t last_ip)
const unsigned char * buf
static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, bool from_mtc)
#define INTEL_PT_INSN_BUF_SZ
unsigned max_non_turbo_ratio
static uint64_t intel_pt_lower_power_of_2(uint64_t x)
#define INTEL_PT_STATE_ERR2
double max_non_turbo_ratio_fp
static void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
enum intel_pt_insn_op insn_op
static void intel_pt_clear_stack(struct intel_pt_stack *stack)
bool have_calc_cyc_to_tsc
struct intel_pt_state state
enum intel_pt_insn_branch branch
unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]
struct intel_pt_pkt packet
static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
unsigned char * intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, size_t len_b, bool have_tsc, bool *consecutive)
static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, intel_pt_pkt_cb_t cb, void *data)
#define INTEL_PT_STATE_ERR1
struct intel_pt_decoder * intel_pt_decoder_new(struct intel_pt_params *params)
uint64_t ip[INTEL_PT_BLK_SIZE]
int(* get_trace)(struct intel_pt_buffer *buffer, void *data)
unsigned char buf[INTEL_PT_INSN_BUF_SZ]
static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
#define INTEL_PT_BLK_SIZE
unsigned int fup_tx_flags
struct intel_pt_blk * spare
static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
int intel_pt_get_packet(const unsigned char *buf, size_t len, struct intel_pt_pkt *packet)
static bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, uint32_t *last_mtc)
uint64_t last_masked_timestamp
int(* get_trace)(struct intel_pt_buffer *buffer, void *data)
static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
enum intel_pt_period_type period_type
static void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
const unsigned char * next_buf
enum intel_pt_sample_type type
char insn[INTEL_PT_INSN_BUF_SZ]
uint64_t sample_timestamp
static bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, struct intel_pt_insn *intel_pt_insn, uint64_t ip, int err)
struct intel_pt_blk * prev
#define INTEL_PT_ABORT_TX
static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
static void intel_pt_log_at(const char *msg, uint64_t u)
static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
static int intel_pt_sync(struct intel_pt_decoder *decoder)
x86 movsq based memcpy() in arch/x86/lib/memcpy_64.S") MEMCPY_FN(memcpy_erms
static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
#define INTEL_PT_MAX_LOOPS
#define INTEL_PT_NEED_MORE_BYTES
enum intel_pt_pkt_state pkt_state
enum intel_pt_param_flags flags
static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
struct intel_pt_stack stack
static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
struct intel_pt_blk * blk
static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
#define INTEL_PT_PKT_MAX_SZ
static void intel_pt_set_ip(struct intel_pt_decoder *decoder)
static const char * intel_pt_err_msgs[]
static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
enum intel_pt_period_type period_type
int(* intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info)
static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
enum intel_pt_pkt_type type
struct intel_pt_decoder * decoder
bool(* pgd_ip)(uint64_t ip, void *data)
struct intel_pt_pkt packet
static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
const unsigned char * buf
uint64_t cyc_ref_timestamp
static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
static void intel_pt_log_to(const char *msg, uint64_t u)
static int intel_pt_ext_err(int code)
static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
enum intel_pt_param_flags flags
static int intel_pt_bug(struct intel_pt_decoder *decoder)
static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, int part_psb)
unsigned int max_non_turbo_ratio
static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
static int intel_pt_overflow(struct intel_pt_decoder *decoder)
uint64_t fup_pwre_payload
static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
static unsigned char * intel_pt_find_overlap_tsc(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, size_t len_b, bool *consecutive)
uint64_t timestamp_insn_cnt
static int intel_pt_get_data(struct intel_pt_decoder *decoder)
static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
static void intel_pt_free_stack(struct intel_pt_stack *stack)
int intel_pt__strerror(int code, char *buf, size_t buflen)
int(* walk_insn)(struct intel_pt_insn *intel_pt_insn, uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, uint64_t max_insn_cnt, void *data)
static void intel_pt_pop_blk(struct intel_pt_stack *stack)
static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, struct intel_pt_insn *intel_pt_insn, uint64_t ip)
static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
static bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
const struct intel_pt_state * intel_pt_decode(struct intel_pt_decoder *decoder)
#define intel_pt_log_packet(arg,...)
#define INTEL_PT_STATE_ERR3
static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
void static void * zalloc(size_t size)
static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
static unsigned char * intel_pt_last_psb(unsigned char *buf, size_t len)