Linux Perf
intel-pt-decoder.c
Go to the documentation of this file.
1 /*
2  * intel_pt_decoder.c: Intel Processor Trace support
3  * Copyright (c) 2013-2014, Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12  * more details.
13  *
14  */
15 
16 #ifndef _GNU_SOURCE
17 #define _GNU_SOURCE
18 #endif
19 #include <stdlib.h>
20 #include <stdbool.h>
21 #include <string.h>
22 #include <errno.h>
23 #include <stdint.h>
24 #include <inttypes.h>
25 #include <linux/compiler.h>
26 
27 #include "../cache.h"
28 #include "../util.h"
29 
30 #include "intel-pt-insn-decoder.h"
31 #include "intel-pt-pkt-decoder.h"
32 #include "intel-pt-decoder.h"
33 #include "intel-pt-log.h"
34 
35 #define INTEL_PT_BLK_SIZE 1024
36 
37 #define BIT63 (((uint64_t)1 << 63))
38 
39 #define INTEL_PT_RETURN 1
40 
41 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
42 #define INTEL_PT_MAX_LOOPS 10000
43 
44 struct intel_pt_blk {
45  struct intel_pt_blk *prev;
46  uint64_t ip[INTEL_PT_BLK_SIZE];
47 };
48 
50  struct intel_pt_blk *blk;
52  int pos;
53 };
54 
65 };
66 
67 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
68 {
69  switch (pkt_state) {
74  case INTEL_PT_STATE_TNT:
75  return true;
76  case INTEL_PT_STATE_TIP:
78  case INTEL_PT_STATE_FUP:
80  return false;
81  default:
82  return true;
83  };
84 }
85 
86 #ifdef INTEL_PT_STRICT
87 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
88 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
89 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
90 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
91 #else
92 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
93 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
94 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
95 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
96 #endif
97 
99  int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
100  int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
101  uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
102  uint64_t max_insn_cnt, void *data);
103  bool (*pgd_ip)(uint64_t ip, void *data);
104  void *data;
106  const unsigned char *buf;
107  size_t len;
110  bool mtc_insn;
111  bool pge;
112  bool have_tma;
113  bool have_cyc;
117  uint64_t pos;
118  uint64_t last_ip;
119  uint64_t ip;
120  uint64_t cr3;
121  uint64_t timestamp;
122  uint64_t tsc_timestamp;
123  uint64_t ref_timestamp;
125  uint64_t ret_addr;
126  uint64_t ctc_timestamp;
127  uint64_t ctc_delta;
128  uint64_t cycle_cnt;
130  uint32_t last_mtc;
131  uint32_t tsc_ctc_ratio_n;
132  uint32_t tsc_ctc_ratio_d;
133  uint32_t tsc_ctc_mult;
134  uint32_t tsc_slip;
135  uint32_t ctc_rem_mask;
137  struct intel_pt_stack stack;
138  enum intel_pt_pkt_state pkt_state;
139  struct intel_pt_pkt packet;
140  struct intel_pt_pkt tnt;
141  int pkt_step;
142  int pkt_len;
144  unsigned int cbr;
145  unsigned int cbr_seen;
146  unsigned int max_non_turbo_ratio;
152  unsigned int insn_bytes;
153  uint64_t period;
154  enum intel_pt_period_type period_type;
155  uint64_t tot_insn_cnt;
156  uint64_t period_insn_cnt;
157  uint64_t period_mask;
158  uint64_t period_ticks;
161  bool overflow;
167  unsigned int fup_tx_flags;
168  unsigned int tx_flags;
169  uint64_t fup_ptw_payload;
172  uint64_t cbr_payload;
174  uint64_t sample_insn_cnt;
175  uint64_t stuck_ip;
179  const unsigned char *next_buf;
180  size_t next_len;
181  unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
182 };
183 
184 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
185 {
186  int i;
187 
188  for (i = 0; x != 1; i++)
189  x >>= 1;
190 
191  return x << i;
192 }
193 
194 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
195 {
196  if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
197  uint64_t period;
198 
199  period = intel_pt_lower_power_of_2(decoder->period);
200  decoder->period_mask = ~(period - 1);
201  decoder->period_ticks = period;
202  }
203 }
204 
205 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
206 {
207  if (!d)
208  return 0;
209  return (t / d) * n + ((t % d) * n) / d;
210 }
211 
213 {
214  struct intel_pt_decoder *decoder;
215 
216  if (!params->get_trace || !params->walk_insn)
217  return NULL;
218 
219  decoder = zalloc(sizeof(struct intel_pt_decoder));
220  if (!decoder)
221  return NULL;
222 
223  decoder->get_trace = params->get_trace;
224  decoder->walk_insn = params->walk_insn;
225  decoder->pgd_ip = params->pgd_ip;
226  decoder->data = params->data;
227  decoder->return_compression = params->return_compression;
228  decoder->branch_enable = params->branch_enable;
229 
230  decoder->flags = params->flags;
231 
232  decoder->period = params->period;
233  decoder->period_type = params->period_type;
234 
235  decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
236  decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
237 
238  intel_pt_setup_period(decoder);
239 
240  decoder->mtc_shift = params->mtc_period;
241  decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
242 
243  decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
244  decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
245 
246  if (!decoder->tsc_ctc_ratio_n)
247  decoder->tsc_ctc_ratio_d = 0;
248 
249  if (decoder->tsc_ctc_ratio_d) {
250  if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
251  decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
252  decoder->tsc_ctc_ratio_d;
253 
254  /*
255  * Allow for timestamps appearing to backwards because a TSC
256  * packet has slipped past a MTC packet, so allow 2 MTC ticks
257  * or ...
258  */
259  decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
260  decoder->tsc_ctc_ratio_n,
261  decoder->tsc_ctc_ratio_d);
262  }
263  /* ... or 0x100 paranoia */
264  if (decoder->tsc_slip < 0x100)
265  decoder->tsc_slip = 0x100;
266 
267  intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
268  intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
269  intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
270  intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
271  intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
272 
273  return decoder;
274 }
275 
277 {
278  struct intel_pt_blk *blk = stack->blk;
279 
280  stack->blk = blk->prev;
281  if (!stack->spare)
282  stack->spare = blk;
283  else
284  free(blk);
285 }
286 
287 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
288 {
289  if (!stack->pos) {
290  if (!stack->blk)
291  return 0;
292  intel_pt_pop_blk(stack);
293  if (!stack->blk)
294  return 0;
295  stack->pos = INTEL_PT_BLK_SIZE;
296  }
297  return stack->blk->ip[--stack->pos];
298 }
299 
300 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
301 {
302  struct intel_pt_blk *blk;
303 
304  if (stack->spare) {
305  blk = stack->spare;
306  stack->spare = NULL;
307  } else {
308  blk = malloc(sizeof(struct intel_pt_blk));
309  if (!blk)
310  return -ENOMEM;
311  }
312 
313  blk->prev = stack->blk;
314  stack->blk = blk;
315  stack->pos = 0;
316  return 0;
317 }
318 
319 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
320 {
321  int err;
322 
323  if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
324  err = intel_pt_alloc_blk(stack);
325  if (err)
326  return err;
327  }
328 
329  stack->blk->ip[stack->pos++] = ip;
330  return 0;
331 }
332 
333 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
334 {
335  while (stack->blk)
336  intel_pt_pop_blk(stack);
337  stack->pos = 0;
338 }
339 
340 static void intel_pt_free_stack(struct intel_pt_stack *stack)
341 {
342  intel_pt_clear_stack(stack);
343  zfree(&stack->blk);
344  zfree(&stack->spare);
345 }
346 
348 {
349  intel_pt_free_stack(&decoder->stack);
350  free(decoder);
351 }
352 
353 static int intel_pt_ext_err(int code)
354 {
355  switch (code) {
356  case -ENOMEM:
357  return INTEL_PT_ERR_NOMEM;
358  case -ENOSYS:
359  return INTEL_PT_ERR_INTERN;
360  case -EBADMSG:
361  return INTEL_PT_ERR_BADPKT;
362  case -ENODATA:
363  return INTEL_PT_ERR_NODATA;
364  case -EILSEQ:
365  return INTEL_PT_ERR_NOINSN;
366  case -ENOENT:
367  return INTEL_PT_ERR_MISMAT;
368  case -EOVERFLOW:
369  return INTEL_PT_ERR_OVR;
370  case -ENOSPC:
371  return INTEL_PT_ERR_LOST;
372  case -ELOOP:
373  return INTEL_PT_ERR_NELOOP;
374  default:
375  return INTEL_PT_ERR_UNK;
376  }
377 }
378 
379 static const char *intel_pt_err_msgs[] = {
380  [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
381  [INTEL_PT_ERR_INTERN] = "Internal error",
382  [INTEL_PT_ERR_BADPKT] = "Bad packet",
383  [INTEL_PT_ERR_NODATA] = "No more data",
384  [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
385  [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
386  [INTEL_PT_ERR_OVR] = "Overflow packet",
387  [INTEL_PT_ERR_LOST] = "Lost trace data",
388  [INTEL_PT_ERR_UNK] = "Unknown error!",
389  [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
390 };
391 
392 int intel_pt__strerror(int code, char *buf, size_t buflen)
393 {
394  if (code < 1 || code >= INTEL_PT_ERR_MAX)
395  code = INTEL_PT_ERR_UNK;
396  strlcpy(buf, intel_pt_err_msgs[code], buflen);
397  return 0;
398 }
399 
400 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
401  uint64_t last_ip)
402 {
403  uint64_t ip;
404 
405  switch (packet->count) {
406  case 1:
407  ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
408  packet->payload;
409  break;
410  case 2:
411  ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
412  packet->payload;
413  break;
414  case 3:
415  ip = packet->payload;
416  /* Sign-extend 6-byte ip */
417  if (ip & (uint64_t)0x800000000000ULL)
418  ip |= (uint64_t)0xffff000000000000ULL;
419  break;
420  case 4:
421  ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
422  packet->payload;
423  break;
424  case 6:
425  ip = packet->payload;
426  break;
427  default:
428  return 0;
429  }
430 
431  return ip;
432 }
433 
434 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
435 {
436  decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
437  decoder->have_last_ip = true;
438 }
439 
440 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
441 {
442  intel_pt_set_last_ip(decoder);
443  decoder->ip = decoder->last_ip;
444 }
445 
446 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
447 {
448  intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
449  decoder->buf);
450 }
451 
452 static int intel_pt_bug(struct intel_pt_decoder *decoder)
453 {
454  intel_pt_log("ERROR: Internal error\n");
455  decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
456  return -ENOSYS;
457 }
458 
459 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
460 {
461  decoder->tx_flags = 0;
462 }
463 
464 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
465 {
466  decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
467 }
468 
469 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
470 {
471  intel_pt_clear_tx_flags(decoder);
472  decoder->have_tma = false;
473  decoder->pkt_len = 1;
474  decoder->pkt_step = 1;
476  if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
477  intel_pt_log("ERROR: Bad packet\n");
478  decoder->pkt_state = INTEL_PT_STATE_ERR1;
479  }
480  return -EBADMSG;
481 }
482 
483 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
484 {
485  struct intel_pt_buffer buffer = { .buf = 0, };
486  int ret;
487 
488  decoder->pkt_step = 0;
489 
490  intel_pt_log("Getting more data\n");
491  ret = decoder->get_trace(&buffer, decoder->data);
492  if (ret)
493  return ret;
494  decoder->buf = buffer.buf;
495  decoder->len = buffer.len;
496  if (!decoder->len) {
497  intel_pt_log("No more data\n");
498  return -ENODATA;
499  }
500  if (!buffer.consecutive) {
501  decoder->ip = 0;
502  decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
503  decoder->ref_timestamp = buffer.ref_timestamp;
504  decoder->timestamp = 0;
505  decoder->have_tma = false;
506  decoder->state.trace_nr = buffer.trace_nr;
507  intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
508  decoder->ref_timestamp);
509  return -ENOLINK;
510  }
511 
512  return 0;
513 }
514 
515 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
516 {
517  if (!decoder->next_buf)
518  return intel_pt_get_data(decoder);
519 
520  decoder->buf = decoder->next_buf;
521  decoder->len = decoder->next_len;
522  decoder->next_buf = 0;
523  decoder->next_len = 0;
524  return 0;
525 }
526 
527 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
528 {
529  unsigned char *buf = decoder->temp_buf;
530  size_t old_len, len, n;
531  int ret;
532 
533  old_len = decoder->len;
534  len = decoder->len;
535  memcpy(buf, decoder->buf, len);
536 
537  ret = intel_pt_get_data(decoder);
538  if (ret) {
539  decoder->pos += old_len;
540  return ret < 0 ? ret : -EINVAL;
541  }
542 
543  n = INTEL_PT_PKT_MAX_SZ - len;
544  if (n > decoder->len)
545  n = decoder->len;
546  memcpy(buf + len, decoder->buf, n);
547  len += n;
548 
549  ret = intel_pt_get_packet(buf, len, &decoder->packet);
550  if (ret < (int)old_len) {
551  decoder->next_buf = decoder->buf;
552  decoder->next_len = decoder->len;
553  decoder->buf = buf;
554  decoder->len = old_len;
555  return intel_pt_bad_packet(decoder);
556  }
557 
558  decoder->next_buf = decoder->buf + (ret - old_len);
559  decoder->next_len = decoder->len - (ret - old_len);
560 
561  decoder->buf = buf;
562  decoder->len = ret;
563 
564  return ret;
565 }
566 
569  struct intel_pt_pkt packet;
570  uint64_t pos;
571  int pkt_len;
573  void *data;
574 };
575 
576 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
577 
578 /* Lookahead packets in current buffer */
580  intel_pt_pkt_cb_t cb, void *data)
581 {
582  struct intel_pt_pkt_info pkt_info;
583  const unsigned char *buf = decoder->buf;
584  size_t len = decoder->len;
585  int ret;
586 
587  pkt_info.decoder = decoder;
588  pkt_info.pos = decoder->pos;
589  pkt_info.pkt_len = decoder->pkt_step;
590  pkt_info.last_packet_type = decoder->last_packet_type;
591  pkt_info.data = data;
592 
593  while (1) {
594  do {
595  pkt_info.pos += pkt_info.pkt_len;
596  buf += pkt_info.pkt_len;
597  len -= pkt_info.pkt_len;
598 
599  if (!len)
601 
602  ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
603  if (!ret)
605  if (ret < 0)
606  return ret;
607 
608  pkt_info.pkt_len = ret;
609  } while (pkt_info.packet.type == INTEL_PT_PAD);
610 
611  ret = cb(&pkt_info);
612  if (ret)
613  return 0;
614 
615  pkt_info.last_packet_type = pkt_info.packet.type;
616  }
617 }
618 
620  uint64_t cycle_cnt;
621  unsigned int cbr;
622  uint32_t last_mtc;
623  uint64_t ctc_timestamp;
624  uint64_t ctc_delta;
625  uint64_t tsc_timestamp;
626  uint64_t timestamp;
627  bool have_tma;
629  bool from_mtc;
631 };
632 
633 /*
634  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
635  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
636  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
637  * packet by copying the missing bits from the current MTC assuming the least
638  * difference between the two, and that the current MTC comes after last_mtc.
639  */
640 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
641  uint32_t *last_mtc)
642 {
643  uint32_t first_missing_bit = 1U << (16 - mtc_shift);
644  uint32_t mask = ~(first_missing_bit - 1);
645 
646  *last_mtc |= mtc & mask;
647  if (*last_mtc >= mtc) {
648  *last_mtc -= first_missing_bit;
649  *last_mtc &= 0xff;
650  }
651 }
652 
653 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
654 {
655  struct intel_pt_decoder *decoder = pkt_info->decoder;
656  struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
657  uint64_t timestamp;
658  double cyc_to_tsc;
659  unsigned int cbr;
660  uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
661 
662  switch (pkt_info->packet.type) {
663  case INTEL_PT_TNT:
664  case INTEL_PT_TIP_PGE:
665  case INTEL_PT_TIP:
666  case INTEL_PT_FUP:
667  case INTEL_PT_PSB:
668  case INTEL_PT_PIP:
669  case INTEL_PT_MODE_EXEC:
670  case INTEL_PT_MODE_TSX:
671  case INTEL_PT_PSBEND:
672  case INTEL_PT_PAD:
673  case INTEL_PT_VMCS:
674  case INTEL_PT_MNT:
675  case INTEL_PT_PTWRITE:
676  case INTEL_PT_PTWRITE_IP:
677  return 0;
678 
679  case INTEL_PT_MTC:
680  if (!data->have_tma)
681  return 0;
682 
683  mtc = pkt_info->packet.payload;
684  if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
685  data->fixup_last_mtc = false;
686  intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
687  &data->last_mtc);
688  }
689  if (mtc > data->last_mtc)
690  mtc_delta = mtc - data->last_mtc;
691  else
692  mtc_delta = mtc + 256 - data->last_mtc;
693  data->ctc_delta += mtc_delta << decoder->mtc_shift;
694  data->last_mtc = mtc;
695 
696  if (decoder->tsc_ctc_mult) {
697  timestamp = data->ctc_timestamp +
698  data->ctc_delta * decoder->tsc_ctc_mult;
699  } else {
700  timestamp = data->ctc_timestamp +
701  multdiv(data->ctc_delta,
702  decoder->tsc_ctc_ratio_n,
703  decoder->tsc_ctc_ratio_d);
704  }
705 
706  if (timestamp < data->timestamp)
707  return 1;
708 
709  if (pkt_info->last_packet_type != INTEL_PT_CYC) {
710  data->timestamp = timestamp;
711  return 0;
712  }
713 
714  break;
715 
716  case INTEL_PT_TSC:
717  /*
718  * For now, do not support using TSC packets - refer
719  * intel_pt_calc_cyc_to_tsc().
720  */
721  if (data->from_mtc)
722  return 1;
723  timestamp = pkt_info->packet.payload |
724  (data->timestamp & (0xffULL << 56));
725  if (data->from_mtc && timestamp < data->timestamp &&
726  data->timestamp - timestamp < decoder->tsc_slip)
727  return 1;
728  if (timestamp < data->timestamp)
729  timestamp += (1ULL << 56);
730  if (pkt_info->last_packet_type != INTEL_PT_CYC) {
731  if (data->from_mtc)
732  return 1;
733  data->tsc_timestamp = timestamp;
734  data->timestamp = timestamp;
735  return 0;
736  }
737  break;
738 
739  case INTEL_PT_TMA:
740  if (data->from_mtc)
741  return 1;
742 
743  if (!decoder->tsc_ctc_ratio_d)
744  return 0;
745 
746  ctc = pkt_info->packet.payload;
747  fc = pkt_info->packet.count;
748  ctc_rem = ctc & decoder->ctc_rem_mask;
749 
750  data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
751 
752  data->ctc_timestamp = data->tsc_timestamp - fc;
753  if (decoder->tsc_ctc_mult) {
754  data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
755  } else {
756  data->ctc_timestamp -=
757  multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
758  decoder->tsc_ctc_ratio_d);
759  }
760 
761  data->ctc_delta = 0;
762  data->have_tma = true;
763  data->fixup_last_mtc = true;
764 
765  return 0;
766 
767  case INTEL_PT_CYC:
768  data->cycle_cnt += pkt_info->packet.payload;
769  return 0;
770 
771  case INTEL_PT_CBR:
772  cbr = pkt_info->packet.payload;
773  if (data->cbr && data->cbr != cbr)
774  return 1;
775  data->cbr = cbr;
776  data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
777  return 0;
778 
779  case INTEL_PT_TIP_PGD:
780  case INTEL_PT_TRACESTOP:
781  case INTEL_PT_EXSTOP:
782  case INTEL_PT_EXSTOP_IP:
783  case INTEL_PT_MWAIT:
784  case INTEL_PT_PWRE:
785  case INTEL_PT_PWRX:
786  case INTEL_PT_OVF:
787  case INTEL_PT_BAD: /* Does not happen */
788  default:
789  return 1;
790  }
791 
792  if (!data->cbr && decoder->cbr) {
793  data->cbr = decoder->cbr;
794  data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
795  }
796 
797  if (!data->cycle_cnt)
798  return 1;
799 
800  cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
801 
802  if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
803  cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
804  intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
805  cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
806  return 1;
807  }
808 
809  decoder->calc_cyc_to_tsc = cyc_to_tsc;
810  decoder->have_calc_cyc_to_tsc = true;
811 
812  if (data->cbr) {
813  intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
814  cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
815  } else {
816  intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
817  cyc_to_tsc, pkt_info->pos);
818  }
819 
820  return 1;
821 }
822 
823 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
824  bool from_mtc)
825 {
826  struct intel_pt_calc_cyc_to_tsc_info data = {
827  .cycle_cnt = 0,
828  .cbr = 0,
829  .last_mtc = decoder->last_mtc,
830  .ctc_timestamp = decoder->ctc_timestamp,
831  .ctc_delta = decoder->ctc_delta,
832  .tsc_timestamp = decoder->tsc_timestamp,
833  .timestamp = decoder->timestamp,
834  .have_tma = decoder->have_tma,
835  .fixup_last_mtc = decoder->fixup_last_mtc,
836  .from_mtc = from_mtc,
837  .cbr_cyc_to_tsc = 0,
838  };
839 
840  /*
841  * For now, do not support using TSC packets for at least the reasons:
842  * 1) timing might have stopped
843  * 2) TSC packets within PSB+ can slip against CYC packets
844  */
845  if (!from_mtc)
846  return;
847 
849 }
850 
851 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
852 {
853  int ret;
854 
855  decoder->last_packet_type = decoder->packet.type;
856 
857  do {
858  decoder->pos += decoder->pkt_step;
859  decoder->buf += decoder->pkt_step;
860  decoder->len -= decoder->pkt_step;
861 
862  if (!decoder->len) {
863  ret = intel_pt_get_next_data(decoder);
864  if (ret)
865  return ret;
866  }
867 
868  ret = intel_pt_get_packet(decoder->buf, decoder->len,
869  &decoder->packet);
870  if (ret == INTEL_PT_NEED_MORE_BYTES &&
871  decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
872  ret = intel_pt_get_split_packet(decoder);
873  if (ret < 0)
874  return ret;
875  }
876  if (ret <= 0)
877  return intel_pt_bad_packet(decoder);
878 
879  decoder->pkt_len = ret;
880  decoder->pkt_step = ret;
882  } while (decoder->packet.type == INTEL_PT_PAD);
883 
884  return 0;
885 }
886 
887 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
888 {
889  uint64_t timestamp, masked_timestamp;
890 
891  timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
892  masked_timestamp = timestamp & decoder->period_mask;
893  if (decoder->continuous_period) {
894  if (masked_timestamp != decoder->last_masked_timestamp)
895  return 1;
896  } else {
897  timestamp += 1;
898  masked_timestamp = timestamp & decoder->period_mask;
899  if (masked_timestamp != decoder->last_masked_timestamp) {
900  decoder->last_masked_timestamp = masked_timestamp;
901  decoder->continuous_period = true;
902  }
903  }
904  return decoder->period_ticks - (timestamp - masked_timestamp);
905 }
906 
907 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
908 {
909  switch (decoder->period_type) {
911  return decoder->period - decoder->period_insn_cnt;
913  return intel_pt_next_period(decoder);
915  case INTEL_PT_PERIOD_MTC:
916  default:
917  return 0;
918  }
919 }
920 
921 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
922 {
923  uint64_t timestamp, masked_timestamp;
924 
925  switch (decoder->period_type) {
927  decoder->period_insn_cnt = 0;
928  break;
930  timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
931  masked_timestamp = timestamp & decoder->period_mask;
932  decoder->last_masked_timestamp = masked_timestamp;
933  break;
935  case INTEL_PT_PERIOD_MTC:
936  default:
937  break;
938  }
939 
940  decoder->state.type |= INTEL_PT_INSTRUCTION;
941 }
942 
943 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
944  struct intel_pt_insn *intel_pt_insn, uint64_t ip)
945 {
946  uint64_t max_insn_cnt, insn_cnt = 0;
947  int err;
948 
949  if (!decoder->mtc_insn)
950  decoder->mtc_insn = true;
951 
952  max_insn_cnt = intel_pt_next_sample(decoder);
953 
954  err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
955  max_insn_cnt, decoder->data);
956 
957  decoder->tot_insn_cnt += insn_cnt;
958  decoder->timestamp_insn_cnt += insn_cnt;
959  decoder->sample_insn_cnt += insn_cnt;
960  decoder->period_insn_cnt += insn_cnt;
961 
962  if (err) {
963  decoder->no_progress = 0;
964  decoder->pkt_state = INTEL_PT_STATE_ERR2;
965  intel_pt_log_at("ERROR: Failed to get instruction",
966  decoder->ip);
967  if (err == -ENOENT)
968  return -ENOLINK;
969  return -EILSEQ;
970  }
971 
972  if (ip && decoder->ip == ip) {
973  err = -EAGAIN;
974  goto out;
975  }
976 
977  if (max_insn_cnt && insn_cnt >= max_insn_cnt)
978  intel_pt_sample_insn(decoder);
979 
980  if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
981  decoder->state.type = INTEL_PT_INSTRUCTION;
982  decoder->state.from_ip = decoder->ip;
983  decoder->state.to_ip = 0;
984  decoder->ip += intel_pt_insn->length;
985  err = INTEL_PT_RETURN;
986  goto out;
987  }
988 
989  if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
990  /* Zero-length calls are excluded */
991  if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
992  intel_pt_insn->rel) {
993  err = intel_pt_push(&decoder->stack, decoder->ip +
994  intel_pt_insn->length);
995  if (err)
996  goto out;
997  }
998  } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
999  decoder->ret_addr = intel_pt_pop(&decoder->stack);
1000  }
1001 
1002  if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1003  int cnt = decoder->no_progress++;
1004 
1005  decoder->state.from_ip = decoder->ip;
1006  decoder->ip += intel_pt_insn->length +
1007  intel_pt_insn->rel;
1008  decoder->state.to_ip = decoder->ip;
1009  err = INTEL_PT_RETURN;
1010 
1011  /*
1012  * Check for being stuck in a loop. This can happen if a
1013  * decoder error results in the decoder erroneously setting the
1014  * ip to an address that is itself in an infinite loop that
1015  * consumes no packets. When that happens, there must be an
1016  * unconditional branch.
1017  */
1018  if (cnt) {
1019  if (cnt == 1) {
1020  decoder->stuck_ip = decoder->state.to_ip;
1021  decoder->stuck_ip_prd = 1;
1022  decoder->stuck_ip_cnt = 1;
1023  } else if (cnt > INTEL_PT_MAX_LOOPS ||
1024  decoder->state.to_ip == decoder->stuck_ip) {
1025  intel_pt_log_at("ERROR: Never-ending loop",
1026  decoder->state.to_ip);
1028  err = -ELOOP;
1029  goto out;
1030  } else if (!--decoder->stuck_ip_cnt) {
1031  decoder->stuck_ip_prd += 1;
1032  decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1033  decoder->stuck_ip = decoder->state.to_ip;
1034  }
1035  }
1036  goto out_no_progress;
1037  }
1038 out:
1039  decoder->no_progress = 0;
1040 out_no_progress:
1041  decoder->state.insn_op = intel_pt_insn->op;
1042  decoder->state.insn_len = intel_pt_insn->length;
1043  memcpy(decoder->state.insn, intel_pt_insn->buf,
1045 
1046  if (decoder->tx_flags & INTEL_PT_IN_TX)
1047  decoder->state.flags |= INTEL_PT_IN_TX;
1048 
1049  return err;
1050 }
1051 
1052 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1053 {
1054  bool ret = false;
1055 
1056  if (decoder->set_fup_tx_flags) {
1057  decoder->set_fup_tx_flags = false;
1058  decoder->tx_flags = decoder->fup_tx_flags;
1059  decoder->state.type = INTEL_PT_TRANSACTION;
1060  decoder->state.from_ip = decoder->ip;
1061  decoder->state.to_ip = 0;
1062  decoder->state.flags = decoder->fup_tx_flags;
1063  return true;
1064  }
1065  if (decoder->set_fup_ptw) {
1066  decoder->set_fup_ptw = false;
1067  decoder->state.type = INTEL_PT_PTW;
1068  decoder->state.flags |= INTEL_PT_FUP_IP;
1069  decoder->state.from_ip = decoder->ip;
1070  decoder->state.to_ip = 0;
1071  decoder->state.ptw_payload = decoder->fup_ptw_payload;
1072  return true;
1073  }
1074  if (decoder->set_fup_mwait) {
1075  decoder->set_fup_mwait = false;
1076  decoder->state.type = INTEL_PT_MWAIT_OP;
1077  decoder->state.from_ip = decoder->ip;
1078  decoder->state.to_ip = 0;
1079  decoder->state.mwait_payload = decoder->fup_mwait_payload;
1080  ret = true;
1081  }
1082  if (decoder->set_fup_pwre) {
1083  decoder->set_fup_pwre = false;
1084  decoder->state.type |= INTEL_PT_PWR_ENTRY;
1085  decoder->state.type &= ~INTEL_PT_BRANCH;
1086  decoder->state.from_ip = decoder->ip;
1087  decoder->state.to_ip = 0;
1088  decoder->state.pwre_payload = decoder->fup_pwre_payload;
1089  ret = true;
1090  }
1091  if (decoder->set_fup_exstop) {
1092  decoder->set_fup_exstop = false;
1093  decoder->state.type |= INTEL_PT_EX_STOP;
1094  decoder->state.type &= ~INTEL_PT_BRANCH;
1095  decoder->state.flags |= INTEL_PT_FUP_IP;
1096  decoder->state.from_ip = decoder->ip;
1097  decoder->state.to_ip = 0;
1098  ret = true;
1099  }
1100  return ret;
1101 }
1102 
1103 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1104  struct intel_pt_insn *intel_pt_insn,
1105  uint64_t ip, int err)
1106 {
1107  return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1108  intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1109  ip == decoder->ip + intel_pt_insn->length;
1110 }
1111 
1112 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1113 {
1114  struct intel_pt_insn intel_pt_insn;
1115  uint64_t ip;
1116  int err;
1117 
1118  ip = decoder->last_ip;
1119 
1120  while (1) {
1121  err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1122  if (err == INTEL_PT_RETURN)
1123  return 0;
1124  if (err == -EAGAIN ||
1125  intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1126  if (intel_pt_fup_event(decoder))
1127  return 0;
1128  return -EAGAIN;
1129  }
1130  decoder->set_fup_tx_flags = false;
1131  if (err)
1132  return err;
1133 
1134  if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1135  intel_pt_log_at("ERROR: Unexpected indirect branch",
1136  decoder->ip);
1138  return -ENOENT;
1139  }
1140 
1141  if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1142  intel_pt_log_at("ERROR: Unexpected conditional branch",
1143  decoder->ip);
1145  return -ENOENT;
1146  }
1147 
1148  intel_pt_bug(decoder);
1149  }
1150 }
1151 
1152 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1153 {
1154  struct intel_pt_insn intel_pt_insn;
1155  int err;
1156 
1157  err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1158  if (err == INTEL_PT_RETURN &&
1159  decoder->pgd_ip &&
1160  decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1161  (decoder->state.type & INTEL_PT_BRANCH) &&
1162  decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1163  /* Unconditional branch leaving filter region */
1164  decoder->no_progress = 0;
1165  decoder->pge = false;
1166  decoder->continuous_period = false;
1167  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1168  decoder->state.to_ip = 0;
1169  return 0;
1170  }
1171  if (err == INTEL_PT_RETURN)
1172  return 0;
1173  if (err)
1174  return err;
1175 
1176  if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1177  if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1178  decoder->pge = false;
1179  decoder->continuous_period = false;
1180  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1181  decoder->state.from_ip = decoder->ip;
1182  decoder->state.to_ip = 0;
1183  if (decoder->packet.count != 0)
1184  decoder->ip = decoder->last_ip;
1185  } else {
1186  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1187  decoder->state.from_ip = decoder->ip;
1188  if (decoder->packet.count == 0) {
1189  decoder->state.to_ip = 0;
1190  } else {
1191  decoder->state.to_ip = decoder->last_ip;
1192  decoder->ip = decoder->last_ip;
1193  }
1194  }
1195  return 0;
1196  }
1197 
1198  if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1199  uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1200  intel_pt_insn.rel;
1201 
1202  if (decoder->pgd_ip &&
1203  decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1204  decoder->pgd_ip(to_ip, decoder->data)) {
1205  /* Conditional branch leaving filter region */
1206  decoder->pge = false;
1207  decoder->continuous_period = false;
1208  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1209  decoder->ip = to_ip;
1210  decoder->state.from_ip = decoder->ip;
1211  decoder->state.to_ip = 0;
1212  return 0;
1213  }
1214  intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1215  decoder->ip);
1217  return -ENOENT;
1218  }
1219 
1220  return intel_pt_bug(decoder);
1221 }
1222 
1223 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1224 {
1225  struct intel_pt_insn intel_pt_insn;
1226  int err;
1227 
1228  while (1) {
1229  err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1230  if (err == INTEL_PT_RETURN)
1231  return 0;
1232  if (err)
1233  return err;
1234 
1235  if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1236  if (!decoder->return_compression) {
1237  intel_pt_log_at("ERROR: RET when expecting conditional branch",
1238  decoder->ip);
1239  decoder->pkt_state = INTEL_PT_STATE_ERR3;
1240  return -ENOENT;
1241  }
1242  if (!decoder->ret_addr) {
1243  intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1244  decoder->ip);
1245  decoder->pkt_state = INTEL_PT_STATE_ERR3;
1246  return -ENOENT;
1247  }
1248  if (!(decoder->tnt.payload & BIT63)) {
1249  intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1250  decoder->ip);
1251  decoder->pkt_state = INTEL_PT_STATE_ERR3;
1252  return -ENOENT;
1253  }
1254  decoder->tnt.count -= 1;
1255  if (!decoder->tnt.count)
1256  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1257  decoder->tnt.payload <<= 1;
1258  decoder->state.from_ip = decoder->ip;
1259  decoder->ip = decoder->ret_addr;
1260  decoder->state.to_ip = decoder->ip;
1261  return 0;
1262  }
1263 
1264  if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1265  /* Handle deferred TIPs */
1266  err = intel_pt_get_next_packet(decoder);
1267  if (err)
1268  return err;
1269  if (decoder->packet.type != INTEL_PT_TIP ||
1270  decoder->packet.count == 0) {
1271  intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1272  decoder->ip);
1273  decoder->pkt_state = INTEL_PT_STATE_ERR3;
1274  decoder->pkt_step = 0;
1275  return -ENOENT;
1276  }
1277  intel_pt_set_last_ip(decoder);
1278  decoder->state.from_ip = decoder->ip;
1279  decoder->state.to_ip = decoder->last_ip;
1280  decoder->ip = decoder->last_ip;
1281  return 0;
1282  }
1283 
1284  if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1285  decoder->tnt.count -= 1;
1286  if (!decoder->tnt.count)
1287  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1288  if (decoder->tnt.payload & BIT63) {
1289  decoder->tnt.payload <<= 1;
1290  decoder->state.from_ip = decoder->ip;
1291  decoder->ip += intel_pt_insn.length +
1292  intel_pt_insn.rel;
1293  decoder->state.to_ip = decoder->ip;
1294  return 0;
1295  }
1296  /* Instruction sample for a non-taken branch */
1297  if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1298  decoder->tnt.payload <<= 1;
1299  decoder->state.type = INTEL_PT_INSTRUCTION;
1300  decoder->state.from_ip = decoder->ip;
1301  decoder->state.to_ip = 0;
1302  decoder->ip += intel_pt_insn.length;
1303  return 0;
1304  }
1305  decoder->ip += intel_pt_insn.length;
1306  if (!decoder->tnt.count)
1307  return -EAGAIN;
1308  decoder->tnt.payload <<= 1;
1309  continue;
1310  }
1311 
1312  return intel_pt_bug(decoder);
1313  }
1314 }
1315 
1316 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1317 {
1318  unsigned int fup_tx_flags;
1319  int err;
1320 
1321  fup_tx_flags = decoder->packet.payload &
1323  err = intel_pt_get_next_packet(decoder);
1324  if (err)
1325  return err;
1326  if (decoder->packet.type == INTEL_PT_FUP) {
1327  decoder->fup_tx_flags = fup_tx_flags;
1328  decoder->set_fup_tx_flags = true;
1329  if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1330  *no_tip = true;
1331  } else {
1332  intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1333  decoder->pos);
1334  intel_pt_update_in_tx(decoder);
1335  }
1336  return 0;
1337 }
1338 
1339 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1340 {
1341  uint64_t timestamp;
1342 
1343  decoder->have_tma = false;
1344 
1345  if (decoder->ref_timestamp) {
1346  timestamp = decoder->packet.payload |
1347  (decoder->ref_timestamp & (0xffULL << 56));
1348  if (timestamp < decoder->ref_timestamp) {
1349  if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1350  timestamp += (1ULL << 56);
1351  } else {
1352  if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1353  timestamp -= (1ULL << 56);
1354  }
1355  decoder->tsc_timestamp = timestamp;
1356  decoder->timestamp = timestamp;
1357  decoder->ref_timestamp = 0;
1358  decoder->timestamp_insn_cnt = 0;
1359  } else if (decoder->timestamp) {
1360  timestamp = decoder->packet.payload |
1361  (decoder->timestamp & (0xffULL << 56));
1362  decoder->tsc_timestamp = timestamp;
1363  if (timestamp < decoder->timestamp &&
1364  decoder->timestamp - timestamp < decoder->tsc_slip) {
1365  intel_pt_log_to("Suppressing backwards timestamp",
1366  timestamp);
1367  timestamp = decoder->timestamp;
1368  }
1369  if (timestamp < decoder->timestamp) {
1370  intel_pt_log_to("Wraparound timestamp", timestamp);
1371  timestamp += (1ULL << 56);
1372  decoder->tsc_timestamp = timestamp;
1373  }
1374  decoder->timestamp = timestamp;
1375  decoder->timestamp_insn_cnt = 0;
1376  }
1377 
1378  if (decoder->last_packet_type == INTEL_PT_CYC) {
1379  decoder->cyc_ref_timestamp = decoder->timestamp;
1380  decoder->cycle_cnt = 0;
1381  decoder->have_calc_cyc_to_tsc = false;
1382  intel_pt_calc_cyc_to_tsc(decoder, false);
1383  }
1384 
1385  intel_pt_log_to("Setting timestamp", decoder->timestamp);
1386 }
1387 
1388 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1389 {
1390  intel_pt_log("ERROR: Buffer overflow\n");
1391  intel_pt_clear_tx_flags(decoder);
1392  decoder->cbr = 0;
1393  decoder->timestamp_insn_cnt = 0;
1395  decoder->overflow = true;
1396  return -EOVERFLOW;
1397 }
1398 
1399 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1400 {
1401  uint32_t ctc = decoder->packet.payload;
1402  uint32_t fc = decoder->packet.count;
1403  uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1404 
1405  if (!decoder->tsc_ctc_ratio_d)
1406  return;
1407 
1408  decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1409  decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1410  if (decoder->tsc_ctc_mult) {
1411  decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1412  } else {
1413  decoder->ctc_timestamp -= multdiv(ctc_rem,
1414  decoder->tsc_ctc_ratio_n,
1415  decoder->tsc_ctc_ratio_d);
1416  }
1417  decoder->ctc_delta = 0;
1418  decoder->have_tma = true;
1419  decoder->fixup_last_mtc = true;
1420  intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1421  decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1422 }
1423 
1424 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1425 {
1426  uint64_t timestamp;
1427  uint32_t mtc, mtc_delta;
1428 
1429  if (!decoder->have_tma)
1430  return;
1431 
1432  mtc = decoder->packet.payload;
1433 
1434  if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1435  decoder->fixup_last_mtc = false;
1436  intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1437  &decoder->last_mtc);
1438  }
1439 
1440  if (mtc > decoder->last_mtc)
1441  mtc_delta = mtc - decoder->last_mtc;
1442  else
1443  mtc_delta = mtc + 256 - decoder->last_mtc;
1444 
1445  decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1446 
1447  if (decoder->tsc_ctc_mult) {
1448  timestamp = decoder->ctc_timestamp +
1449  decoder->ctc_delta * decoder->tsc_ctc_mult;
1450  } else {
1451  timestamp = decoder->ctc_timestamp +
1452  multdiv(decoder->ctc_delta,
1453  decoder->tsc_ctc_ratio_n,
1454  decoder->tsc_ctc_ratio_d);
1455  }
1456 
1457  if (timestamp < decoder->timestamp)
1458  intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1459  timestamp, decoder->timestamp);
1460  else
1461  decoder->timestamp = timestamp;
1462 
1463  decoder->timestamp_insn_cnt = 0;
1464  decoder->last_mtc = mtc;
1465 
1466  if (decoder->last_packet_type == INTEL_PT_CYC) {
1467  decoder->cyc_ref_timestamp = decoder->timestamp;
1468  decoder->cycle_cnt = 0;
1469  decoder->have_calc_cyc_to_tsc = false;
1470  intel_pt_calc_cyc_to_tsc(decoder, true);
1471  }
1472 }
1473 
1474 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1475 {
1476  unsigned int cbr = decoder->packet.payload & 0xff;
1477 
1478  decoder->cbr_payload = decoder->packet.payload;
1479 
1480  if (decoder->cbr == cbr)
1481  return;
1482 
1483  decoder->cbr = cbr;
1484  decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1485 }
1486 
1487 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1488 {
1489  uint64_t timestamp = decoder->cyc_ref_timestamp;
1490 
1491  decoder->have_cyc = true;
1492 
1493  decoder->cycle_cnt += decoder->packet.payload;
1494 
1495  if (!decoder->cyc_ref_timestamp)
1496  return;
1497 
1498  if (decoder->have_calc_cyc_to_tsc)
1499  timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1500  else if (decoder->cbr)
1501  timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1502  else
1503  return;
1504 
1505  if (timestamp < decoder->timestamp)
1506  intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1507  timestamp, decoder->timestamp);
1508  else
1509  decoder->timestamp = timestamp;
1510 
1511  decoder->timestamp_insn_cnt = 0;
1512 }
1513 
1514 /* Walk PSB+ packets when already in sync. */
1515 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1516 {
1517  int err;
1518 
1519  while (1) {
1520  err = intel_pt_get_next_packet(decoder);
1521  if (err)
1522  return err;
1523 
1524  switch (decoder->packet.type) {
1525  case INTEL_PT_PSBEND:
1526  return 0;
1527 
1528  case INTEL_PT_TIP_PGD:
1529  case INTEL_PT_TIP_PGE:
1530  case INTEL_PT_TIP:
1531  case INTEL_PT_TNT:
1532  case INTEL_PT_TRACESTOP:
1533  case INTEL_PT_BAD:
1534  case INTEL_PT_PSB:
1535  case INTEL_PT_PTWRITE:
1536  case INTEL_PT_PTWRITE_IP:
1537  case INTEL_PT_EXSTOP:
1538  case INTEL_PT_EXSTOP_IP:
1539  case INTEL_PT_MWAIT:
1540  case INTEL_PT_PWRE:
1541  case INTEL_PT_PWRX:
1542  decoder->have_tma = false;
1543  intel_pt_log("ERROR: Unexpected packet\n");
1544  return -EAGAIN;
1545 
1546  case INTEL_PT_OVF:
1547  return intel_pt_overflow(decoder);
1548 
1549  case INTEL_PT_TSC:
1550  intel_pt_calc_tsc_timestamp(decoder);
1551  break;
1552 
1553  case INTEL_PT_TMA:
1554  intel_pt_calc_tma(decoder);
1555  break;
1556 
1557  case INTEL_PT_CBR:
1558  intel_pt_calc_cbr(decoder);
1559  break;
1560 
1561  case INTEL_PT_MODE_EXEC:
1562  decoder->exec_mode = decoder->packet.payload;
1563  break;
1564 
1565  case INTEL_PT_PIP:
1566  decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1567  break;
1568 
1569  case INTEL_PT_FUP:
1570  decoder->pge = true;
1571  if (decoder->packet.count)
1572  intel_pt_set_last_ip(decoder);
1573  break;
1574 
1575  case INTEL_PT_MODE_TSX:
1576  intel_pt_update_in_tx(decoder);
1577  break;
1578 
1579  case INTEL_PT_MTC:
1580  intel_pt_calc_mtc_timestamp(decoder);
1581  if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1582  decoder->state.type |= INTEL_PT_INSTRUCTION;
1583  break;
1584 
1585  case INTEL_PT_CYC:
1586  case INTEL_PT_VMCS:
1587  case INTEL_PT_MNT:
1588  case INTEL_PT_PAD:
1589  default:
1590  break;
1591  }
1592  }
1593 }
1594 
1595 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1596 {
1597  int err;
1598 
1599  if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1600  decoder->tx_flags = 0;
1601  decoder->state.flags &= ~INTEL_PT_IN_TX;
1602  decoder->state.flags |= INTEL_PT_ABORT_TX;
1603  } else {
1604  decoder->state.flags |= INTEL_PT_ASYNC;
1605  }
1606 
1607  while (1) {
1608  err = intel_pt_get_next_packet(decoder);
1609  if (err)
1610  return err;
1611 
1612  switch (decoder->packet.type) {
1613  case INTEL_PT_TNT:
1614  case INTEL_PT_FUP:
1615  case INTEL_PT_TRACESTOP:
1616  case INTEL_PT_PSB:
1617  case INTEL_PT_TSC:
1618  case INTEL_PT_TMA:
1619  case INTEL_PT_MODE_TSX:
1620  case INTEL_PT_BAD:
1621  case INTEL_PT_PSBEND:
1622  case INTEL_PT_PTWRITE:
1623  case INTEL_PT_PTWRITE_IP:
1624  case INTEL_PT_EXSTOP:
1625  case INTEL_PT_EXSTOP_IP:
1626  case INTEL_PT_MWAIT:
1627  case INTEL_PT_PWRE:
1628  case INTEL_PT_PWRX:
1629  intel_pt_log("ERROR: Missing TIP after FUP\n");
1630  decoder->pkt_state = INTEL_PT_STATE_ERR3;
1631  decoder->pkt_step = 0;
1632  return -ENOENT;
1633 
1634  case INTEL_PT_CBR:
1635  intel_pt_calc_cbr(decoder);
1636  break;
1637 
1638  case INTEL_PT_OVF:
1639  return intel_pt_overflow(decoder);
1640 
1641  case INTEL_PT_TIP_PGD:
1642  decoder->state.from_ip = decoder->ip;
1643  decoder->state.to_ip = 0;
1644  if (decoder->packet.count != 0) {
1645  intel_pt_set_ip(decoder);
1646  intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1647  decoder->ip);
1648  }
1649  decoder->pge = false;
1650  decoder->continuous_period = false;
1651  return 0;
1652 
1653  case INTEL_PT_TIP_PGE:
1654  decoder->pge = true;
1655  intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1656  decoder->ip);
1657  decoder->state.from_ip = 0;
1658  if (decoder->packet.count == 0) {
1659  decoder->state.to_ip = 0;
1660  } else {
1661  intel_pt_set_ip(decoder);
1662  decoder->state.to_ip = decoder->ip;
1663  }
1664  return 0;
1665 
1666  case INTEL_PT_TIP:
1667  decoder->state.from_ip = decoder->ip;
1668  if (decoder->packet.count == 0) {
1669  decoder->state.to_ip = 0;
1670  } else {
1671  intel_pt_set_ip(decoder);
1672  decoder->state.to_ip = decoder->ip;
1673  }
1674  return 0;
1675 
1676  case INTEL_PT_PIP:
1677  decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1678  break;
1679 
1680  case INTEL_PT_MTC:
1681  intel_pt_calc_mtc_timestamp(decoder);
1682  if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1683  decoder->state.type |= INTEL_PT_INSTRUCTION;
1684  break;
1685 
1686  case INTEL_PT_CYC:
1687  intel_pt_calc_cyc_timestamp(decoder);
1688  break;
1689 
1690  case INTEL_PT_MODE_EXEC:
1691  decoder->exec_mode = decoder->packet.payload;
1692  break;
1693 
1694  case INTEL_PT_VMCS:
1695  case INTEL_PT_MNT:
1696  case INTEL_PT_PAD:
1697  break;
1698 
1699  default:
1700  return intel_pt_bug(decoder);
1701  }
1702  }
1703 }
1704 
1705 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1706 {
1707  bool no_tip = false;
1708  int err;
1709 
1710  while (1) {
1711  err = intel_pt_get_next_packet(decoder);
1712  if (err)
1713  return err;
1714 next:
1715  switch (decoder->packet.type) {
1716  case INTEL_PT_TNT:
1717  if (!decoder->packet.count)
1718  break;
1719  decoder->tnt = decoder->packet;
1720  decoder->pkt_state = INTEL_PT_STATE_TNT;
1721  err = intel_pt_walk_tnt(decoder);
1722  if (err == -EAGAIN)
1723  break;
1724  return err;
1725 
1726  case INTEL_PT_TIP_PGD:
1727  if (decoder->packet.count != 0)
1728  intel_pt_set_last_ip(decoder);
1729  decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1730  return intel_pt_walk_tip(decoder);
1731 
1732  case INTEL_PT_TIP_PGE: {
1733  decoder->pge = true;
1734  if (decoder->packet.count == 0) {
1735  intel_pt_log_at("Skipping zero TIP.PGE",
1736  decoder->pos);
1737  break;
1738  }
1739  intel_pt_set_ip(decoder);
1740  decoder->state.from_ip = 0;
1741  decoder->state.to_ip = decoder->ip;
1742  return 0;
1743  }
1744 
1745  case INTEL_PT_OVF:
1746  return intel_pt_overflow(decoder);
1747 
1748  case INTEL_PT_TIP:
1749  if (decoder->packet.count != 0)
1750  intel_pt_set_last_ip(decoder);
1751  decoder->pkt_state = INTEL_PT_STATE_TIP;
1752  return intel_pt_walk_tip(decoder);
1753 
1754  case INTEL_PT_FUP:
1755  if (decoder->packet.count == 0) {
1756  intel_pt_log_at("Skipping zero FUP",
1757  decoder->pos);
1758  no_tip = false;
1759  break;
1760  }
1761  intel_pt_set_last_ip(decoder);
1762  if (!decoder->branch_enable) {
1763  decoder->ip = decoder->last_ip;
1764  if (intel_pt_fup_event(decoder))
1765  return 0;
1766  no_tip = false;
1767  break;
1768  }
1769  if (decoder->set_fup_mwait)
1770  no_tip = true;
1771  err = intel_pt_walk_fup(decoder);
1772  if (err != -EAGAIN) {
1773  if (err)
1774  return err;
1775  if (no_tip)
1776  decoder->pkt_state =
1778  else
1779  decoder->pkt_state = INTEL_PT_STATE_FUP;
1780  return 0;
1781  }
1782  if (no_tip) {
1783  no_tip = false;
1784  break;
1785  }
1786  return intel_pt_walk_fup_tip(decoder);
1787 
1788  case INTEL_PT_TRACESTOP:
1789  decoder->pge = false;
1790  decoder->continuous_period = false;
1791  intel_pt_clear_tx_flags(decoder);
1792  decoder->have_tma = false;
1793  break;
1794 
1795  case INTEL_PT_PSB:
1796  decoder->last_ip = 0;
1797  decoder->have_last_ip = true;
1798  intel_pt_clear_stack(&decoder->stack);
1799  err = intel_pt_walk_psbend(decoder);
1800  if (err == -EAGAIN)
1801  goto next;
1802  if (err)
1803  return err;
1804  break;
1805 
1806  case INTEL_PT_PIP:
1807  decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1808  break;
1809 
1810  case INTEL_PT_MTC:
1811  intel_pt_calc_mtc_timestamp(decoder);
1812  if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1813  break;
1814  /*
1815  * Ensure that there has been an instruction since the
1816  * last MTC.
1817  */
1818  if (!decoder->mtc_insn)
1819  break;
1820  decoder->mtc_insn = false;
1821  /* Ensure that there is a timestamp */
1822  if (!decoder->timestamp)
1823  break;
1824  decoder->state.type = INTEL_PT_INSTRUCTION;
1825  decoder->state.from_ip = decoder->ip;
1826  decoder->state.to_ip = 0;
1827  decoder->mtc_insn = false;
1828  return 0;
1829 
1830  case INTEL_PT_TSC:
1831  intel_pt_calc_tsc_timestamp(decoder);
1832  break;
1833 
1834  case INTEL_PT_TMA:
1835  intel_pt_calc_tma(decoder);
1836  break;
1837 
1838  case INTEL_PT_CYC:
1839  intel_pt_calc_cyc_timestamp(decoder);
1840  break;
1841 
1842  case INTEL_PT_CBR:
1843  intel_pt_calc_cbr(decoder);
1844  if (!decoder->branch_enable &&
1845  decoder->cbr != decoder->cbr_seen) {
1846  decoder->cbr_seen = decoder->cbr;
1847  decoder->state.type = INTEL_PT_CBR_CHG;
1848  decoder->state.from_ip = decoder->ip;
1849  decoder->state.to_ip = 0;
1850  decoder->state.cbr_payload =
1851  decoder->packet.payload;
1852  return 0;
1853  }
1854  break;
1855 
1856  case INTEL_PT_MODE_EXEC:
1857  decoder->exec_mode = decoder->packet.payload;
1858  break;
1859 
1860  case INTEL_PT_MODE_TSX:
1861  /* MODE_TSX need not be followed by FUP */
1862  if (!decoder->pge) {
1863  intel_pt_update_in_tx(decoder);
1864  break;
1865  }
1866  err = intel_pt_mode_tsx(decoder, &no_tip);
1867  if (err)
1868  return err;
1869  goto next;
1870 
1871  case INTEL_PT_BAD: /* Does not happen */
1872  return intel_pt_bug(decoder);
1873 
1874  case INTEL_PT_PSBEND:
1875  case INTEL_PT_VMCS:
1876  case INTEL_PT_MNT:
1877  case INTEL_PT_PAD:
1878  break;
1879 
1880  case INTEL_PT_PTWRITE_IP:
1881  decoder->fup_ptw_payload = decoder->packet.payload;
1882  err = intel_pt_get_next_packet(decoder);
1883  if (err)
1884  return err;
1885  if (decoder->packet.type == INTEL_PT_FUP) {
1886  decoder->set_fup_ptw = true;
1887  no_tip = true;
1888  } else {
1889  intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1890  decoder->pos);
1891  }
1892  goto next;
1893 
1894  case INTEL_PT_PTWRITE:
1895  decoder->state.type = INTEL_PT_PTW;
1896  decoder->state.from_ip = decoder->ip;
1897  decoder->state.to_ip = 0;
1898  decoder->state.ptw_payload = decoder->packet.payload;
1899  return 0;
1900 
1901  case INTEL_PT_MWAIT:
1902  decoder->fup_mwait_payload = decoder->packet.payload;
1903  decoder->set_fup_mwait = true;
1904  break;
1905 
1906  case INTEL_PT_PWRE:
1907  if (decoder->set_fup_mwait) {
1908  decoder->fup_pwre_payload =
1909  decoder->packet.payload;
1910  decoder->set_fup_pwre = true;
1911  break;
1912  }
1913  decoder->state.type = INTEL_PT_PWR_ENTRY;
1914  decoder->state.from_ip = decoder->ip;
1915  decoder->state.to_ip = 0;
1916  decoder->state.pwrx_payload = decoder->packet.payload;
1917  return 0;
1918 
1919  case INTEL_PT_EXSTOP_IP:
1920  err = intel_pt_get_next_packet(decoder);
1921  if (err)
1922  return err;
1923  if (decoder->packet.type == INTEL_PT_FUP) {
1924  decoder->set_fup_exstop = true;
1925  no_tip = true;
1926  } else {
1927  intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1928  decoder->pos);
1929  }
1930  goto next;
1931 
1932  case INTEL_PT_EXSTOP:
1933  decoder->state.type = INTEL_PT_EX_STOP;
1934  decoder->state.from_ip = decoder->ip;
1935  decoder->state.to_ip = 0;
1936  return 0;
1937 
1938  case INTEL_PT_PWRX:
1939  decoder->state.type = INTEL_PT_PWR_EXIT;
1940  decoder->state.from_ip = decoder->ip;
1941  decoder->state.to_ip = 0;
1942  decoder->state.pwrx_payload = decoder->packet.payload;
1943  return 0;
1944 
1945  default:
1946  return intel_pt_bug(decoder);
1947  }
1948  }
1949 }
1950 
1951 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1952 {
1953  return decoder->packet.count &&
1954  (decoder->have_last_ip || decoder->packet.count == 3 ||
1955  decoder->packet.count == 6);
1956 }
1957 
1958 /* Walk PSB+ packets to get in sync. */
1959 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1960 {
1961  int err;
1962 
1963  while (1) {
1964  err = intel_pt_get_next_packet(decoder);
1965  if (err)
1966  return err;
1967 
1968  switch (decoder->packet.type) {
1969  case INTEL_PT_TIP_PGD:
1970  decoder->continuous_period = false;
1971  __fallthrough;
1972  case INTEL_PT_TIP_PGE:
1973  case INTEL_PT_TIP:
1974  case INTEL_PT_PTWRITE:
1975  case INTEL_PT_PTWRITE_IP:
1976  case INTEL_PT_EXSTOP:
1977  case INTEL_PT_EXSTOP_IP:
1978  case INTEL_PT_MWAIT:
1979  case INTEL_PT_PWRE:
1980  case INTEL_PT_PWRX:
1981  intel_pt_log("ERROR: Unexpected packet\n");
1982  return -ENOENT;
1983 
1984  case INTEL_PT_FUP:
1985  decoder->pge = true;
1986  if (intel_pt_have_ip(decoder)) {
1987  uint64_t current_ip = decoder->ip;
1988 
1989  intel_pt_set_ip(decoder);
1990  if (current_ip)
1991  intel_pt_log_to("Setting IP",
1992  decoder->ip);
1993  }
1994  break;
1995 
1996  case INTEL_PT_MTC:
1997  intel_pt_calc_mtc_timestamp(decoder);
1998  break;
1999 
2000  case INTEL_PT_TSC:
2001  intel_pt_calc_tsc_timestamp(decoder);
2002  break;
2003 
2004  case INTEL_PT_TMA:
2005  intel_pt_calc_tma(decoder);
2006  break;
2007 
2008  case INTEL_PT_CYC:
2009  intel_pt_calc_cyc_timestamp(decoder);
2010  break;
2011 
2012  case INTEL_PT_CBR:
2013  intel_pt_calc_cbr(decoder);
2014  break;
2015 
2016  case INTEL_PT_PIP:
2017  decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2018  break;
2019 
2020  case INTEL_PT_MODE_EXEC:
2021  decoder->exec_mode = decoder->packet.payload;
2022  break;
2023 
2024  case INTEL_PT_MODE_TSX:
2025  intel_pt_update_in_tx(decoder);
2026  break;
2027 
2028  case INTEL_PT_TRACESTOP:
2029  decoder->pge = false;
2030  decoder->continuous_period = false;
2031  intel_pt_clear_tx_flags(decoder);
2032  __fallthrough;
2033 
2034  case INTEL_PT_TNT:
2035  decoder->have_tma = false;
2036  intel_pt_log("ERROR: Unexpected packet\n");
2037  if (decoder->ip)
2038  decoder->pkt_state = INTEL_PT_STATE_ERR4;
2039  else
2040  decoder->pkt_state = INTEL_PT_STATE_ERR3;
2041  return -ENOENT;
2042 
2043  case INTEL_PT_BAD: /* Does not happen */
2044  return intel_pt_bug(decoder);
2045 
2046  case INTEL_PT_OVF:
2047  return intel_pt_overflow(decoder);
2048 
2049  case INTEL_PT_PSBEND:
2050  return 0;
2051 
2052  case INTEL_PT_PSB:
2053  case INTEL_PT_VMCS:
2054  case INTEL_PT_MNT:
2055  case INTEL_PT_PAD:
2056  default:
2057  break;
2058  }
2059  }
2060 }
2061 
2062 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2063 {
2064  int err;
2065 
2066  while (1) {
2067  err = intel_pt_get_next_packet(decoder);
2068  if (err)
2069  return err;
2070 
2071  switch (decoder->packet.type) {
2072  case INTEL_PT_TIP_PGD:
2073  decoder->continuous_period = false;
2074  __fallthrough;
2075  case INTEL_PT_TIP_PGE:
2076  case INTEL_PT_TIP:
2077  decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
2078  if (intel_pt_have_ip(decoder))
2079  intel_pt_set_ip(decoder);
2080  if (decoder->ip)
2081  return 0;
2082  break;
2083 
2084  case INTEL_PT_FUP:
2085  if (intel_pt_have_ip(decoder))
2086  intel_pt_set_ip(decoder);
2087  if (decoder->ip)
2088  return 0;
2089  break;
2090 
2091  case INTEL_PT_MTC:
2092  intel_pt_calc_mtc_timestamp(decoder);
2093  break;
2094 
2095  case INTEL_PT_TSC:
2096  intel_pt_calc_tsc_timestamp(decoder);
2097  break;
2098 
2099  case INTEL_PT_TMA:
2100  intel_pt_calc_tma(decoder);
2101  break;
2102 
2103  case INTEL_PT_CYC:
2104  intel_pt_calc_cyc_timestamp(decoder);
2105  break;
2106 
2107  case INTEL_PT_CBR:
2108  intel_pt_calc_cbr(decoder);
2109  break;
2110 
2111  case INTEL_PT_PIP:
2112  decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2113  break;
2114 
2115  case INTEL_PT_MODE_EXEC:
2116  decoder->exec_mode = decoder->packet.payload;
2117  break;
2118 
2119  case INTEL_PT_MODE_TSX:
2120  intel_pt_update_in_tx(decoder);
2121  break;
2122 
2123  case INTEL_PT_OVF:
2124  return intel_pt_overflow(decoder);
2125 
2126  case INTEL_PT_BAD: /* Does not happen */
2127  return intel_pt_bug(decoder);
2128 
2129  case INTEL_PT_TRACESTOP:
2130  decoder->pge = false;
2131  decoder->continuous_period = false;
2132  intel_pt_clear_tx_flags(decoder);
2133  decoder->have_tma = false;
2134  break;
2135 
2136  case INTEL_PT_PSB:
2137  decoder->last_ip = 0;
2138  decoder->have_last_ip = true;
2139  intel_pt_clear_stack(&decoder->stack);
2140  err = intel_pt_walk_psb(decoder);
2141  if (err)
2142  return err;
2143  if (decoder->ip) {
2144  /* Do not have a sample */
2145  decoder->state.type = 0;
2146  return 0;
2147  }
2148  break;
2149 
2150  case INTEL_PT_TNT:
2151  case INTEL_PT_PSBEND:
2152  case INTEL_PT_VMCS:
2153  case INTEL_PT_MNT:
2154  case INTEL_PT_PAD:
2155  case INTEL_PT_PTWRITE:
2156  case INTEL_PT_PTWRITE_IP:
2157  case INTEL_PT_EXSTOP:
2158  case INTEL_PT_EXSTOP_IP:
2159  case INTEL_PT_MWAIT:
2160  case INTEL_PT_PWRE:
2161  case INTEL_PT_PWRX:
2162  default:
2163  break;
2164  }
2165  }
2166 }
2167 
2168 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2169 {
2170  int err;
2171 
2172  decoder->set_fup_tx_flags = false;
2173  decoder->set_fup_ptw = false;
2174  decoder->set_fup_mwait = false;
2175  decoder->set_fup_pwre = false;
2176  decoder->set_fup_exstop = false;
2177 
2178  if (!decoder->branch_enable) {
2179  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2180  decoder->overflow = false;
2181  decoder->state.type = 0; /* Do not have a sample */
2182  return 0;
2183  }
2184 
2185  intel_pt_log("Scanning for full IP\n");
2186  err = intel_pt_walk_to_ip(decoder);
2187  if (err)
2188  return err;
2189 
2190  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2191  decoder->overflow = false;
2192 
2193  decoder->state.from_ip = 0;
2194  decoder->state.to_ip = decoder->ip;
2195  intel_pt_log_to("Setting IP", decoder->ip);
2196 
2197  return 0;
2198 }
2199 
2200 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2201 {
2202  const unsigned char *end = decoder->buf + decoder->len;
2203  size_t i;
2204 
2205  for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2206  if (i > decoder->len)
2207  continue;
2208  if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2209  return i;
2210  }
2211  return 0;
2212 }
2213 
2214 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2215 {
2216  size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2217  const char *psb = INTEL_PT_PSB_STR;
2218 
2219  if (rest_psb > decoder->len ||
2220  memcmp(decoder->buf, psb + part_psb, rest_psb))
2221  return 0;
2222 
2223  return rest_psb;
2224 }
2225 
2226 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2227  int part_psb)
2228 {
2229  int rest_psb, ret;
2230 
2231  decoder->pos += decoder->len;
2232  decoder->len = 0;
2233 
2234  ret = intel_pt_get_next_data(decoder);
2235  if (ret)
2236  return ret;
2237 
2238  rest_psb = intel_pt_rest_psb(decoder, part_psb);
2239  if (!rest_psb)
2240  return 0;
2241 
2242  decoder->pos -= part_psb;
2243  decoder->next_buf = decoder->buf + rest_psb;
2244  decoder->next_len = decoder->len - rest_psb;
2246  decoder->buf = decoder->temp_buf;
2247  decoder->len = INTEL_PT_PSB_LEN;
2248 
2249  return 0;
2250 }
2251 
2252 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2253 {
2254  unsigned char *next;
2255  int ret;
2256 
2257  intel_pt_log("Scanning for PSB\n");
2258  while (1) {
2259  if (!decoder->len) {
2260  ret = intel_pt_get_next_data(decoder);
2261  if (ret)
2262  return ret;
2263  }
2264 
2265  next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2267  if (!next) {
2268  int part_psb;
2269 
2270  part_psb = intel_pt_part_psb(decoder);
2271  if (part_psb) {
2272  ret = intel_pt_get_split_psb(decoder, part_psb);
2273  if (ret)
2274  return ret;
2275  } else {
2276  decoder->pos += decoder->len;
2277  decoder->len = 0;
2278  }
2279  continue;
2280  }
2281 
2282  decoder->pkt_step = next - decoder->buf;
2283  return intel_pt_get_next_packet(decoder);
2284  }
2285 }
2286 
2287 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2288 {
2289  int err;
2290 
2291  decoder->pge = false;
2292  decoder->continuous_period = false;
2293  decoder->have_last_ip = false;
2294  decoder->last_ip = 0;
2295  decoder->ip = 0;
2296  intel_pt_clear_stack(&decoder->stack);
2297 
2298  err = intel_pt_scan_for_psb(decoder);
2299  if (err)
2300  return err;
2301 
2302  decoder->have_last_ip = true;
2303  decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2304 
2305  err = intel_pt_walk_psb(decoder);
2306  if (err)
2307  return err;
2308 
2309  if (decoder->ip) {
2310  decoder->state.type = 0; /* Do not have a sample */
2311  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2312  } else {
2313  return intel_pt_sync_ip(decoder);
2314  }
2315 
2316  return 0;
2317 }
2318 
2319 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2320 {
2321  uint64_t est = decoder->sample_insn_cnt << 1;
2322 
2323  if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2324  goto out;
2325 
2326  est *= decoder->max_non_turbo_ratio;
2327  est /= decoder->cbr;
2328 out:
2329  return decoder->sample_timestamp + est;
2330 }
2331 
2332 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2333 {
2334  int err;
2335 
2336  do {
2337  decoder->state.type = INTEL_PT_BRANCH;
2338  decoder->state.flags = 0;
2339 
2340  switch (decoder->pkt_state) {
2341  case INTEL_PT_STATE_NO_PSB:
2342  err = intel_pt_sync(decoder);
2343  break;
2344  case INTEL_PT_STATE_NO_IP:
2345  decoder->have_last_ip = false;
2346  decoder->last_ip = 0;
2347  decoder->ip = 0;
2348  __fallthrough;
2350  err = intel_pt_sync_ip(decoder);
2351  break;
2353  err = intel_pt_walk_trace(decoder);
2354  break;
2355  case INTEL_PT_STATE_TNT:
2356  err = intel_pt_walk_tnt(decoder);
2357  if (err == -EAGAIN)
2358  err = intel_pt_walk_trace(decoder);
2359  break;
2360  case INTEL_PT_STATE_TIP:
2362  err = intel_pt_walk_tip(decoder);
2363  break;
2364  case INTEL_PT_STATE_FUP:
2365  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2366  err = intel_pt_walk_fup(decoder);
2367  if (err == -EAGAIN)
2368  err = intel_pt_walk_fup_tip(decoder);
2369  else if (!err)
2370  decoder->pkt_state = INTEL_PT_STATE_FUP;
2371  break;
2373  decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2374  err = intel_pt_walk_fup(decoder);
2375  if (err == -EAGAIN)
2376  err = intel_pt_walk_trace(decoder);
2377  break;
2378  default:
2379  err = intel_pt_bug(decoder);
2380  break;
2381  }
2382  } while (err == -ENOLINK);
2383 
2384  if (err) {
2385  decoder->state.err = intel_pt_ext_err(err);
2386  decoder->state.from_ip = decoder->ip;
2387  decoder->sample_timestamp = decoder->timestamp;
2388  decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2389  } else {
2390  decoder->state.err = 0;
2391  if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2392  decoder->cbr_seen = decoder->cbr;
2393  decoder->state.type |= INTEL_PT_CBR_CHG;
2394  decoder->state.cbr_payload = decoder->cbr_payload;
2395  }
2396  if (intel_pt_sample_time(decoder->pkt_state)) {
2397  decoder->sample_timestamp = decoder->timestamp;
2398  decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2399  }
2400  }
2401 
2402  decoder->state.timestamp = decoder->sample_timestamp;
2403  decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2404  decoder->state.cr3 = decoder->cr3;
2405  decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2406 
2407  return &decoder->state;
2408 }
2409 
2421 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2422 {
2423  unsigned char *next;
2424 
2425  next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2426  if (next) {
2427  *len -= next - *buf;
2428  *buf = next;
2429  return true;
2430  }
2431  return false;
2432 }
2433 
2446 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2447 {
2448  unsigned char *next;
2449 
2450  if (!*len)
2451  return false;
2452 
2453  next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2454  if (next) {
2455  *len -= next - *buf;
2456  *buf = next;
2457  return true;
2458  }
2459  return false;
2460 }
2461 
2471 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2472 {
2473  const char *n = INTEL_PT_PSB_STR;
2474  unsigned char *p;
2475  size_t k;
2476 
2477  if (len < INTEL_PT_PSB_LEN)
2478  return NULL;
2479 
2480  k = len - INTEL_PT_PSB_LEN + 1;
2481  while (1) {
2482  p = memrchr(buf, n[0], k);
2483  if (!p)
2484  return NULL;
2485  if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2486  return p;
2487  k = p - buf;
2488  if (!k)
2489  return NULL;
2490  }
2491 }
2492 
2506 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2507  size_t *rem)
2508 {
2509  struct intel_pt_pkt packet;
2510  int ret;
2511 
2512  while (len) {
2513  ret = intel_pt_get_packet(buf, len, &packet);
2514  if (ret <= 0)
2515  return false;
2516  if (packet.type == INTEL_PT_TSC) {
2517  *tsc = packet.payload;
2518  *rem = len;
2519  return true;
2520  }
2521  if (packet.type == INTEL_PT_PSBEND)
2522  return false;
2523  buf += ret;
2524  len -= ret;
2525  }
2526  return false;
2527 }
2528 
2542 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2543 {
2544  const uint64_t halfway = (1ULL << 55);
2545 
2546  if (tsc1 == tsc2)
2547  return 0;
2548 
2549  if (tsc1 < tsc2) {
2550  if (tsc2 - tsc1 < halfway)
2551  return -1;
2552  else
2553  return 1;
2554  } else {
2555  if (tsc1 - tsc2 < halfway)
2556  return 1;
2557  else
2558  return -1;
2559  }
2560 }
2561 
2580 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2581  size_t len_a,
2582  unsigned char *buf_b,
2583  size_t len_b, bool *consecutive)
2584 {
2585  uint64_t tsc_a, tsc_b;
2586  unsigned char *p;
2587  size_t len, rem_a, rem_b;
2588 
2589  p = intel_pt_last_psb(buf_a, len_a);
2590  if (!p)
2591  return buf_b; /* No PSB in buf_a => no overlap */
2592 
2593  len = len_a - (p - buf_a);
2594  if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2595  /* The last PSB+ in buf_a is incomplete, so go back one more */
2596  len_a -= len;
2597  p = intel_pt_last_psb(buf_a, len_a);
2598  if (!p)
2599  return buf_b; /* No full PSB+ => assume no overlap */
2600  len = len_a - (p - buf_a);
2601  if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2602  return buf_b; /* No TSC in buf_a => assume no overlap */
2603  }
2604 
2605  while (1) {
2606  /* Ignore PSB+ with no TSC */
2607  if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2608  int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2609 
2610  /* Same TSC, so buffers are consecutive */
2611  if (!cmp && rem_b >= rem_a) {
2612  *consecutive = true;
2613  return buf_b + len_b - (rem_b - rem_a);
2614  }
2615  if (cmp < 0)
2616  return buf_b; /* tsc_a < tsc_b => no overlap */
2617  }
2618 
2619  if (!intel_pt_step_psb(&buf_b, &len_b))
2620  return buf_b + len_b; /* No PSB in buf_b => no data */
2621  }
2622 }
2623 
2641 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2642  unsigned char *buf_b, size_t len_b,
2643  bool have_tsc, bool *consecutive)
2644 {
2645  unsigned char *found;
2646 
2647  /* Buffer 'b' must start at PSB so throw away everything before that */
2648  if (!intel_pt_next_psb(&buf_b, &len_b))
2649  return buf_b + len_b; /* No PSB */
2650 
2651  if (!intel_pt_next_psb(&buf_a, &len_a))
2652  return buf_b; /* No overlap */
2653 
2654  if (have_tsc) {
2655  found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2656  consecutive);
2657  if (found)
2658  return found;
2659  }
2660 
2661  /*
2662  * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2663  * we can ignore the first part of buffer 'a'.
2664  */
2665  while (len_b < len_a) {
2666  if (!intel_pt_step_psb(&buf_a, &len_a))
2667  return buf_b; /* No overlap */
2668  }
2669 
2670  /* Now len_b >= len_a */
2671  while (1) {
2672  /* Potential overlap so check the bytes */
2673  found = memmem(buf_a, len_a, buf_b, len_a);
2674  if (found) {
2675  *consecutive = true;
2676  return buf_b + len_a;
2677  }
2678 
2679  /* Try again at next PSB in buffer 'a' */
2680  if (!intel_pt_step_psb(&buf_a, &len_a))
2681  return buf_b; /* No overlap */
2682  }
2683 }
static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
#define INTEL_PT_ASYNC
#define intel_pt_log(fmt,...)
Definition: intel-pt-log.h:40
uint32_t tsc_ctc_ratio_d
static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
#define INTEL_PT_STATE_ERR4
int(* walk_insn)(struct intel_pt_insn *intel_pt_insn, uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, uint64_t max_insn_cnt, void *data)
static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
bool(* pgd_ip)(uint64_t ip, void *data)
static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
static void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
uint64_t fup_mwait_payload
static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, size_t *rem)
static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, uint64_t last_ip)
uint64_t cbr_payload
unsigned int mtc_period
uint64_t ptw_payload
const unsigned char * buf
static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, bool from_mtc)
#define INTEL_PT_INSN_BUF_SZ
unsigned max_non_turbo_ratio
static uint64_t intel_pt_lower_power_of_2(uint64_t x)
#define INTEL_PT_STATE_ERR2
dictionary data
Definition: stat-cpi.py:4
#define INTEL_PT_RETURN
int int err
Definition: 5sec.c:44
uint64_t pwre_payload
static void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
enum intel_pt_insn_op insn_op
static void intel_pt_clear_stack(struct intel_pt_stack *stack)
struct intel_pt_state state
uint64_t pwrx_payload
#define INTEL_PT_PSB_STR
enum intel_pt_insn_branch branch
unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]
struct intel_pt_pkt packet
static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
uint64_t mwait_payload
static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
unsigned int cbr_seen
static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
unsigned char * intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, size_t len_b, bool have_tsc, bool *consecutive)
static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, intel_pt_pkt_cb_t cb, void *data)
#define INTEL_PT_STATE_ERR1
struct intel_pt_decoder * intel_pt_decoder_new(struct intel_pt_params *params)
uint64_t ip[INTEL_PT_BLK_SIZE]
int(* get_trace)(struct intel_pt_buffer *buffer, void *data)
uint64_t tot_insn_cnt
unsigned int insn_bytes
unsigned char buf[INTEL_PT_INSN_BUF_SZ]
static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
void * malloc(YYSIZE_T)
static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
#define INTEL_PT_BLK_SIZE
unsigned int fup_tx_flags
enum intel_pt_insn_op op
struct intel_pt_blk * spare
static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
int intel_pt_get_packet(const unsigned char *buf, size_t len, struct intel_pt_pkt *packet)
static bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, uint32_t *last_mtc)
uint64_t last_masked_timestamp
struct intel_pt_pkt tnt
int(* get_trace)(struct intel_pt_buffer *buffer, void *data)
static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
enum intel_pt_period_type period_type
static void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
intel_pt_period_type
const unsigned char * next_buf
enum intel_pt_sample_type type
char insn[INTEL_PT_INSN_BUF_SZ]
static bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, struct intel_pt_insn *intel_pt_insn, uint64_t ip, int err)
intel_pt_pkt_state
struct intel_pt_blk * prev
#define INTEL_PT_ABORT_TX
static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
static void intel_pt_log_at(const char *msg, uint64_t u)
Definition: intel-pt-log.h:68
static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
static int intel_pt_sync(struct intel_pt_decoder *decoder)
x86 movsq based memcpy() in arch/x86/lib/memcpy_64.S") MEMCPY_FN(memcpy_erms
static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
#define INTEL_PT_IN_TX
#define zfree(ptr)
Definition: util.h:25
#define INTEL_PT_MAX_LOOPS
#define INTEL_PT_NEED_MORE_BYTES
enum intel_pt_pkt_state pkt_state
enum intel_pt_param_flags flags
static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
struct intel_pt_stack stack
static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
struct intel_pt_blk * blk
unsigned int tx_flags
static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
#define INTEL_PT_PKT_MAX_SZ
static void intel_pt_set_ip(struct intel_pt_decoder *decoder)
static const char * intel_pt_err_msgs[]
static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
intel_pt_param_flags
enum intel_pt_period_type period_type
int(* intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info)
static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
uint64_t est_timestamp
enum intel_pt_pkt_type type
u32 flags
struct intel_pt_decoder * decoder
bool(* pgd_ip)(uint64_t ip, void *data)
struct intel_pt_pkt packet
static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
const unsigned char * buf
Definition: numa.c:71
uint64_t cyc_ref_timestamp
static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
#define BIT63
void free(void *)
static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
static void intel_pt_log_to(const char *msg, uint64_t u)
Definition: intel-pt-log.h:73
static int intel_pt_ext_err(int code)
static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
enum intel_pt_param_flags flags
static int intel_pt_bug(struct intel_pt_decoder *decoder)
#define x64_fmt
Definition: intel-pt-log.h:64
static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, int part_psb)
unsigned int max_non_turbo_ratio
uint32_t tsc_ctc_ratio_n
static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
static int intel_pt_overflow(struct intel_pt_decoder *decoder)
static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
static unsigned char * intel_pt_find_overlap_tsc(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, size_t len_b, bool *consecutive)
uint64_t timestamp_insn_cnt
static int intel_pt_get_data(struct intel_pt_decoder *decoder)
static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
static void intel_pt_free_stack(struct intel_pt_stack *stack)
int intel_pt__strerror(int code, char *buf, size_t buflen)
#define INTEL_PT_PSB_LEN
int(* walk_insn)(struct intel_pt_insn *intel_pt_insn, uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, uint64_t max_insn_cnt, void *data)
#define INTEL_PT_FUP_IP
static void intel_pt_pop_blk(struct intel_pt_stack *stack)
static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, struct intel_pt_insn *intel_pt_insn, uint64_t ip)
static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
uint64_t ref_timestamp
static bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
const struct intel_pt_state * intel_pt_decode(struct intel_pt_decoder *decoder)
#define intel_pt_log_packet(arg,...)
Definition: intel-pt-log.h:46
#define INTEL_PT_STATE_ERR3
static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
void static void * zalloc(size_t size)
Definition: util.h:20
static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
static unsigned char * intel_pt_last_psb(unsigned char *buf, size_t len)