2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
29 #include "intel-pt-insn-decoder.h"
30 #include "intel-pt-pkt-decoder.h"
31 #include "intel-pt-decoder.h"
32 #include "intel-pt-log.h"
34 #define INTEL_PT_BLK_SIZE 1024
36 #define BIT63 (((uint64_t)1 << 63))
38 #define INTEL_PT_RETURN 1
40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
41 #define INTEL_PT_MAX_LOOPS 10000
44 struct intel_pt_blk
*prev
;
45 uint64_t ip
[INTEL_PT_BLK_SIZE
];
48 struct intel_pt_stack
{
49 struct intel_pt_blk
*blk
;
50 struct intel_pt_blk
*spare
;
54 enum intel_pt_pkt_state
{
55 INTEL_PT_STATE_NO_PSB
,
57 INTEL_PT_STATE_ERR_RESYNC
,
58 INTEL_PT_STATE_IN_SYNC
,
61 INTEL_PT_STATE_TIP_PGD
,
63 INTEL_PT_STATE_FUP_NO_TIP
,
66 #ifdef INTEL_PT_STRICT
67 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
68 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
69 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
70 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
72 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
73 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
74 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
75 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
78 struct intel_pt_decoder
{
79 int (*get_trace
)(struct intel_pt_buffer
*buffer
, void *data
);
80 int (*walk_insn
)(struct intel_pt_insn
*intel_pt_insn
,
81 uint64_t *insn_cnt_ptr
, uint64_t *ip
, uint64_t to_ip
,
82 uint64_t max_insn_cnt
, void *data
);
83 bool (*pgd_ip
)(uint64_t ip
, void *data
);
85 struct intel_pt_state state
;
86 const unsigned char *buf
;
88 bool return_compression
;
99 uint64_t tsc_timestamp
;
100 uint64_t ref_timestamp
;
102 uint64_t ctc_timestamp
;
105 uint64_t cyc_ref_timestamp
;
107 uint32_t tsc_ctc_ratio_n
;
108 uint32_t tsc_ctc_ratio_d
;
109 uint32_t tsc_ctc_mult
;
111 uint32_t ctc_rem_mask
;
113 struct intel_pt_stack stack
;
114 enum intel_pt_pkt_state pkt_state
;
115 struct intel_pt_pkt packet
;
116 struct intel_pt_pkt tnt
;
119 int last_packet_type
;
121 unsigned int max_non_turbo_ratio
;
122 double max_non_turbo_ratio_fp
;
123 double cbr_cyc_to_tsc
;
124 double calc_cyc_to_tsc
;
125 bool have_calc_cyc_to_tsc
;
127 unsigned int insn_bytes
;
129 enum intel_pt_period_type period_type
;
130 uint64_t tot_insn_cnt
;
131 uint64_t period_insn_cnt
;
132 uint64_t period_mask
;
133 uint64_t period_ticks
;
134 uint64_t last_masked_timestamp
;
135 bool continuous_period
;
137 bool set_fup_tx_flags
;
138 unsigned int fup_tx_flags
;
139 unsigned int tx_flags
;
140 uint64_t timestamp_insn_cnt
;
145 const unsigned char *next_buf
;
147 unsigned char temp_buf
[INTEL_PT_PKT_MAX_SZ
];
150 static uint64_t intel_pt_lower_power_of_2(uint64_t x
)
154 for (i
= 0; x
!= 1; i
++)
160 static void intel_pt_setup_period(struct intel_pt_decoder
*decoder
)
162 if (decoder
->period_type
== INTEL_PT_PERIOD_TICKS
) {
165 period
= intel_pt_lower_power_of_2(decoder
->period
);
166 decoder
->period_mask
= ~(period
- 1);
167 decoder
->period_ticks
= period
;
171 static uint64_t multdiv(uint64_t t
, uint32_t n
, uint32_t d
)
175 return (t
/ d
) * n
+ ((t
% d
) * n
) / d
;
178 struct intel_pt_decoder
*intel_pt_decoder_new(struct intel_pt_params
*params
)
180 struct intel_pt_decoder
*decoder
;
182 if (!params
->get_trace
|| !params
->walk_insn
)
185 decoder
= zalloc(sizeof(struct intel_pt_decoder
));
189 decoder
->get_trace
= params
->get_trace
;
190 decoder
->walk_insn
= params
->walk_insn
;
191 decoder
->pgd_ip
= params
->pgd_ip
;
192 decoder
->data
= params
->data
;
193 decoder
->return_compression
= params
->return_compression
;
195 decoder
->period
= params
->period
;
196 decoder
->period_type
= params
->period_type
;
198 decoder
->max_non_turbo_ratio
= params
->max_non_turbo_ratio
;
199 decoder
->max_non_turbo_ratio_fp
= params
->max_non_turbo_ratio
;
201 intel_pt_setup_period(decoder
);
203 decoder
->mtc_shift
= params
->mtc_period
;
204 decoder
->ctc_rem_mask
= (1 << decoder
->mtc_shift
) - 1;
206 decoder
->tsc_ctc_ratio_n
= params
->tsc_ctc_ratio_n
;
207 decoder
->tsc_ctc_ratio_d
= params
->tsc_ctc_ratio_d
;
209 if (!decoder
->tsc_ctc_ratio_n
)
210 decoder
->tsc_ctc_ratio_d
= 0;
212 if (decoder
->tsc_ctc_ratio_d
) {
213 if (!(decoder
->tsc_ctc_ratio_n
% decoder
->tsc_ctc_ratio_d
))
214 decoder
->tsc_ctc_mult
= decoder
->tsc_ctc_ratio_n
/
215 decoder
->tsc_ctc_ratio_d
;
218 * Allow for timestamps appearing to backwards because a TSC
219 * packet has slipped past a MTC packet, so allow 2 MTC ticks
222 decoder
->tsc_slip
= multdiv(2 << decoder
->mtc_shift
,
223 decoder
->tsc_ctc_ratio_n
,
224 decoder
->tsc_ctc_ratio_d
);
226 /* ... or 0x100 paranoia */
227 if (decoder
->tsc_slip
< 0x100)
228 decoder
->tsc_slip
= 0x100;
230 intel_pt_log("timestamp: mtc_shift %u\n", decoder
->mtc_shift
);
231 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder
->tsc_ctc_ratio_n
);
232 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder
->tsc_ctc_ratio_d
);
233 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder
->tsc_ctc_mult
);
234 intel_pt_log("timestamp: tsc_slip %#x\n", decoder
->tsc_slip
);
239 static void intel_pt_pop_blk(struct intel_pt_stack
*stack
)
241 struct intel_pt_blk
*blk
= stack
->blk
;
243 stack
->blk
= blk
->prev
;
250 static uint64_t intel_pt_pop(struct intel_pt_stack
*stack
)
255 intel_pt_pop_blk(stack
);
258 stack
->pos
= INTEL_PT_BLK_SIZE
;
260 return stack
->blk
->ip
[--stack
->pos
];
263 static int intel_pt_alloc_blk(struct intel_pt_stack
*stack
)
265 struct intel_pt_blk
*blk
;
271 blk
= malloc(sizeof(struct intel_pt_blk
));
276 blk
->prev
= stack
->blk
;
282 static int intel_pt_push(struct intel_pt_stack
*stack
, uint64_t ip
)
286 if (!stack
->blk
|| stack
->pos
== INTEL_PT_BLK_SIZE
) {
287 err
= intel_pt_alloc_blk(stack
);
292 stack
->blk
->ip
[stack
->pos
++] = ip
;
296 static void intel_pt_clear_stack(struct intel_pt_stack
*stack
)
299 intel_pt_pop_blk(stack
);
303 static void intel_pt_free_stack(struct intel_pt_stack
*stack
)
305 intel_pt_clear_stack(stack
);
307 zfree(&stack
->spare
);
310 void intel_pt_decoder_free(struct intel_pt_decoder
*decoder
)
312 intel_pt_free_stack(&decoder
->stack
);
316 static int intel_pt_ext_err(int code
)
320 return INTEL_PT_ERR_NOMEM
;
322 return INTEL_PT_ERR_INTERN
;
324 return INTEL_PT_ERR_BADPKT
;
326 return INTEL_PT_ERR_NODATA
;
328 return INTEL_PT_ERR_NOINSN
;
330 return INTEL_PT_ERR_MISMAT
;
332 return INTEL_PT_ERR_OVR
;
334 return INTEL_PT_ERR_LOST
;
336 return INTEL_PT_ERR_NELOOP
;
338 return INTEL_PT_ERR_UNK
;
342 static const char *intel_pt_err_msgs
[] = {
343 [INTEL_PT_ERR_NOMEM
] = "Memory allocation failed",
344 [INTEL_PT_ERR_INTERN
] = "Internal error",
345 [INTEL_PT_ERR_BADPKT
] = "Bad packet",
346 [INTEL_PT_ERR_NODATA
] = "No more data",
347 [INTEL_PT_ERR_NOINSN
] = "Failed to get instruction",
348 [INTEL_PT_ERR_MISMAT
] = "Trace doesn't match instruction",
349 [INTEL_PT_ERR_OVR
] = "Overflow packet",
350 [INTEL_PT_ERR_LOST
] = "Lost trace data",
351 [INTEL_PT_ERR_UNK
] = "Unknown error!",
352 [INTEL_PT_ERR_NELOOP
] = "Never-ending loop",
355 int intel_pt__strerror(int code
, char *buf
, size_t buflen
)
357 if (code
< 1 || code
>= INTEL_PT_ERR_MAX
)
358 code
= INTEL_PT_ERR_UNK
;
359 strlcpy(buf
, intel_pt_err_msgs
[code
], buflen
);
363 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt
*packet
,
368 switch (packet
->count
) {
370 ip
= (last_ip
& (uint64_t)0xffffffffffff0000ULL
) |
374 ip
= (last_ip
& (uint64_t)0xffffffff00000000ULL
) |
378 ip
= packet
->payload
;
379 /* Sign-extend 6-byte ip */
380 if (ip
& (uint64_t)0x800000000000ULL
)
381 ip
|= (uint64_t)0xffff000000000000ULL
;
384 ip
= (last_ip
& (uint64_t)0xffff000000000000ULL
) |
388 ip
= packet
->payload
;
397 static inline void intel_pt_set_last_ip(struct intel_pt_decoder
*decoder
)
399 decoder
->last_ip
= intel_pt_calc_ip(&decoder
->packet
, decoder
->last_ip
);
402 static inline void intel_pt_set_ip(struct intel_pt_decoder
*decoder
)
404 intel_pt_set_last_ip(decoder
);
405 decoder
->ip
= decoder
->last_ip
;
408 static void intel_pt_decoder_log_packet(struct intel_pt_decoder
*decoder
)
410 intel_pt_log_packet(&decoder
->packet
, decoder
->pkt_len
, decoder
->pos
,
414 static int intel_pt_bug(struct intel_pt_decoder
*decoder
)
416 intel_pt_log("ERROR: Internal error\n");
417 decoder
->pkt_state
= INTEL_PT_STATE_NO_PSB
;
421 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder
*decoder
)
423 decoder
->tx_flags
= 0;
426 static inline void intel_pt_update_in_tx(struct intel_pt_decoder
*decoder
)
428 decoder
->tx_flags
= decoder
->packet
.payload
& INTEL_PT_IN_TX
;
431 static int intel_pt_bad_packet(struct intel_pt_decoder
*decoder
)
433 intel_pt_clear_tx_flags(decoder
);
434 decoder
->have_tma
= false;
435 decoder
->pkt_len
= 1;
436 decoder
->pkt_step
= 1;
437 intel_pt_decoder_log_packet(decoder
);
438 if (decoder
->pkt_state
!= INTEL_PT_STATE_NO_PSB
) {
439 intel_pt_log("ERROR: Bad packet\n");
440 decoder
->pkt_state
= INTEL_PT_STATE_ERR1
;
445 static int intel_pt_get_data(struct intel_pt_decoder
*decoder
)
447 struct intel_pt_buffer buffer
= { .buf
= 0, };
450 decoder
->pkt_step
= 0;
452 intel_pt_log("Getting more data\n");
453 ret
= decoder
->get_trace(&buffer
, decoder
->data
);
456 decoder
->buf
= buffer
.buf
;
457 decoder
->len
= buffer
.len
;
459 intel_pt_log("No more data\n");
462 if (!buffer
.consecutive
) {
464 decoder
->pkt_state
= INTEL_PT_STATE_NO_PSB
;
465 decoder
->ref_timestamp
= buffer
.ref_timestamp
;
466 decoder
->timestamp
= 0;
467 decoder
->have_tma
= false;
468 decoder
->state
.trace_nr
= buffer
.trace_nr
;
469 intel_pt_log("Reference timestamp 0x%" PRIx64
"\n",
470 decoder
->ref_timestamp
);
477 static int intel_pt_get_next_data(struct intel_pt_decoder
*decoder
)
479 if (!decoder
->next_buf
)
480 return intel_pt_get_data(decoder
);
482 decoder
->buf
= decoder
->next_buf
;
483 decoder
->len
= decoder
->next_len
;
484 decoder
->next_buf
= 0;
485 decoder
->next_len
= 0;
489 static int intel_pt_get_split_packet(struct intel_pt_decoder
*decoder
)
491 unsigned char *buf
= decoder
->temp_buf
;
492 size_t old_len
, len
, n
;
495 old_len
= decoder
->len
;
497 memcpy(buf
, decoder
->buf
, len
);
499 ret
= intel_pt_get_data(decoder
);
501 decoder
->pos
+= old_len
;
502 return ret
< 0 ? ret
: -EINVAL
;
505 n
= INTEL_PT_PKT_MAX_SZ
- len
;
506 if (n
> decoder
->len
)
508 memcpy(buf
+ len
, decoder
->buf
, n
);
511 ret
= intel_pt_get_packet(buf
, len
, &decoder
->packet
);
512 if (ret
< (int)old_len
) {
513 decoder
->next_buf
= decoder
->buf
;
514 decoder
->next_len
= decoder
->len
;
516 decoder
->len
= old_len
;
517 return intel_pt_bad_packet(decoder
);
520 decoder
->next_buf
= decoder
->buf
+ (ret
- old_len
);
521 decoder
->next_len
= decoder
->len
- (ret
- old_len
);
529 struct intel_pt_pkt_info
{
530 struct intel_pt_decoder
*decoder
;
531 struct intel_pt_pkt packet
;
534 int last_packet_type
;
538 typedef int (*intel_pt_pkt_cb_t
)(struct intel_pt_pkt_info
*pkt_info
);
540 /* Lookahead packets in current buffer */
541 static int intel_pt_pkt_lookahead(struct intel_pt_decoder
*decoder
,
542 intel_pt_pkt_cb_t cb
, void *data
)
544 struct intel_pt_pkt_info pkt_info
;
545 const unsigned char *buf
= decoder
->buf
;
546 size_t len
= decoder
->len
;
549 pkt_info
.decoder
= decoder
;
550 pkt_info
.pos
= decoder
->pos
;
551 pkt_info
.pkt_len
= decoder
->pkt_step
;
552 pkt_info
.last_packet_type
= decoder
->last_packet_type
;
553 pkt_info
.data
= data
;
557 pkt_info
.pos
+= pkt_info
.pkt_len
;
558 buf
+= pkt_info
.pkt_len
;
559 len
-= pkt_info
.pkt_len
;
562 return INTEL_PT_NEED_MORE_BYTES
;
564 ret
= intel_pt_get_packet(buf
, len
, &pkt_info
.packet
);
566 return INTEL_PT_NEED_MORE_BYTES
;
570 pkt_info
.pkt_len
= ret
;
571 } while (pkt_info
.packet
.type
== INTEL_PT_PAD
);
577 pkt_info
.last_packet_type
= pkt_info
.packet
.type
;
581 struct intel_pt_calc_cyc_to_tsc_info
{
585 uint64_t ctc_timestamp
;
587 uint64_t tsc_timestamp
;
592 double cbr_cyc_to_tsc
;
596 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
597 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
598 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
599 * packet by copying the missing bits from the current MTC assuming the least
600 * difference between the two, and that the current MTC comes after last_mtc.
602 static void intel_pt_fixup_last_mtc(uint32_t mtc
, int mtc_shift
,
605 uint32_t first_missing_bit
= 1U << (16 - mtc_shift
);
606 uint32_t mask
= ~(first_missing_bit
- 1);
608 *last_mtc
|= mtc
& mask
;
609 if (*last_mtc
>= mtc
) {
610 *last_mtc
-= first_missing_bit
;
615 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info
*pkt_info
)
617 struct intel_pt_decoder
*decoder
= pkt_info
->decoder
;
618 struct intel_pt_calc_cyc_to_tsc_info
*data
= pkt_info
->data
;
622 uint32_t mtc
, mtc_delta
, ctc
, fc
, ctc_rem
;
624 switch (pkt_info
->packet
.type
) {
626 case INTEL_PT_TIP_PGE
:
631 case INTEL_PT_MODE_EXEC
:
632 case INTEL_PT_MODE_TSX
:
633 case INTEL_PT_PSBEND
:
643 mtc
= pkt_info
->packet
.payload
;
644 if (decoder
->mtc_shift
> 8 && data
->fixup_last_mtc
) {
645 data
->fixup_last_mtc
= false;
646 intel_pt_fixup_last_mtc(mtc
, decoder
->mtc_shift
,
649 if (mtc
> data
->last_mtc
)
650 mtc_delta
= mtc
- data
->last_mtc
;
652 mtc_delta
= mtc
+ 256 - data
->last_mtc
;
653 data
->ctc_delta
+= mtc_delta
<< decoder
->mtc_shift
;
654 data
->last_mtc
= mtc
;
656 if (decoder
->tsc_ctc_mult
) {
657 timestamp
= data
->ctc_timestamp
+
658 data
->ctc_delta
* decoder
->tsc_ctc_mult
;
660 timestamp
= data
->ctc_timestamp
+
661 multdiv(data
->ctc_delta
,
662 decoder
->tsc_ctc_ratio_n
,
663 decoder
->tsc_ctc_ratio_d
);
666 if (timestamp
< data
->timestamp
)
669 if (pkt_info
->last_packet_type
!= INTEL_PT_CYC
) {
670 data
->timestamp
= timestamp
;
677 timestamp
= pkt_info
->packet
.payload
|
678 (data
->timestamp
& (0xffULL
<< 56));
679 if (data
->from_mtc
&& timestamp
< data
->timestamp
&&
680 data
->timestamp
- timestamp
< decoder
->tsc_slip
)
682 if (timestamp
< data
->timestamp
)
683 timestamp
+= (1ULL << 56);
684 if (pkt_info
->last_packet_type
!= INTEL_PT_CYC
) {
687 data
->tsc_timestamp
= timestamp
;
688 data
->timestamp
= timestamp
;
697 if (!decoder
->tsc_ctc_ratio_d
)
700 ctc
= pkt_info
->packet
.payload
;
701 fc
= pkt_info
->packet
.count
;
702 ctc_rem
= ctc
& decoder
->ctc_rem_mask
;
704 data
->last_mtc
= (ctc
>> decoder
->mtc_shift
) & 0xff;
706 data
->ctc_timestamp
= data
->tsc_timestamp
- fc
;
707 if (decoder
->tsc_ctc_mult
) {
708 data
->ctc_timestamp
-= ctc_rem
* decoder
->tsc_ctc_mult
;
710 data
->ctc_timestamp
-=
711 multdiv(ctc_rem
, decoder
->tsc_ctc_ratio_n
,
712 decoder
->tsc_ctc_ratio_d
);
716 data
->have_tma
= true;
717 data
->fixup_last_mtc
= true;
722 data
->cycle_cnt
+= pkt_info
->packet
.payload
;
726 cbr
= pkt_info
->packet
.payload
;
727 if (data
->cbr
&& data
->cbr
!= cbr
)
730 data
->cbr_cyc_to_tsc
= decoder
->max_non_turbo_ratio_fp
/ cbr
;
733 case INTEL_PT_TIP_PGD
:
734 case INTEL_PT_TRACESTOP
:
736 case INTEL_PT_BAD
: /* Does not happen */
741 if (!data
->cbr
&& decoder
->cbr
) {
742 data
->cbr
= decoder
->cbr
;
743 data
->cbr_cyc_to_tsc
= decoder
->cbr_cyc_to_tsc
;
746 if (!data
->cycle_cnt
)
749 cyc_to_tsc
= (double)(timestamp
- decoder
->timestamp
) / data
->cycle_cnt
;
751 if (data
->cbr
&& cyc_to_tsc
> data
->cbr_cyc_to_tsc
&&
752 cyc_to_tsc
/ data
->cbr_cyc_to_tsc
> 1.25) {
753 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt
"\n",
754 cyc_to_tsc
, data
->cbr_cyc_to_tsc
, pkt_info
->pos
);
758 decoder
->calc_cyc_to_tsc
= cyc_to_tsc
;
759 decoder
->have_calc_cyc_to_tsc
= true;
762 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt
"\n",
763 cyc_to_tsc
, data
->cbr_cyc_to_tsc
, pkt_info
->pos
);
765 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt
"\n",
766 cyc_to_tsc
, pkt_info
->pos
);
772 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder
*decoder
,
775 struct intel_pt_calc_cyc_to_tsc_info data
= {
778 .last_mtc
= decoder
->last_mtc
,
779 .ctc_timestamp
= decoder
->ctc_timestamp
,
780 .ctc_delta
= decoder
->ctc_delta
,
781 .tsc_timestamp
= decoder
->tsc_timestamp
,
782 .timestamp
= decoder
->timestamp
,
783 .have_tma
= decoder
->have_tma
,
784 .fixup_last_mtc
= decoder
->fixup_last_mtc
,
785 .from_mtc
= from_mtc
,
789 intel_pt_pkt_lookahead(decoder
, intel_pt_calc_cyc_cb
, &data
);
792 static int intel_pt_get_next_packet(struct intel_pt_decoder
*decoder
)
796 decoder
->last_packet_type
= decoder
->packet
.type
;
799 decoder
->pos
+= decoder
->pkt_step
;
800 decoder
->buf
+= decoder
->pkt_step
;
801 decoder
->len
-= decoder
->pkt_step
;
804 ret
= intel_pt_get_next_data(decoder
);
809 ret
= intel_pt_get_packet(decoder
->buf
, decoder
->len
,
811 if (ret
== INTEL_PT_NEED_MORE_BYTES
&&
812 decoder
->len
< INTEL_PT_PKT_MAX_SZ
&& !decoder
->next_buf
) {
813 ret
= intel_pt_get_split_packet(decoder
);
818 return intel_pt_bad_packet(decoder
);
820 decoder
->pkt_len
= ret
;
821 decoder
->pkt_step
= ret
;
822 intel_pt_decoder_log_packet(decoder
);
823 } while (decoder
->packet
.type
== INTEL_PT_PAD
);
828 static uint64_t intel_pt_next_period(struct intel_pt_decoder
*decoder
)
830 uint64_t timestamp
, masked_timestamp
;
832 timestamp
= decoder
->timestamp
+ decoder
->timestamp_insn_cnt
;
833 masked_timestamp
= timestamp
& decoder
->period_mask
;
834 if (decoder
->continuous_period
) {
835 if (masked_timestamp
!= decoder
->last_masked_timestamp
)
839 masked_timestamp
= timestamp
& decoder
->period_mask
;
840 if (masked_timestamp
!= decoder
->last_masked_timestamp
) {
841 decoder
->last_masked_timestamp
= masked_timestamp
;
842 decoder
->continuous_period
= true;
845 return decoder
->period_ticks
- (timestamp
- masked_timestamp
);
848 static uint64_t intel_pt_next_sample(struct intel_pt_decoder
*decoder
)
850 switch (decoder
->period_type
) {
851 case INTEL_PT_PERIOD_INSTRUCTIONS
:
852 return decoder
->period
- decoder
->period_insn_cnt
;
853 case INTEL_PT_PERIOD_TICKS
:
854 return intel_pt_next_period(decoder
);
855 case INTEL_PT_PERIOD_NONE
:
856 case INTEL_PT_PERIOD_MTC
:
862 static void intel_pt_sample_insn(struct intel_pt_decoder
*decoder
)
864 uint64_t timestamp
, masked_timestamp
;
866 switch (decoder
->period_type
) {
867 case INTEL_PT_PERIOD_INSTRUCTIONS
:
868 decoder
->period_insn_cnt
= 0;
870 case INTEL_PT_PERIOD_TICKS
:
871 timestamp
= decoder
->timestamp
+ decoder
->timestamp_insn_cnt
;
872 masked_timestamp
= timestamp
& decoder
->period_mask
;
873 decoder
->last_masked_timestamp
= masked_timestamp
;
875 case INTEL_PT_PERIOD_NONE
:
876 case INTEL_PT_PERIOD_MTC
:
881 decoder
->state
.type
|= INTEL_PT_INSTRUCTION
;
884 static int intel_pt_walk_insn(struct intel_pt_decoder
*decoder
,
885 struct intel_pt_insn
*intel_pt_insn
, uint64_t ip
)
887 uint64_t max_insn_cnt
, insn_cnt
= 0;
890 if (!decoder
->mtc_insn
)
891 decoder
->mtc_insn
= true;
893 max_insn_cnt
= intel_pt_next_sample(decoder
);
895 err
= decoder
->walk_insn(intel_pt_insn
, &insn_cnt
, &decoder
->ip
, ip
,
896 max_insn_cnt
, decoder
->data
);
898 decoder
->tot_insn_cnt
+= insn_cnt
;
899 decoder
->timestamp_insn_cnt
+= insn_cnt
;
900 decoder
->period_insn_cnt
+= insn_cnt
;
903 decoder
->no_progress
= 0;
904 decoder
->pkt_state
= INTEL_PT_STATE_ERR2
;
905 intel_pt_log_at("ERROR: Failed to get instruction",
912 if (ip
&& decoder
->ip
== ip
) {
917 if (max_insn_cnt
&& insn_cnt
>= max_insn_cnt
)
918 intel_pt_sample_insn(decoder
);
920 if (intel_pt_insn
->branch
== INTEL_PT_BR_NO_BRANCH
) {
921 decoder
->state
.type
= INTEL_PT_INSTRUCTION
;
922 decoder
->state
.from_ip
= decoder
->ip
;
923 decoder
->state
.to_ip
= 0;
924 decoder
->ip
+= intel_pt_insn
->length
;
925 err
= INTEL_PT_RETURN
;
929 if (intel_pt_insn
->op
== INTEL_PT_OP_CALL
) {
930 /* Zero-length calls are excluded */
931 if (intel_pt_insn
->branch
!= INTEL_PT_BR_UNCONDITIONAL
||
932 intel_pt_insn
->rel
) {
933 err
= intel_pt_push(&decoder
->stack
, decoder
->ip
+
934 intel_pt_insn
->length
);
938 } else if (intel_pt_insn
->op
== INTEL_PT_OP_RET
) {
939 decoder
->ret_addr
= intel_pt_pop(&decoder
->stack
);
942 if (intel_pt_insn
->branch
== INTEL_PT_BR_UNCONDITIONAL
) {
943 int cnt
= decoder
->no_progress
++;
945 decoder
->state
.from_ip
= decoder
->ip
;
946 decoder
->ip
+= intel_pt_insn
->length
+
948 decoder
->state
.to_ip
= decoder
->ip
;
949 err
= INTEL_PT_RETURN
;
952 * Check for being stuck in a loop. This can happen if a
953 * decoder error results in the decoder erroneously setting the
954 * ip to an address that is itself in an infinite loop that
955 * consumes no packets. When that happens, there must be an
956 * unconditional branch.
960 decoder
->stuck_ip
= decoder
->state
.to_ip
;
961 decoder
->stuck_ip_prd
= 1;
962 decoder
->stuck_ip_cnt
= 1;
963 } else if (cnt
> INTEL_PT_MAX_LOOPS
||
964 decoder
->state
.to_ip
== decoder
->stuck_ip
) {
965 intel_pt_log_at("ERROR: Never-ending loop",
966 decoder
->state
.to_ip
);
967 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
970 } else if (!--decoder
->stuck_ip_cnt
) {
971 decoder
->stuck_ip_prd
+= 1;
972 decoder
->stuck_ip_cnt
= decoder
->stuck_ip_prd
;
973 decoder
->stuck_ip
= decoder
->state
.to_ip
;
976 goto out_no_progress
;
979 decoder
->no_progress
= 0;
981 decoder
->state
.insn_op
= intel_pt_insn
->op
;
982 decoder
->state
.insn_len
= intel_pt_insn
->length
;
983 memcpy(decoder
->state
.insn
, intel_pt_insn
->buf
,
984 INTEL_PT_INSN_BUF_SZ
);
986 if (decoder
->tx_flags
& INTEL_PT_IN_TX
)
987 decoder
->state
.flags
|= INTEL_PT_IN_TX
;
992 static int intel_pt_walk_fup(struct intel_pt_decoder
*decoder
)
994 struct intel_pt_insn intel_pt_insn
;
998 ip
= decoder
->last_ip
;
1001 err
= intel_pt_walk_insn(decoder
, &intel_pt_insn
, ip
);
1002 if (err
== INTEL_PT_RETURN
)
1004 if (err
== -EAGAIN
) {
1005 if (decoder
->set_fup_tx_flags
) {
1006 decoder
->set_fup_tx_flags
= false;
1007 decoder
->tx_flags
= decoder
->fup_tx_flags
;
1008 decoder
->state
.type
= INTEL_PT_TRANSACTION
;
1009 decoder
->state
.from_ip
= decoder
->ip
;
1010 decoder
->state
.to_ip
= 0;
1011 decoder
->state
.flags
= decoder
->fup_tx_flags
;
1016 decoder
->set_fup_tx_flags
= false;
1020 if (intel_pt_insn
.branch
== INTEL_PT_BR_INDIRECT
) {
1021 intel_pt_log_at("ERROR: Unexpected indirect branch",
1023 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1027 if (intel_pt_insn
.branch
== INTEL_PT_BR_CONDITIONAL
) {
1028 intel_pt_log_at("ERROR: Unexpected conditional branch",
1030 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1034 intel_pt_bug(decoder
);
1038 static int intel_pt_walk_tip(struct intel_pt_decoder
*decoder
)
1040 struct intel_pt_insn intel_pt_insn
;
1043 err
= intel_pt_walk_insn(decoder
, &intel_pt_insn
, 0);
1044 if (err
== INTEL_PT_RETURN
&&
1046 decoder
->pkt_state
== INTEL_PT_STATE_TIP_PGD
&&
1047 (decoder
->state
.type
& INTEL_PT_BRANCH
) &&
1048 decoder
->pgd_ip(decoder
->state
.to_ip
, decoder
->data
)) {
1049 /* Unconditional branch leaving filter region */
1050 decoder
->no_progress
= 0;
1051 decoder
->pge
= false;
1052 decoder
->continuous_period
= false;
1053 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1054 decoder
->state
.to_ip
= 0;
1057 if (err
== INTEL_PT_RETURN
)
1062 if (intel_pt_insn
.branch
== INTEL_PT_BR_INDIRECT
) {
1063 if (decoder
->pkt_state
== INTEL_PT_STATE_TIP_PGD
) {
1064 decoder
->pge
= false;
1065 decoder
->continuous_period
= false;
1066 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1067 decoder
->state
.from_ip
= decoder
->ip
;
1068 decoder
->state
.to_ip
= 0;
1069 if (decoder
->packet
.count
!= 0)
1070 decoder
->ip
= decoder
->last_ip
;
1072 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1073 decoder
->state
.from_ip
= decoder
->ip
;
1074 if (decoder
->packet
.count
== 0) {
1075 decoder
->state
.to_ip
= 0;
1077 decoder
->state
.to_ip
= decoder
->last_ip
;
1078 decoder
->ip
= decoder
->last_ip
;
1084 if (intel_pt_insn
.branch
== INTEL_PT_BR_CONDITIONAL
) {
1085 uint64_t to_ip
= decoder
->ip
+ intel_pt_insn
.length
+
1088 if (decoder
->pgd_ip
&&
1089 decoder
->pkt_state
== INTEL_PT_STATE_TIP_PGD
&&
1090 decoder
->pgd_ip(to_ip
, decoder
->data
)) {
1091 /* Conditional branch leaving filter region */
1092 decoder
->pge
= false;
1093 decoder
->continuous_period
= false;
1094 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1095 decoder
->ip
= to_ip
;
1096 decoder
->state
.from_ip
= decoder
->ip
;
1097 decoder
->state
.to_ip
= 0;
1100 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1102 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1106 return intel_pt_bug(decoder
);
1109 static int intel_pt_walk_tnt(struct intel_pt_decoder
*decoder
)
1111 struct intel_pt_insn intel_pt_insn
;
1115 err
= intel_pt_walk_insn(decoder
, &intel_pt_insn
, 0);
1116 if (err
== INTEL_PT_RETURN
)
1121 if (intel_pt_insn
.op
== INTEL_PT_OP_RET
) {
1122 if (!decoder
->return_compression
) {
1123 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1125 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1128 if (!decoder
->ret_addr
) {
1129 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1131 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1134 if (!(decoder
->tnt
.payload
& BIT63
)) {
1135 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1137 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1140 decoder
->tnt
.count
-= 1;
1141 if (!decoder
->tnt
.count
)
1142 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1143 decoder
->tnt
.payload
<<= 1;
1144 decoder
->state
.from_ip
= decoder
->ip
;
1145 decoder
->ip
= decoder
->ret_addr
;
1146 decoder
->state
.to_ip
= decoder
->ip
;
1150 if (intel_pt_insn
.branch
== INTEL_PT_BR_INDIRECT
) {
1151 /* Handle deferred TIPs */
1152 err
= intel_pt_get_next_packet(decoder
);
1155 if (decoder
->packet
.type
!= INTEL_PT_TIP
||
1156 decoder
->packet
.count
== 0) {
1157 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1159 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1160 decoder
->pkt_step
= 0;
1163 intel_pt_set_last_ip(decoder
);
1164 decoder
->state
.from_ip
= decoder
->ip
;
1165 decoder
->state
.to_ip
= decoder
->last_ip
;
1166 decoder
->ip
= decoder
->last_ip
;
1170 if (intel_pt_insn
.branch
== INTEL_PT_BR_CONDITIONAL
) {
1171 decoder
->tnt
.count
-= 1;
1172 if (!decoder
->tnt
.count
)
1173 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1174 if (decoder
->tnt
.payload
& BIT63
) {
1175 decoder
->tnt
.payload
<<= 1;
1176 decoder
->state
.from_ip
= decoder
->ip
;
1177 decoder
->ip
+= intel_pt_insn
.length
+
1179 decoder
->state
.to_ip
= decoder
->ip
;
1182 /* Instruction sample for a non-taken branch */
1183 if (decoder
->state
.type
& INTEL_PT_INSTRUCTION
) {
1184 decoder
->tnt
.payload
<<= 1;
1185 decoder
->state
.type
= INTEL_PT_INSTRUCTION
;
1186 decoder
->state
.from_ip
= decoder
->ip
;
1187 decoder
->state
.to_ip
= 0;
1188 decoder
->ip
+= intel_pt_insn
.length
;
1191 decoder
->ip
+= intel_pt_insn
.length
;
1192 if (!decoder
->tnt
.count
)
1194 decoder
->tnt
.payload
<<= 1;
1198 return intel_pt_bug(decoder
);
1202 static int intel_pt_mode_tsx(struct intel_pt_decoder
*decoder
, bool *no_tip
)
1204 unsigned int fup_tx_flags
;
1207 fup_tx_flags
= decoder
->packet
.payload
&
1208 (INTEL_PT_IN_TX
| INTEL_PT_ABORT_TX
);
1209 err
= intel_pt_get_next_packet(decoder
);
1212 if (decoder
->packet
.type
== INTEL_PT_FUP
) {
1213 decoder
->fup_tx_flags
= fup_tx_flags
;
1214 decoder
->set_fup_tx_flags
= true;
1215 if (!(decoder
->fup_tx_flags
& INTEL_PT_ABORT_TX
))
1218 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1220 intel_pt_update_in_tx(decoder
);
1225 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder
*decoder
)
1229 decoder
->have_tma
= false;
1231 if (decoder
->ref_timestamp
) {
1232 timestamp
= decoder
->packet
.payload
|
1233 (decoder
->ref_timestamp
& (0xffULL
<< 56));
1234 if (timestamp
< decoder
->ref_timestamp
) {
1235 if (decoder
->ref_timestamp
- timestamp
> (1ULL << 55))
1236 timestamp
+= (1ULL << 56);
1238 if (timestamp
- decoder
->ref_timestamp
> (1ULL << 55))
1239 timestamp
-= (1ULL << 56);
1241 decoder
->tsc_timestamp
= timestamp
;
1242 decoder
->timestamp
= timestamp
;
1243 decoder
->ref_timestamp
= 0;
1244 decoder
->timestamp_insn_cnt
= 0;
1245 } else if (decoder
->timestamp
) {
1246 timestamp
= decoder
->packet
.payload
|
1247 (decoder
->timestamp
& (0xffULL
<< 56));
1248 decoder
->tsc_timestamp
= timestamp
;
1249 if (timestamp
< decoder
->timestamp
&&
1250 decoder
->timestamp
- timestamp
< decoder
->tsc_slip
) {
1251 intel_pt_log_to("Suppressing backwards timestamp",
1253 timestamp
= decoder
->timestamp
;
1255 if (timestamp
< decoder
->timestamp
) {
1256 intel_pt_log_to("Wraparound timestamp", timestamp
);
1257 timestamp
+= (1ULL << 56);
1258 decoder
->tsc_timestamp
= timestamp
;
1260 decoder
->timestamp
= timestamp
;
1261 decoder
->timestamp_insn_cnt
= 0;
1264 if (decoder
->last_packet_type
== INTEL_PT_CYC
) {
1265 decoder
->cyc_ref_timestamp
= decoder
->timestamp
;
1266 decoder
->cycle_cnt
= 0;
1267 decoder
->have_calc_cyc_to_tsc
= false;
1268 intel_pt_calc_cyc_to_tsc(decoder
, false);
1271 intel_pt_log_to("Setting timestamp", decoder
->timestamp
);
1274 static int intel_pt_overflow(struct intel_pt_decoder
*decoder
)
1276 intel_pt_log("ERROR: Buffer overflow\n");
1277 intel_pt_clear_tx_flags(decoder
);
1278 decoder
->have_tma
= false;
1280 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1281 decoder
->overflow
= true;
1285 static void intel_pt_calc_tma(struct intel_pt_decoder
*decoder
)
1287 uint32_t ctc
= decoder
->packet
.payload
;
1288 uint32_t fc
= decoder
->packet
.count
;
1289 uint32_t ctc_rem
= ctc
& decoder
->ctc_rem_mask
;
1291 if (!decoder
->tsc_ctc_ratio_d
)
1294 decoder
->last_mtc
= (ctc
>> decoder
->mtc_shift
) & 0xff;
1295 decoder
->ctc_timestamp
= decoder
->tsc_timestamp
- fc
;
1296 if (decoder
->tsc_ctc_mult
) {
1297 decoder
->ctc_timestamp
-= ctc_rem
* decoder
->tsc_ctc_mult
;
1299 decoder
->ctc_timestamp
-= multdiv(ctc_rem
,
1300 decoder
->tsc_ctc_ratio_n
,
1301 decoder
->tsc_ctc_ratio_d
);
1303 decoder
->ctc_delta
= 0;
1304 decoder
->have_tma
= true;
1305 decoder
->fixup_last_mtc
= true;
1306 intel_pt_log("CTC timestamp " x64_fmt
" last MTC %#x CTC rem %#x\n",
1307 decoder
->ctc_timestamp
, decoder
->last_mtc
, ctc_rem
);
1310 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder
*decoder
)
1313 uint32_t mtc
, mtc_delta
;
1315 if (!decoder
->have_tma
)
1318 mtc
= decoder
->packet
.payload
;
1320 if (decoder
->mtc_shift
> 8 && decoder
->fixup_last_mtc
) {
1321 decoder
->fixup_last_mtc
= false;
1322 intel_pt_fixup_last_mtc(mtc
, decoder
->mtc_shift
,
1323 &decoder
->last_mtc
);
1326 if (mtc
> decoder
->last_mtc
)
1327 mtc_delta
= mtc
- decoder
->last_mtc
;
1329 mtc_delta
= mtc
+ 256 - decoder
->last_mtc
;
1331 decoder
->ctc_delta
+= mtc_delta
<< decoder
->mtc_shift
;
1333 if (decoder
->tsc_ctc_mult
) {
1334 timestamp
= decoder
->ctc_timestamp
+
1335 decoder
->ctc_delta
* decoder
->tsc_ctc_mult
;
1337 timestamp
= decoder
->ctc_timestamp
+
1338 multdiv(decoder
->ctc_delta
,
1339 decoder
->tsc_ctc_ratio_n
,
1340 decoder
->tsc_ctc_ratio_d
);
1343 if (timestamp
< decoder
->timestamp
)
1344 intel_pt_log("Suppressing MTC timestamp " x64_fmt
" less than current timestamp " x64_fmt
"\n",
1345 timestamp
, decoder
->timestamp
);
1347 decoder
->timestamp
= timestamp
;
1349 decoder
->timestamp_insn_cnt
= 0;
1350 decoder
->last_mtc
= mtc
;
1352 if (decoder
->last_packet_type
== INTEL_PT_CYC
) {
1353 decoder
->cyc_ref_timestamp
= decoder
->timestamp
;
1354 decoder
->cycle_cnt
= 0;
1355 decoder
->have_calc_cyc_to_tsc
= false;
1356 intel_pt_calc_cyc_to_tsc(decoder
, true);
1360 static void intel_pt_calc_cbr(struct intel_pt_decoder
*decoder
)
1362 unsigned int cbr
= decoder
->packet
.payload
;
1364 if (decoder
->cbr
== cbr
)
1368 decoder
->cbr_cyc_to_tsc
= decoder
->max_non_turbo_ratio_fp
/ cbr
;
1371 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder
*decoder
)
1373 uint64_t timestamp
= decoder
->cyc_ref_timestamp
;
1375 decoder
->have_cyc
= true;
1377 decoder
->cycle_cnt
+= decoder
->packet
.payload
;
1379 if (!decoder
->cyc_ref_timestamp
)
1382 if (decoder
->have_calc_cyc_to_tsc
)
1383 timestamp
+= decoder
->cycle_cnt
* decoder
->calc_cyc_to_tsc
;
1384 else if (decoder
->cbr
)
1385 timestamp
+= decoder
->cycle_cnt
* decoder
->cbr_cyc_to_tsc
;
1389 if (timestamp
< decoder
->timestamp
)
1390 intel_pt_log("Suppressing CYC timestamp " x64_fmt
" less than current timestamp " x64_fmt
"\n",
1391 timestamp
, decoder
->timestamp
);
1393 decoder
->timestamp
= timestamp
;
1395 decoder
->timestamp_insn_cnt
= 0;
1398 /* Walk PSB+ packets when already in sync. */
1399 static int intel_pt_walk_psbend(struct intel_pt_decoder
*decoder
)
1404 err
= intel_pt_get_next_packet(decoder
);
1408 switch (decoder
->packet
.type
) {
1409 case INTEL_PT_PSBEND
:
1412 case INTEL_PT_TIP_PGD
:
1413 case INTEL_PT_TIP_PGE
:
1416 case INTEL_PT_TRACESTOP
:
1419 decoder
->have_tma
= false;
1420 intel_pt_log("ERROR: Unexpected packet\n");
1424 return intel_pt_overflow(decoder
);
1427 intel_pt_calc_tsc_timestamp(decoder
);
1431 intel_pt_calc_tma(decoder
);
1435 intel_pt_calc_cbr(decoder
);
1438 case INTEL_PT_MODE_EXEC
:
1439 decoder
->exec_mode
= decoder
->packet
.payload
;
1443 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
1447 decoder
->pge
= true;
1448 intel_pt_set_last_ip(decoder
);
1451 case INTEL_PT_MODE_TSX
:
1452 intel_pt_update_in_tx(decoder
);
1456 intel_pt_calc_mtc_timestamp(decoder
);
1457 if (decoder
->period_type
== INTEL_PT_PERIOD_MTC
)
1458 decoder
->state
.type
|= INTEL_PT_INSTRUCTION
;
1471 static int intel_pt_walk_fup_tip(struct intel_pt_decoder
*decoder
)
1475 if (decoder
->tx_flags
& INTEL_PT_ABORT_TX
) {
1476 decoder
->tx_flags
= 0;
1477 decoder
->state
.flags
&= ~INTEL_PT_IN_TX
;
1478 decoder
->state
.flags
|= INTEL_PT_ABORT_TX
;
1480 decoder
->state
.flags
|= INTEL_PT_ASYNC
;
1484 err
= intel_pt_get_next_packet(decoder
);
1488 switch (decoder
->packet
.type
) {
1491 case INTEL_PT_TRACESTOP
:
1496 case INTEL_PT_MODE_TSX
:
1498 case INTEL_PT_PSBEND
:
1499 intel_pt_log("ERROR: Missing TIP after FUP\n");
1500 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1504 return intel_pt_overflow(decoder
);
1506 case INTEL_PT_TIP_PGD
:
1507 decoder
->state
.from_ip
= decoder
->ip
;
1508 decoder
->state
.to_ip
= 0;
1509 if (decoder
->packet
.count
!= 0) {
1510 intel_pt_set_ip(decoder
);
1511 intel_pt_log("Omitting PGD ip " x64_fmt
"\n",
1514 decoder
->pge
= false;
1515 decoder
->continuous_period
= false;
1518 case INTEL_PT_TIP_PGE
:
1519 decoder
->pge
= true;
1520 intel_pt_log("Omitting PGE ip " x64_fmt
"\n",
1522 decoder
->state
.from_ip
= 0;
1523 if (decoder
->packet
.count
== 0) {
1524 decoder
->state
.to_ip
= 0;
1526 intel_pt_set_ip(decoder
);
1527 decoder
->state
.to_ip
= decoder
->ip
;
1532 decoder
->state
.from_ip
= decoder
->ip
;
1533 if (decoder
->packet
.count
== 0) {
1534 decoder
->state
.to_ip
= 0;
1536 intel_pt_set_ip(decoder
);
1537 decoder
->state
.to_ip
= decoder
->ip
;
1542 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
1546 intel_pt_calc_mtc_timestamp(decoder
);
1547 if (decoder
->period_type
== INTEL_PT_PERIOD_MTC
)
1548 decoder
->state
.type
|= INTEL_PT_INSTRUCTION
;
1552 intel_pt_calc_cyc_timestamp(decoder
);
1555 case INTEL_PT_MODE_EXEC
:
1556 decoder
->exec_mode
= decoder
->packet
.payload
;
1565 return intel_pt_bug(decoder
);
1570 static int intel_pt_walk_trace(struct intel_pt_decoder
*decoder
)
1572 bool no_tip
= false;
1576 err
= intel_pt_get_next_packet(decoder
);
1580 switch (decoder
->packet
.type
) {
1582 if (!decoder
->packet
.count
)
1584 decoder
->tnt
= decoder
->packet
;
1585 decoder
->pkt_state
= INTEL_PT_STATE_TNT
;
1586 err
= intel_pt_walk_tnt(decoder
);
1591 case INTEL_PT_TIP_PGD
:
1592 if (decoder
->packet
.count
!= 0)
1593 intel_pt_set_last_ip(decoder
);
1594 decoder
->pkt_state
= INTEL_PT_STATE_TIP_PGD
;
1595 return intel_pt_walk_tip(decoder
);
1597 case INTEL_PT_TIP_PGE
: {
1598 decoder
->pge
= true;
1599 if (decoder
->packet
.count
== 0) {
1600 intel_pt_log_at("Skipping zero TIP.PGE",
1604 intel_pt_set_ip(decoder
);
1605 decoder
->state
.from_ip
= 0;
1606 decoder
->state
.to_ip
= decoder
->ip
;
1611 return intel_pt_overflow(decoder
);
1614 if (decoder
->packet
.count
!= 0)
1615 intel_pt_set_last_ip(decoder
);
1616 decoder
->pkt_state
= INTEL_PT_STATE_TIP
;
1617 return intel_pt_walk_tip(decoder
);
1620 if (decoder
->packet
.count
== 0) {
1621 intel_pt_log_at("Skipping zero FUP",
1626 intel_pt_set_last_ip(decoder
);
1627 err
= intel_pt_walk_fup(decoder
);
1628 if (err
!= -EAGAIN
) {
1632 decoder
->pkt_state
=
1633 INTEL_PT_STATE_FUP_NO_TIP
;
1635 decoder
->pkt_state
= INTEL_PT_STATE_FUP
;
1642 return intel_pt_walk_fup_tip(decoder
);
1644 case INTEL_PT_TRACESTOP
:
1645 decoder
->pge
= false;
1646 decoder
->continuous_period
= false;
1647 intel_pt_clear_tx_flags(decoder
);
1648 decoder
->have_tma
= false;
1652 intel_pt_clear_stack(&decoder
->stack
);
1653 err
= intel_pt_walk_psbend(decoder
);
1661 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
1665 intel_pt_calc_mtc_timestamp(decoder
);
1666 if (decoder
->period_type
!= INTEL_PT_PERIOD_MTC
)
1669 * Ensure that there has been an instruction since the
1672 if (!decoder
->mtc_insn
)
1674 decoder
->mtc_insn
= false;
1675 /* Ensure that there is a timestamp */
1676 if (!decoder
->timestamp
)
1678 decoder
->state
.type
= INTEL_PT_INSTRUCTION
;
1679 decoder
->state
.from_ip
= decoder
->ip
;
1680 decoder
->state
.to_ip
= 0;
1681 decoder
->mtc_insn
= false;
1685 intel_pt_calc_tsc_timestamp(decoder
);
1689 intel_pt_calc_tma(decoder
);
1693 intel_pt_calc_cyc_timestamp(decoder
);
1697 intel_pt_calc_cbr(decoder
);
1700 case INTEL_PT_MODE_EXEC
:
1701 decoder
->exec_mode
= decoder
->packet
.payload
;
1704 case INTEL_PT_MODE_TSX
:
1705 /* MODE_TSX need not be followed by FUP */
1706 if (!decoder
->pge
) {
1707 intel_pt_update_in_tx(decoder
);
1710 err
= intel_pt_mode_tsx(decoder
, &no_tip
);
1715 case INTEL_PT_BAD
: /* Does not happen */
1716 return intel_pt_bug(decoder
);
1718 case INTEL_PT_PSBEND
:
1725 return intel_pt_bug(decoder
);
1730 static inline bool intel_pt_have_ip(struct intel_pt_decoder
*decoder
)
1732 return decoder
->last_ip
|| decoder
->packet
.count
== 0 ||
1733 decoder
->packet
.count
== 3 || decoder
->packet
.count
== 6;
1736 /* Walk PSB+ packets to get in sync. */
1737 static int intel_pt_walk_psb(struct intel_pt_decoder
*decoder
)
1742 err
= intel_pt_get_next_packet(decoder
);
1746 switch (decoder
->packet
.type
) {
1747 case INTEL_PT_TIP_PGD
:
1748 decoder
->continuous_period
= false;
1749 case INTEL_PT_TIP_PGE
:
1751 intel_pt_log("ERROR: Unexpected packet\n");
1755 decoder
->pge
= true;
1756 if (intel_pt_have_ip(decoder
)) {
1757 uint64_t current_ip
= decoder
->ip
;
1759 intel_pt_set_ip(decoder
);
1761 intel_pt_log_to("Setting IP",
1767 intel_pt_calc_mtc_timestamp(decoder
);
1771 intel_pt_calc_tsc_timestamp(decoder
);
1775 intel_pt_calc_tma(decoder
);
1779 intel_pt_calc_cyc_timestamp(decoder
);
1783 intel_pt_calc_cbr(decoder
);
1787 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
1790 case INTEL_PT_MODE_EXEC
:
1791 decoder
->exec_mode
= decoder
->packet
.payload
;
1794 case INTEL_PT_MODE_TSX
:
1795 intel_pt_update_in_tx(decoder
);
1798 case INTEL_PT_TRACESTOP
:
1799 decoder
->pge
= false;
1800 decoder
->continuous_period
= false;
1801 intel_pt_clear_tx_flags(decoder
);
1803 decoder
->have_tma
= false;
1804 intel_pt_log("ERROR: Unexpected packet\n");
1806 decoder
->pkt_state
= INTEL_PT_STATE_ERR4
;
1808 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1811 case INTEL_PT_BAD
: /* Does not happen */
1812 return intel_pt_bug(decoder
);
1815 return intel_pt_overflow(decoder
);
1817 case INTEL_PT_PSBEND
:
1830 static int intel_pt_walk_to_ip(struct intel_pt_decoder
*decoder
)
1835 err
= intel_pt_get_next_packet(decoder
);
1839 switch (decoder
->packet
.type
) {
1840 case INTEL_PT_TIP_PGD
:
1841 decoder
->continuous_period
= false;
1842 case INTEL_PT_TIP_PGE
:
1844 decoder
->pge
= decoder
->packet
.type
!= INTEL_PT_TIP_PGD
;
1845 if (intel_pt_have_ip(decoder
))
1846 intel_pt_set_ip(decoder
);
1852 if (decoder
->overflow
) {
1853 if (intel_pt_have_ip(decoder
))
1854 intel_pt_set_ip(decoder
);
1858 if (decoder
->packet
.count
)
1859 intel_pt_set_last_ip(decoder
);
1863 intel_pt_calc_mtc_timestamp(decoder
);
1867 intel_pt_calc_tsc_timestamp(decoder
);
1871 intel_pt_calc_tma(decoder
);
1875 intel_pt_calc_cyc_timestamp(decoder
);
1879 intel_pt_calc_cbr(decoder
);
1883 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
1886 case INTEL_PT_MODE_EXEC
:
1887 decoder
->exec_mode
= decoder
->packet
.payload
;
1890 case INTEL_PT_MODE_TSX
:
1891 intel_pt_update_in_tx(decoder
);
1895 return intel_pt_overflow(decoder
);
1897 case INTEL_PT_BAD
: /* Does not happen */
1898 return intel_pt_bug(decoder
);
1900 case INTEL_PT_TRACESTOP
:
1901 decoder
->pge
= false;
1902 decoder
->continuous_period
= false;
1903 intel_pt_clear_tx_flags(decoder
);
1904 decoder
->have_tma
= false;
1908 err
= intel_pt_walk_psb(decoder
);
1912 /* Do not have a sample */
1913 decoder
->state
.type
= 0;
1919 case INTEL_PT_PSBEND
:
1929 static int intel_pt_sync_ip(struct intel_pt_decoder
*decoder
)
1933 intel_pt_log("Scanning for full IP\n");
1934 err
= intel_pt_walk_to_ip(decoder
);
1938 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1939 decoder
->overflow
= false;
1941 decoder
->state
.from_ip
= 0;
1942 decoder
->state
.to_ip
= decoder
->ip
;
1943 intel_pt_log_to("Setting IP", decoder
->ip
);
1948 static int intel_pt_part_psb(struct intel_pt_decoder
*decoder
)
1950 const unsigned char *end
= decoder
->buf
+ decoder
->len
;
1953 for (i
= INTEL_PT_PSB_LEN
- 1; i
; i
--) {
1954 if (i
> decoder
->len
)
1956 if (!memcmp(end
- i
, INTEL_PT_PSB_STR
, i
))
1962 static int intel_pt_rest_psb(struct intel_pt_decoder
*decoder
, int part_psb
)
1964 size_t rest_psb
= INTEL_PT_PSB_LEN
- part_psb
;
1965 const char *psb
= INTEL_PT_PSB_STR
;
1967 if (rest_psb
> decoder
->len
||
1968 memcmp(decoder
->buf
, psb
+ part_psb
, rest_psb
))
1974 static int intel_pt_get_split_psb(struct intel_pt_decoder
*decoder
,
1979 decoder
->pos
+= decoder
->len
;
1982 ret
= intel_pt_get_next_data(decoder
);
1986 rest_psb
= intel_pt_rest_psb(decoder
, part_psb
);
1990 decoder
->pos
-= part_psb
;
1991 decoder
->next_buf
= decoder
->buf
+ rest_psb
;
1992 decoder
->next_len
= decoder
->len
- rest_psb
;
1993 memcpy(decoder
->temp_buf
, INTEL_PT_PSB_STR
, INTEL_PT_PSB_LEN
);
1994 decoder
->buf
= decoder
->temp_buf
;
1995 decoder
->len
= INTEL_PT_PSB_LEN
;
2000 static int intel_pt_scan_for_psb(struct intel_pt_decoder
*decoder
)
2002 unsigned char *next
;
2005 intel_pt_log("Scanning for PSB\n");
2007 if (!decoder
->len
) {
2008 ret
= intel_pt_get_next_data(decoder
);
2013 next
= memmem(decoder
->buf
, decoder
->len
, INTEL_PT_PSB_STR
,
2018 part_psb
= intel_pt_part_psb(decoder
);
2020 ret
= intel_pt_get_split_psb(decoder
, part_psb
);
2024 decoder
->pos
+= decoder
->len
;
2030 decoder
->pkt_step
= next
- decoder
->buf
;
2031 return intel_pt_get_next_packet(decoder
);
2035 static int intel_pt_sync(struct intel_pt_decoder
*decoder
)
2039 decoder
->pge
= false;
2040 decoder
->continuous_period
= false;
2041 decoder
->last_ip
= 0;
2043 intel_pt_clear_stack(&decoder
->stack
);
2045 err
= intel_pt_scan_for_psb(decoder
);
2049 decoder
->pkt_state
= INTEL_PT_STATE_NO_IP
;
2051 err
= intel_pt_walk_psb(decoder
);
2056 decoder
->state
.type
= 0; /* Do not have a sample */
2057 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
2059 return intel_pt_sync_ip(decoder
);
2065 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder
*decoder
)
2067 uint64_t est
= decoder
->timestamp_insn_cnt
<< 1;
2069 if (!decoder
->cbr
|| !decoder
->max_non_turbo_ratio
)
2072 est
*= decoder
->max_non_turbo_ratio
;
2073 est
/= decoder
->cbr
;
2075 return decoder
->timestamp
+ est
;
2078 const struct intel_pt_state
*intel_pt_decode(struct intel_pt_decoder
*decoder
)
2083 decoder
->state
.type
= INTEL_PT_BRANCH
;
2084 decoder
->state
.flags
= 0;
2086 switch (decoder
->pkt_state
) {
2087 case INTEL_PT_STATE_NO_PSB
:
2088 err
= intel_pt_sync(decoder
);
2090 case INTEL_PT_STATE_NO_IP
:
2091 decoder
->last_ip
= 0;
2093 case INTEL_PT_STATE_ERR_RESYNC
:
2094 err
= intel_pt_sync_ip(decoder
);
2096 case INTEL_PT_STATE_IN_SYNC
:
2097 err
= intel_pt_walk_trace(decoder
);
2099 case INTEL_PT_STATE_TNT
:
2100 err
= intel_pt_walk_tnt(decoder
);
2102 err
= intel_pt_walk_trace(decoder
);
2104 case INTEL_PT_STATE_TIP
:
2105 case INTEL_PT_STATE_TIP_PGD
:
2106 err
= intel_pt_walk_tip(decoder
);
2108 case INTEL_PT_STATE_FUP
:
2109 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
2110 err
= intel_pt_walk_fup(decoder
);
2112 err
= intel_pt_walk_fup_tip(decoder
);
2114 decoder
->pkt_state
= INTEL_PT_STATE_FUP
;
2116 case INTEL_PT_STATE_FUP_NO_TIP
:
2117 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
2118 err
= intel_pt_walk_fup(decoder
);
2120 err
= intel_pt_walk_trace(decoder
);
2123 err
= intel_pt_bug(decoder
);
2126 } while (err
== -ENOLINK
);
2128 decoder
->state
.err
= err
? intel_pt_ext_err(err
) : 0;
2129 decoder
->state
.timestamp
= decoder
->timestamp
;
2130 decoder
->state
.est_timestamp
= intel_pt_est_timestamp(decoder
);
2131 decoder
->state
.cr3
= decoder
->cr3
;
2132 decoder
->state
.tot_insn_cnt
= decoder
->tot_insn_cnt
;
2135 decoder
->state
.from_ip
= decoder
->ip
;
2137 return &decoder
->state
;
2140 static bool intel_pt_at_psb(unsigned char *buf
, size_t len
)
2142 if (len
< INTEL_PT_PSB_LEN
)
2144 return memmem(buf
, INTEL_PT_PSB_LEN
, INTEL_PT_PSB_STR
,
2149 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2150 * @buf: pointer to buffer pointer
2151 * @len: size of buffer
2153 * Updates the buffer pointer to point to the start of the next PSB packet if
2154 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2155 * @len is adjusted accordingly.
2157 * Return: %true if a PSB packet is found, %false otherwise.
2159 static bool intel_pt_next_psb(unsigned char **buf
, size_t *len
)
2161 unsigned char *next
;
2163 next
= memmem(*buf
, *len
, INTEL_PT_PSB_STR
, INTEL_PT_PSB_LEN
);
2165 *len
-= next
- *buf
;
2173 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2175 * @buf: pointer to buffer pointer
2176 * @len: size of buffer
2178 * Updates the buffer pointer to point to the start of the following PSB packet
2179 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2180 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2182 * Return: %true if a PSB packet is found, %false otherwise.
2184 static bool intel_pt_step_psb(unsigned char **buf
, size_t *len
)
2186 unsigned char *next
;
2191 next
= memmem(*buf
+ 1, *len
- 1, INTEL_PT_PSB_STR
, INTEL_PT_PSB_LEN
);
2193 *len
-= next
- *buf
;
2201 * intel_pt_last_psb - find the last PSB packet in a buffer.
2203 * @len: size of buffer
2205 * This function finds the last PSB in a buffer.
2207 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2209 static unsigned char *intel_pt_last_psb(unsigned char *buf
, size_t len
)
2211 const char *n
= INTEL_PT_PSB_STR
;
2215 if (len
< INTEL_PT_PSB_LEN
)
2218 k
= len
- INTEL_PT_PSB_LEN
+ 1;
2220 p
= memrchr(buf
, n
[0], k
);
2223 if (!memcmp(p
+ 1, n
+ 1, INTEL_PT_PSB_LEN
- 1))
2232 * intel_pt_next_tsc - find and return next TSC.
2234 * @len: size of buffer
2235 * @tsc: TSC value returned
2237 * Find a TSC packet in @buf and return the TSC value. This function assumes
2238 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2239 * PSBEND packet is found.
2241 * Return: %true if TSC is found, false otherwise.
2243 static bool intel_pt_next_tsc(unsigned char *buf
, size_t len
, uint64_t *tsc
)
2245 struct intel_pt_pkt packet
;
2249 ret
= intel_pt_get_packet(buf
, len
, &packet
);
2252 if (packet
.type
== INTEL_PT_TSC
) {
2253 *tsc
= packet
.payload
;
2256 if (packet
.type
== INTEL_PT_PSBEND
)
2265 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2266 * @tsc1: first TSC to compare
2267 * @tsc2: second TSC to compare
2269 * This function compares 7-byte TSC values allowing for the possibility that
2270 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2271 * around so for that purpose this function assumes the absolute difference is
2272 * less than half the maximum difference.
2274 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2277 static int intel_pt_tsc_cmp(uint64_t tsc1
, uint64_t tsc2
)
2279 const uint64_t halfway
= (1ULL << 55);
2285 if (tsc2
- tsc1
< halfway
)
2290 if (tsc1
- tsc2
< halfway
)
2298 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2300 * @buf_a: first buffer
2301 * @len_a: size of first buffer
2302 * @buf_b: second buffer
2303 * @len_b: size of second buffer
2305 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2306 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2307 * walk forward in @buf_b until a later TSC is found. A precondition is that
2308 * @buf_a and @buf_b are positioned at a PSB.
2310 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2311 * @buf_b + @len_b if there is no non-overlapped data.
2313 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a
,
2315 unsigned char *buf_b
,
2318 uint64_t tsc_a
, tsc_b
;
2322 p
= intel_pt_last_psb(buf_a
, len_a
);
2324 return buf_b
; /* No PSB in buf_a => no overlap */
2326 len
= len_a
- (p
- buf_a
);
2327 if (!intel_pt_next_tsc(p
, len
, &tsc_a
)) {
2328 /* The last PSB+ in buf_a is incomplete, so go back one more */
2330 p
= intel_pt_last_psb(buf_a
, len_a
);
2332 return buf_b
; /* No full PSB+ => assume no overlap */
2333 len
= len_a
- (p
- buf_a
);
2334 if (!intel_pt_next_tsc(p
, len
, &tsc_a
))
2335 return buf_b
; /* No TSC in buf_a => assume no overlap */
2339 /* Ignore PSB+ with no TSC */
2340 if (intel_pt_next_tsc(buf_b
, len_b
, &tsc_b
) &&
2341 intel_pt_tsc_cmp(tsc_a
, tsc_b
) < 0)
2342 return buf_b
; /* tsc_a < tsc_b => no overlap */
2344 if (!intel_pt_step_psb(&buf_b
, &len_b
))
2345 return buf_b
+ len_b
; /* No PSB in buf_b => no data */
2350 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2351 * @buf_a: first buffer
2352 * @len_a: size of first buffer
2353 * @buf_b: second buffer
2354 * @len_b: size of second buffer
2355 * @have_tsc: can use TSC packets to detect overlap
2357 * When trace samples or snapshots are recorded there is the possibility that
2358 * the data overlaps. Note that, for the purposes of decoding, data is only
2359 * useful if it begins with a PSB packet.
2361 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2362 * @buf_b + @len_b if there is no non-overlapped data.
2364 unsigned char *intel_pt_find_overlap(unsigned char *buf_a
, size_t len_a
,
2365 unsigned char *buf_b
, size_t len_b
,
2368 unsigned char *found
;
2370 /* Buffer 'b' must start at PSB so throw away everything before that */
2371 if (!intel_pt_next_psb(&buf_b
, &len_b
))
2372 return buf_b
+ len_b
; /* No PSB */
2374 if (!intel_pt_next_psb(&buf_a
, &len_a
))
2375 return buf_b
; /* No overlap */
2378 found
= intel_pt_find_overlap_tsc(buf_a
, len_a
, buf_b
, len_b
);
2384 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2385 * we can ignore the first part of buffer 'a'.
2387 while (len_b
< len_a
) {
2388 if (!intel_pt_step_psb(&buf_a
, &len_a
))
2389 return buf_b
; /* No overlap */
2392 /* Now len_b >= len_a */
2393 if (len_b
> len_a
) {
2394 /* The leftover buffer 'b' must start at a PSB */
2395 while (!intel_pt_at_psb(buf_b
+ len_a
, len_b
- len_a
)) {
2396 if (!intel_pt_step_psb(&buf_a
, &len_a
))
2397 return buf_b
; /* No overlap */
2402 /* Potential overlap so check the bytes */
2403 found
= memmem(buf_a
, len_a
, buf_b
, len_a
);
2405 return buf_b
+ len_a
;
2407 /* Try again at next PSB in buffer 'a' */
2408 if (!intel_pt_step_psb(&buf_a
, &len_a
))
2409 return buf_b
; /* No overlap */
2411 /* The leftover buffer 'b' must start at a PSB */
2412 while (!intel_pt_at_psb(buf_b
+ len_a
, len_b
- len_a
)) {
2413 if (!intel_pt_step_psb(&buf_a
, &len_a
))
2414 return buf_b
; /* No overlap */