2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
25 #include <linux/compiler.h>
29 #include "../auxtrace.h"
31 #include "intel-pt-insn-decoder.h"
32 #include "intel-pt-pkt-decoder.h"
33 #include "intel-pt-decoder.h"
34 #include "intel-pt-log.h"
36 #define INTEL_PT_BLK_SIZE 1024
38 #define BIT63 (((uint64_t)1 << 63))
40 #define INTEL_PT_RETURN 1
42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
43 #define INTEL_PT_MAX_LOOPS 10000
46 struct intel_pt_blk
*prev
;
47 uint64_t ip
[INTEL_PT_BLK_SIZE
];
50 struct intel_pt_stack
{
51 struct intel_pt_blk
*blk
;
52 struct intel_pt_blk
*spare
;
56 enum intel_pt_pkt_state
{
57 INTEL_PT_STATE_NO_PSB
,
59 INTEL_PT_STATE_ERR_RESYNC
,
60 INTEL_PT_STATE_IN_SYNC
,
63 INTEL_PT_STATE_TIP_PGD
,
65 INTEL_PT_STATE_FUP_NO_TIP
,
68 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state
)
71 case INTEL_PT_STATE_NO_PSB
:
72 case INTEL_PT_STATE_NO_IP
:
73 case INTEL_PT_STATE_ERR_RESYNC
:
74 case INTEL_PT_STATE_IN_SYNC
:
75 case INTEL_PT_STATE_TNT
:
77 case INTEL_PT_STATE_TIP
:
78 case INTEL_PT_STATE_TIP_PGD
:
79 case INTEL_PT_STATE_FUP
:
80 case INTEL_PT_STATE_FUP_NO_TIP
:
87 #ifdef INTEL_PT_STRICT
88 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
89 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
90 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
91 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
93 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
94 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
95 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
96 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
99 struct intel_pt_decoder
{
100 int (*get_trace
)(struct intel_pt_buffer
*buffer
, void *data
);
101 int (*walk_insn
)(struct intel_pt_insn
*intel_pt_insn
,
102 uint64_t *insn_cnt_ptr
, uint64_t *ip
, uint64_t to_ip
,
103 uint64_t max_insn_cnt
, void *data
);
104 bool (*pgd_ip
)(uint64_t ip
, void *data
);
106 struct intel_pt_state state
;
107 const unsigned char *buf
;
109 bool return_compression
;
117 enum intel_pt_param_flags flags
;
123 uint64_t tsc_timestamp
;
124 uint64_t ref_timestamp
;
125 uint64_t sample_timestamp
;
127 uint64_t ctc_timestamp
;
130 uint64_t cyc_ref_timestamp
;
132 uint32_t tsc_ctc_ratio_n
;
133 uint32_t tsc_ctc_ratio_d
;
134 uint32_t tsc_ctc_mult
;
136 uint32_t ctc_rem_mask
;
138 struct intel_pt_stack stack
;
139 enum intel_pt_pkt_state pkt_state
;
140 struct intel_pt_pkt packet
;
141 struct intel_pt_pkt tnt
;
144 int last_packet_type
;
146 unsigned int cbr_seen
;
147 unsigned int max_non_turbo_ratio
;
148 double max_non_turbo_ratio_fp
;
149 double cbr_cyc_to_tsc
;
150 double calc_cyc_to_tsc
;
151 bool have_calc_cyc_to_tsc
;
153 unsigned int insn_bytes
;
155 enum intel_pt_period_type period_type
;
156 uint64_t tot_insn_cnt
;
157 uint64_t period_insn_cnt
;
158 uint64_t period_mask
;
159 uint64_t period_ticks
;
160 uint64_t last_masked_timestamp
;
161 bool continuous_period
;
163 bool set_fup_tx_flags
;
168 unsigned int fup_tx_flags
;
169 unsigned int tx_flags
;
170 uint64_t fup_ptw_payload
;
171 uint64_t fup_mwait_payload
;
172 uint64_t fup_pwre_payload
;
173 uint64_t cbr_payload
;
174 uint64_t timestamp_insn_cnt
;
175 uint64_t sample_insn_cnt
;
180 const unsigned char *next_buf
;
182 unsigned char temp_buf
[INTEL_PT_PKT_MAX_SZ
];
185 static uint64_t intel_pt_lower_power_of_2(uint64_t x
)
189 for (i
= 0; x
!= 1; i
++)
195 static void intel_pt_setup_period(struct intel_pt_decoder
*decoder
)
197 if (decoder
->period_type
== INTEL_PT_PERIOD_TICKS
) {
200 period
= intel_pt_lower_power_of_2(decoder
->period
);
201 decoder
->period_mask
= ~(period
- 1);
202 decoder
->period_ticks
= period
;
206 static uint64_t multdiv(uint64_t t
, uint32_t n
, uint32_t d
)
210 return (t
/ d
) * n
+ ((t
% d
) * n
) / d
;
213 struct intel_pt_decoder
*intel_pt_decoder_new(struct intel_pt_params
*params
)
215 struct intel_pt_decoder
*decoder
;
217 if (!params
->get_trace
|| !params
->walk_insn
)
220 decoder
= zalloc(sizeof(struct intel_pt_decoder
));
224 decoder
->get_trace
= params
->get_trace
;
225 decoder
->walk_insn
= params
->walk_insn
;
226 decoder
->pgd_ip
= params
->pgd_ip
;
227 decoder
->data
= params
->data
;
228 decoder
->return_compression
= params
->return_compression
;
229 decoder
->branch_enable
= params
->branch_enable
;
231 decoder
->flags
= params
->flags
;
233 decoder
->period
= params
->period
;
234 decoder
->period_type
= params
->period_type
;
236 decoder
->max_non_turbo_ratio
= params
->max_non_turbo_ratio
;
237 decoder
->max_non_turbo_ratio_fp
= params
->max_non_turbo_ratio
;
239 intel_pt_setup_period(decoder
);
241 decoder
->mtc_shift
= params
->mtc_period
;
242 decoder
->ctc_rem_mask
= (1 << decoder
->mtc_shift
) - 1;
244 decoder
->tsc_ctc_ratio_n
= params
->tsc_ctc_ratio_n
;
245 decoder
->tsc_ctc_ratio_d
= params
->tsc_ctc_ratio_d
;
247 if (!decoder
->tsc_ctc_ratio_n
)
248 decoder
->tsc_ctc_ratio_d
= 0;
250 if (decoder
->tsc_ctc_ratio_d
) {
251 if (!(decoder
->tsc_ctc_ratio_n
% decoder
->tsc_ctc_ratio_d
))
252 decoder
->tsc_ctc_mult
= decoder
->tsc_ctc_ratio_n
/
253 decoder
->tsc_ctc_ratio_d
;
257 * A TSC packet can slip past MTC packets so that the timestamp appears
258 * to go backwards. One estimate is that can be up to about 40 CPU
259 * cycles, which is certainly less than 0x1000 TSC ticks, but accept
260 * slippage an order of magnitude more to be on the safe side.
262 decoder
->tsc_slip
= 0x10000;
264 intel_pt_log("timestamp: mtc_shift %u\n", decoder
->mtc_shift
);
265 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder
->tsc_ctc_ratio_n
);
266 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder
->tsc_ctc_ratio_d
);
267 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder
->tsc_ctc_mult
);
268 intel_pt_log("timestamp: tsc_slip %#x\n", decoder
->tsc_slip
);
273 static void intel_pt_pop_blk(struct intel_pt_stack
*stack
)
275 struct intel_pt_blk
*blk
= stack
->blk
;
277 stack
->blk
= blk
->prev
;
284 static uint64_t intel_pt_pop(struct intel_pt_stack
*stack
)
289 intel_pt_pop_blk(stack
);
292 stack
->pos
= INTEL_PT_BLK_SIZE
;
294 return stack
->blk
->ip
[--stack
->pos
];
297 static int intel_pt_alloc_blk(struct intel_pt_stack
*stack
)
299 struct intel_pt_blk
*blk
;
305 blk
= malloc(sizeof(struct intel_pt_blk
));
310 blk
->prev
= stack
->blk
;
316 static int intel_pt_push(struct intel_pt_stack
*stack
, uint64_t ip
)
320 if (!stack
->blk
|| stack
->pos
== INTEL_PT_BLK_SIZE
) {
321 err
= intel_pt_alloc_blk(stack
);
326 stack
->blk
->ip
[stack
->pos
++] = ip
;
330 static void intel_pt_clear_stack(struct intel_pt_stack
*stack
)
333 intel_pt_pop_blk(stack
);
337 static void intel_pt_free_stack(struct intel_pt_stack
*stack
)
339 intel_pt_clear_stack(stack
);
341 zfree(&stack
->spare
);
344 void intel_pt_decoder_free(struct intel_pt_decoder
*decoder
)
346 intel_pt_free_stack(&decoder
->stack
);
350 static int intel_pt_ext_err(int code
)
354 return INTEL_PT_ERR_NOMEM
;
356 return INTEL_PT_ERR_INTERN
;
358 return INTEL_PT_ERR_BADPKT
;
360 return INTEL_PT_ERR_NODATA
;
362 return INTEL_PT_ERR_NOINSN
;
364 return INTEL_PT_ERR_MISMAT
;
366 return INTEL_PT_ERR_OVR
;
368 return INTEL_PT_ERR_LOST
;
370 return INTEL_PT_ERR_NELOOP
;
372 return INTEL_PT_ERR_UNK
;
376 static const char *intel_pt_err_msgs
[] = {
377 [INTEL_PT_ERR_NOMEM
] = "Memory allocation failed",
378 [INTEL_PT_ERR_INTERN
] = "Internal error",
379 [INTEL_PT_ERR_BADPKT
] = "Bad packet",
380 [INTEL_PT_ERR_NODATA
] = "No more data",
381 [INTEL_PT_ERR_NOINSN
] = "Failed to get instruction",
382 [INTEL_PT_ERR_MISMAT
] = "Trace doesn't match instruction",
383 [INTEL_PT_ERR_OVR
] = "Overflow packet",
384 [INTEL_PT_ERR_LOST
] = "Lost trace data",
385 [INTEL_PT_ERR_UNK
] = "Unknown error!",
386 [INTEL_PT_ERR_NELOOP
] = "Never-ending loop",
389 int intel_pt__strerror(int code
, char *buf
, size_t buflen
)
391 if (code
< 1 || code
>= INTEL_PT_ERR_MAX
)
392 code
= INTEL_PT_ERR_UNK
;
393 strlcpy(buf
, intel_pt_err_msgs
[code
], buflen
);
397 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt
*packet
,
402 switch (packet
->count
) {
404 ip
= (last_ip
& (uint64_t)0xffffffffffff0000ULL
) |
408 ip
= (last_ip
& (uint64_t)0xffffffff00000000ULL
) |
412 ip
= packet
->payload
;
413 /* Sign-extend 6-byte ip */
414 if (ip
& (uint64_t)0x800000000000ULL
)
415 ip
|= (uint64_t)0xffff000000000000ULL
;
418 ip
= (last_ip
& (uint64_t)0xffff000000000000ULL
) |
422 ip
= packet
->payload
;
431 static inline void intel_pt_set_last_ip(struct intel_pt_decoder
*decoder
)
433 decoder
->last_ip
= intel_pt_calc_ip(&decoder
->packet
, decoder
->last_ip
);
434 decoder
->have_last_ip
= true;
437 static inline void intel_pt_set_ip(struct intel_pt_decoder
*decoder
)
439 intel_pt_set_last_ip(decoder
);
440 decoder
->ip
= decoder
->last_ip
;
443 static void intel_pt_decoder_log_packet(struct intel_pt_decoder
*decoder
)
445 intel_pt_log_packet(&decoder
->packet
, decoder
->pkt_len
, decoder
->pos
,
449 static int intel_pt_bug(struct intel_pt_decoder
*decoder
)
451 intel_pt_log("ERROR: Internal error\n");
452 decoder
->pkt_state
= INTEL_PT_STATE_NO_PSB
;
456 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder
*decoder
)
458 decoder
->tx_flags
= 0;
461 static inline void intel_pt_update_in_tx(struct intel_pt_decoder
*decoder
)
463 decoder
->tx_flags
= decoder
->packet
.payload
& INTEL_PT_IN_TX
;
466 static int intel_pt_bad_packet(struct intel_pt_decoder
*decoder
)
468 intel_pt_clear_tx_flags(decoder
);
469 decoder
->have_tma
= false;
470 decoder
->pkt_len
= 1;
471 decoder
->pkt_step
= 1;
472 intel_pt_decoder_log_packet(decoder
);
473 if (decoder
->pkt_state
!= INTEL_PT_STATE_NO_PSB
) {
474 intel_pt_log("ERROR: Bad packet\n");
475 decoder
->pkt_state
= INTEL_PT_STATE_ERR1
;
480 static int intel_pt_get_data(struct intel_pt_decoder
*decoder
)
482 struct intel_pt_buffer buffer
= { .buf
= 0, };
485 decoder
->pkt_step
= 0;
487 intel_pt_log("Getting more data\n");
488 ret
= decoder
->get_trace(&buffer
, decoder
->data
);
491 decoder
->buf
= buffer
.buf
;
492 decoder
->len
= buffer
.len
;
494 intel_pt_log("No more data\n");
497 if (!buffer
.consecutive
) {
499 decoder
->pkt_state
= INTEL_PT_STATE_NO_PSB
;
500 decoder
->ref_timestamp
= buffer
.ref_timestamp
;
501 decoder
->timestamp
= 0;
502 decoder
->have_tma
= false;
503 decoder
->state
.trace_nr
= buffer
.trace_nr
;
504 intel_pt_log("Reference timestamp 0x%" PRIx64
"\n",
505 decoder
->ref_timestamp
);
512 static int intel_pt_get_next_data(struct intel_pt_decoder
*decoder
)
514 if (!decoder
->next_buf
)
515 return intel_pt_get_data(decoder
);
517 decoder
->buf
= decoder
->next_buf
;
518 decoder
->len
= decoder
->next_len
;
519 decoder
->next_buf
= 0;
520 decoder
->next_len
= 0;
524 static int intel_pt_get_split_packet(struct intel_pt_decoder
*decoder
)
526 unsigned char *buf
= decoder
->temp_buf
;
527 size_t old_len
, len
, n
;
530 old_len
= decoder
->len
;
532 memcpy(buf
, decoder
->buf
, len
);
534 ret
= intel_pt_get_data(decoder
);
536 decoder
->pos
+= old_len
;
537 return ret
< 0 ? ret
: -EINVAL
;
540 n
= INTEL_PT_PKT_MAX_SZ
- len
;
541 if (n
> decoder
->len
)
543 memcpy(buf
+ len
, decoder
->buf
, n
);
546 ret
= intel_pt_get_packet(buf
, len
, &decoder
->packet
);
547 if (ret
< (int)old_len
) {
548 decoder
->next_buf
= decoder
->buf
;
549 decoder
->next_len
= decoder
->len
;
551 decoder
->len
= old_len
;
552 return intel_pt_bad_packet(decoder
);
555 decoder
->next_buf
= decoder
->buf
+ (ret
- old_len
);
556 decoder
->next_len
= decoder
->len
- (ret
- old_len
);
564 struct intel_pt_pkt_info
{
565 struct intel_pt_decoder
*decoder
;
566 struct intel_pt_pkt packet
;
569 int last_packet_type
;
573 typedef int (*intel_pt_pkt_cb_t
)(struct intel_pt_pkt_info
*pkt_info
);
575 /* Lookahead packets in current buffer */
576 static int intel_pt_pkt_lookahead(struct intel_pt_decoder
*decoder
,
577 intel_pt_pkt_cb_t cb
, void *data
)
579 struct intel_pt_pkt_info pkt_info
;
580 const unsigned char *buf
= decoder
->buf
;
581 size_t len
= decoder
->len
;
584 pkt_info
.decoder
= decoder
;
585 pkt_info
.pos
= decoder
->pos
;
586 pkt_info
.pkt_len
= decoder
->pkt_step
;
587 pkt_info
.last_packet_type
= decoder
->last_packet_type
;
588 pkt_info
.data
= data
;
592 pkt_info
.pos
+= pkt_info
.pkt_len
;
593 buf
+= pkt_info
.pkt_len
;
594 len
-= pkt_info
.pkt_len
;
597 return INTEL_PT_NEED_MORE_BYTES
;
599 ret
= intel_pt_get_packet(buf
, len
, &pkt_info
.packet
);
601 return INTEL_PT_NEED_MORE_BYTES
;
605 pkt_info
.pkt_len
= ret
;
606 } while (pkt_info
.packet
.type
== INTEL_PT_PAD
);
612 pkt_info
.last_packet_type
= pkt_info
.packet
.type
;
616 struct intel_pt_calc_cyc_to_tsc_info
{
620 uint64_t ctc_timestamp
;
622 uint64_t tsc_timestamp
;
627 double cbr_cyc_to_tsc
;
631 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
632 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
633 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
634 * packet by copying the missing bits from the current MTC assuming the least
635 * difference between the two, and that the current MTC comes after last_mtc.
637 static void intel_pt_fixup_last_mtc(uint32_t mtc
, int mtc_shift
,
640 uint32_t first_missing_bit
= 1U << (16 - mtc_shift
);
641 uint32_t mask
= ~(first_missing_bit
- 1);
643 *last_mtc
|= mtc
& mask
;
644 if (*last_mtc
>= mtc
) {
645 *last_mtc
-= first_missing_bit
;
650 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info
*pkt_info
)
652 struct intel_pt_decoder
*decoder
= pkt_info
->decoder
;
653 struct intel_pt_calc_cyc_to_tsc_info
*data
= pkt_info
->data
;
657 uint32_t mtc
, mtc_delta
, ctc
, fc
, ctc_rem
;
659 switch (pkt_info
->packet
.type
) {
661 case INTEL_PT_TIP_PGE
:
666 case INTEL_PT_MODE_EXEC
:
667 case INTEL_PT_MODE_TSX
:
668 case INTEL_PT_PSBEND
:
672 case INTEL_PT_PTWRITE
:
673 case INTEL_PT_PTWRITE_IP
:
680 mtc
= pkt_info
->packet
.payload
;
681 if (decoder
->mtc_shift
> 8 && data
->fixup_last_mtc
) {
682 data
->fixup_last_mtc
= false;
683 intel_pt_fixup_last_mtc(mtc
, decoder
->mtc_shift
,
686 if (mtc
> data
->last_mtc
)
687 mtc_delta
= mtc
- data
->last_mtc
;
689 mtc_delta
= mtc
+ 256 - data
->last_mtc
;
690 data
->ctc_delta
+= mtc_delta
<< decoder
->mtc_shift
;
691 data
->last_mtc
= mtc
;
693 if (decoder
->tsc_ctc_mult
) {
694 timestamp
= data
->ctc_timestamp
+
695 data
->ctc_delta
* decoder
->tsc_ctc_mult
;
697 timestamp
= data
->ctc_timestamp
+
698 multdiv(data
->ctc_delta
,
699 decoder
->tsc_ctc_ratio_n
,
700 decoder
->tsc_ctc_ratio_d
);
703 if (timestamp
< data
->timestamp
)
706 if (pkt_info
->last_packet_type
!= INTEL_PT_CYC
) {
707 data
->timestamp
= timestamp
;
715 * For now, do not support using TSC packets - refer
716 * intel_pt_calc_cyc_to_tsc().
720 timestamp
= pkt_info
->packet
.payload
|
721 (data
->timestamp
& (0xffULL
<< 56));
722 if (data
->from_mtc
&& timestamp
< data
->timestamp
&&
723 data
->timestamp
- timestamp
< decoder
->tsc_slip
)
725 if (timestamp
< data
->timestamp
)
726 timestamp
+= (1ULL << 56);
727 if (pkt_info
->last_packet_type
!= INTEL_PT_CYC
) {
730 data
->tsc_timestamp
= timestamp
;
731 data
->timestamp
= timestamp
;
740 if (!decoder
->tsc_ctc_ratio_d
)
743 ctc
= pkt_info
->packet
.payload
;
744 fc
= pkt_info
->packet
.count
;
745 ctc_rem
= ctc
& decoder
->ctc_rem_mask
;
747 data
->last_mtc
= (ctc
>> decoder
->mtc_shift
) & 0xff;
749 data
->ctc_timestamp
= data
->tsc_timestamp
- fc
;
750 if (decoder
->tsc_ctc_mult
) {
751 data
->ctc_timestamp
-= ctc_rem
* decoder
->tsc_ctc_mult
;
753 data
->ctc_timestamp
-=
754 multdiv(ctc_rem
, decoder
->tsc_ctc_ratio_n
,
755 decoder
->tsc_ctc_ratio_d
);
759 data
->have_tma
= true;
760 data
->fixup_last_mtc
= true;
765 data
->cycle_cnt
+= pkt_info
->packet
.payload
;
769 cbr
= pkt_info
->packet
.payload
;
770 if (data
->cbr
&& data
->cbr
!= cbr
)
773 data
->cbr_cyc_to_tsc
= decoder
->max_non_turbo_ratio_fp
/ cbr
;
776 case INTEL_PT_TIP_PGD
:
777 case INTEL_PT_TRACESTOP
:
778 case INTEL_PT_EXSTOP
:
779 case INTEL_PT_EXSTOP_IP
:
784 case INTEL_PT_BAD
: /* Does not happen */
789 if (!data
->cbr
&& decoder
->cbr
) {
790 data
->cbr
= decoder
->cbr
;
791 data
->cbr_cyc_to_tsc
= decoder
->cbr_cyc_to_tsc
;
794 if (!data
->cycle_cnt
)
797 cyc_to_tsc
= (double)(timestamp
- decoder
->timestamp
) / data
->cycle_cnt
;
799 if (data
->cbr
&& cyc_to_tsc
> data
->cbr_cyc_to_tsc
&&
800 cyc_to_tsc
/ data
->cbr_cyc_to_tsc
> 1.25) {
801 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt
"\n",
802 cyc_to_tsc
, data
->cbr_cyc_to_tsc
, pkt_info
->pos
);
806 decoder
->calc_cyc_to_tsc
= cyc_to_tsc
;
807 decoder
->have_calc_cyc_to_tsc
= true;
810 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt
"\n",
811 cyc_to_tsc
, data
->cbr_cyc_to_tsc
, pkt_info
->pos
);
813 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt
"\n",
814 cyc_to_tsc
, pkt_info
->pos
);
820 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder
*decoder
,
823 struct intel_pt_calc_cyc_to_tsc_info data
= {
826 .last_mtc
= decoder
->last_mtc
,
827 .ctc_timestamp
= decoder
->ctc_timestamp
,
828 .ctc_delta
= decoder
->ctc_delta
,
829 .tsc_timestamp
= decoder
->tsc_timestamp
,
830 .timestamp
= decoder
->timestamp
,
831 .have_tma
= decoder
->have_tma
,
832 .fixup_last_mtc
= decoder
->fixup_last_mtc
,
833 .from_mtc
= from_mtc
,
838 * For now, do not support using TSC packets for at least the reasons:
839 * 1) timing might have stopped
840 * 2) TSC packets within PSB+ can slip against CYC packets
845 intel_pt_pkt_lookahead(decoder
, intel_pt_calc_cyc_cb
, &data
);
848 static int intel_pt_get_next_packet(struct intel_pt_decoder
*decoder
)
852 decoder
->last_packet_type
= decoder
->packet
.type
;
855 decoder
->pos
+= decoder
->pkt_step
;
856 decoder
->buf
+= decoder
->pkt_step
;
857 decoder
->len
-= decoder
->pkt_step
;
860 ret
= intel_pt_get_next_data(decoder
);
865 ret
= intel_pt_get_packet(decoder
->buf
, decoder
->len
,
867 if (ret
== INTEL_PT_NEED_MORE_BYTES
&& BITS_PER_LONG
== 32 &&
868 decoder
->len
< INTEL_PT_PKT_MAX_SZ
&& !decoder
->next_buf
) {
869 ret
= intel_pt_get_split_packet(decoder
);
874 return intel_pt_bad_packet(decoder
);
876 decoder
->pkt_len
= ret
;
877 decoder
->pkt_step
= ret
;
878 intel_pt_decoder_log_packet(decoder
);
879 } while (decoder
->packet
.type
== INTEL_PT_PAD
);
884 static uint64_t intel_pt_next_period(struct intel_pt_decoder
*decoder
)
886 uint64_t timestamp
, masked_timestamp
;
888 timestamp
= decoder
->timestamp
+ decoder
->timestamp_insn_cnt
;
889 masked_timestamp
= timestamp
& decoder
->period_mask
;
890 if (decoder
->continuous_period
) {
891 if (masked_timestamp
!= decoder
->last_masked_timestamp
)
895 masked_timestamp
= timestamp
& decoder
->period_mask
;
896 if (masked_timestamp
!= decoder
->last_masked_timestamp
) {
897 decoder
->last_masked_timestamp
= masked_timestamp
;
898 decoder
->continuous_period
= true;
901 return decoder
->period_ticks
- (timestamp
- masked_timestamp
);
904 static uint64_t intel_pt_next_sample(struct intel_pt_decoder
*decoder
)
906 switch (decoder
->period_type
) {
907 case INTEL_PT_PERIOD_INSTRUCTIONS
:
908 return decoder
->period
- decoder
->period_insn_cnt
;
909 case INTEL_PT_PERIOD_TICKS
:
910 return intel_pt_next_period(decoder
);
911 case INTEL_PT_PERIOD_NONE
:
912 case INTEL_PT_PERIOD_MTC
:
918 static void intel_pt_sample_insn(struct intel_pt_decoder
*decoder
)
920 uint64_t timestamp
, masked_timestamp
;
922 switch (decoder
->period_type
) {
923 case INTEL_PT_PERIOD_INSTRUCTIONS
:
924 decoder
->period_insn_cnt
= 0;
926 case INTEL_PT_PERIOD_TICKS
:
927 timestamp
= decoder
->timestamp
+ decoder
->timestamp_insn_cnt
;
928 masked_timestamp
= timestamp
& decoder
->period_mask
;
929 decoder
->last_masked_timestamp
= masked_timestamp
;
931 case INTEL_PT_PERIOD_NONE
:
932 case INTEL_PT_PERIOD_MTC
:
937 decoder
->state
.type
|= INTEL_PT_INSTRUCTION
;
940 static int intel_pt_walk_insn(struct intel_pt_decoder
*decoder
,
941 struct intel_pt_insn
*intel_pt_insn
, uint64_t ip
)
943 uint64_t max_insn_cnt
, insn_cnt
= 0;
946 if (!decoder
->mtc_insn
)
947 decoder
->mtc_insn
= true;
949 max_insn_cnt
= intel_pt_next_sample(decoder
);
951 err
= decoder
->walk_insn(intel_pt_insn
, &insn_cnt
, &decoder
->ip
, ip
,
952 max_insn_cnt
, decoder
->data
);
954 decoder
->tot_insn_cnt
+= insn_cnt
;
955 decoder
->timestamp_insn_cnt
+= insn_cnt
;
956 decoder
->sample_insn_cnt
+= insn_cnt
;
957 decoder
->period_insn_cnt
+= insn_cnt
;
960 decoder
->no_progress
= 0;
961 decoder
->pkt_state
= INTEL_PT_STATE_ERR2
;
962 intel_pt_log_at("ERROR: Failed to get instruction",
969 if (ip
&& decoder
->ip
== ip
) {
974 if (max_insn_cnt
&& insn_cnt
>= max_insn_cnt
)
975 intel_pt_sample_insn(decoder
);
977 if (intel_pt_insn
->branch
== INTEL_PT_BR_NO_BRANCH
) {
978 decoder
->state
.type
= INTEL_PT_INSTRUCTION
;
979 decoder
->state
.from_ip
= decoder
->ip
;
980 decoder
->state
.to_ip
= 0;
981 decoder
->ip
+= intel_pt_insn
->length
;
982 err
= INTEL_PT_RETURN
;
986 if (intel_pt_insn
->op
== INTEL_PT_OP_CALL
) {
987 /* Zero-length calls are excluded */
988 if (intel_pt_insn
->branch
!= INTEL_PT_BR_UNCONDITIONAL
||
989 intel_pt_insn
->rel
) {
990 err
= intel_pt_push(&decoder
->stack
, decoder
->ip
+
991 intel_pt_insn
->length
);
995 } else if (intel_pt_insn
->op
== INTEL_PT_OP_RET
) {
996 decoder
->ret_addr
= intel_pt_pop(&decoder
->stack
);
999 if (intel_pt_insn
->branch
== INTEL_PT_BR_UNCONDITIONAL
) {
1000 int cnt
= decoder
->no_progress
++;
1002 decoder
->state
.from_ip
= decoder
->ip
;
1003 decoder
->ip
+= intel_pt_insn
->length
+
1005 decoder
->state
.to_ip
= decoder
->ip
;
1006 err
= INTEL_PT_RETURN
;
1009 * Check for being stuck in a loop. This can happen if a
1010 * decoder error results in the decoder erroneously setting the
1011 * ip to an address that is itself in an infinite loop that
1012 * consumes no packets. When that happens, there must be an
1013 * unconditional branch.
1017 decoder
->stuck_ip
= decoder
->state
.to_ip
;
1018 decoder
->stuck_ip_prd
= 1;
1019 decoder
->stuck_ip_cnt
= 1;
1020 } else if (cnt
> INTEL_PT_MAX_LOOPS
||
1021 decoder
->state
.to_ip
== decoder
->stuck_ip
) {
1022 intel_pt_log_at("ERROR: Never-ending loop",
1023 decoder
->state
.to_ip
);
1024 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1027 } else if (!--decoder
->stuck_ip_cnt
) {
1028 decoder
->stuck_ip_prd
+= 1;
1029 decoder
->stuck_ip_cnt
= decoder
->stuck_ip_prd
;
1030 decoder
->stuck_ip
= decoder
->state
.to_ip
;
1033 goto out_no_progress
;
1036 decoder
->no_progress
= 0;
1038 decoder
->state
.insn_op
= intel_pt_insn
->op
;
1039 decoder
->state
.insn_len
= intel_pt_insn
->length
;
1040 memcpy(decoder
->state
.insn
, intel_pt_insn
->buf
,
1041 INTEL_PT_INSN_BUF_SZ
);
1043 if (decoder
->tx_flags
& INTEL_PT_IN_TX
)
1044 decoder
->state
.flags
|= INTEL_PT_IN_TX
;
1049 static bool intel_pt_fup_event(struct intel_pt_decoder
*decoder
)
1053 if (decoder
->set_fup_tx_flags
) {
1054 decoder
->set_fup_tx_flags
= false;
1055 decoder
->tx_flags
= decoder
->fup_tx_flags
;
1056 decoder
->state
.type
= INTEL_PT_TRANSACTION
;
1057 decoder
->state
.from_ip
= decoder
->ip
;
1058 decoder
->state
.to_ip
= 0;
1059 decoder
->state
.flags
= decoder
->fup_tx_flags
;
1062 if (decoder
->set_fup_ptw
) {
1063 decoder
->set_fup_ptw
= false;
1064 decoder
->state
.type
= INTEL_PT_PTW
;
1065 decoder
->state
.flags
|= INTEL_PT_FUP_IP
;
1066 decoder
->state
.from_ip
= decoder
->ip
;
1067 decoder
->state
.to_ip
= 0;
1068 decoder
->state
.ptw_payload
= decoder
->fup_ptw_payload
;
1071 if (decoder
->set_fup_mwait
) {
1072 decoder
->set_fup_mwait
= false;
1073 decoder
->state
.type
= INTEL_PT_MWAIT_OP
;
1074 decoder
->state
.from_ip
= decoder
->ip
;
1075 decoder
->state
.to_ip
= 0;
1076 decoder
->state
.mwait_payload
= decoder
->fup_mwait_payload
;
1079 if (decoder
->set_fup_pwre
) {
1080 decoder
->set_fup_pwre
= false;
1081 decoder
->state
.type
|= INTEL_PT_PWR_ENTRY
;
1082 decoder
->state
.type
&= ~INTEL_PT_BRANCH
;
1083 decoder
->state
.from_ip
= decoder
->ip
;
1084 decoder
->state
.to_ip
= 0;
1085 decoder
->state
.pwre_payload
= decoder
->fup_pwre_payload
;
1088 if (decoder
->set_fup_exstop
) {
1089 decoder
->set_fup_exstop
= false;
1090 decoder
->state
.type
|= INTEL_PT_EX_STOP
;
1091 decoder
->state
.type
&= ~INTEL_PT_BRANCH
;
1092 decoder
->state
.flags
|= INTEL_PT_FUP_IP
;
1093 decoder
->state
.from_ip
= decoder
->ip
;
1094 decoder
->state
.to_ip
= 0;
1100 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder
*decoder
,
1101 struct intel_pt_insn
*intel_pt_insn
,
1102 uint64_t ip
, int err
)
1104 return decoder
->flags
& INTEL_PT_FUP_WITH_NLIP
&& !err
&&
1105 intel_pt_insn
->branch
== INTEL_PT_BR_INDIRECT
&&
1106 ip
== decoder
->ip
+ intel_pt_insn
->length
;
1109 static int intel_pt_walk_fup(struct intel_pt_decoder
*decoder
)
1111 struct intel_pt_insn intel_pt_insn
;
1115 ip
= decoder
->last_ip
;
1118 err
= intel_pt_walk_insn(decoder
, &intel_pt_insn
, ip
);
1119 if (err
== INTEL_PT_RETURN
)
1121 if (err
== -EAGAIN
||
1122 intel_pt_fup_with_nlip(decoder
, &intel_pt_insn
, ip
, err
)) {
1123 if (intel_pt_fup_event(decoder
))
1127 decoder
->set_fup_tx_flags
= false;
1131 if (intel_pt_insn
.branch
== INTEL_PT_BR_INDIRECT
) {
1132 intel_pt_log_at("ERROR: Unexpected indirect branch",
1134 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1138 if (intel_pt_insn
.branch
== INTEL_PT_BR_CONDITIONAL
) {
1139 intel_pt_log_at("ERROR: Unexpected conditional branch",
1141 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1145 intel_pt_bug(decoder
);
1149 static int intel_pt_walk_tip(struct intel_pt_decoder
*decoder
)
1151 struct intel_pt_insn intel_pt_insn
;
1154 err
= intel_pt_walk_insn(decoder
, &intel_pt_insn
, 0);
1155 if (err
== INTEL_PT_RETURN
&&
1157 decoder
->pkt_state
== INTEL_PT_STATE_TIP_PGD
&&
1158 (decoder
->state
.type
& INTEL_PT_BRANCH
) &&
1159 decoder
->pgd_ip(decoder
->state
.to_ip
, decoder
->data
)) {
1160 /* Unconditional branch leaving filter region */
1161 decoder
->no_progress
= 0;
1162 decoder
->pge
= false;
1163 decoder
->continuous_period
= false;
1164 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1165 decoder
->state
.type
|= INTEL_PT_TRACE_END
;
1168 if (err
== INTEL_PT_RETURN
)
1173 if (intel_pt_insn
.branch
== INTEL_PT_BR_INDIRECT
) {
1174 if (decoder
->pkt_state
== INTEL_PT_STATE_TIP_PGD
) {
1175 decoder
->pge
= false;
1176 decoder
->continuous_period
= false;
1177 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1178 decoder
->state
.from_ip
= decoder
->ip
;
1179 if (decoder
->packet
.count
== 0) {
1180 decoder
->state
.to_ip
= 0;
1182 decoder
->state
.to_ip
= decoder
->last_ip
;
1183 decoder
->ip
= decoder
->last_ip
;
1185 decoder
->state
.type
|= INTEL_PT_TRACE_END
;
1187 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1188 decoder
->state
.from_ip
= decoder
->ip
;
1189 if (decoder
->packet
.count
== 0) {
1190 decoder
->state
.to_ip
= 0;
1192 decoder
->state
.to_ip
= decoder
->last_ip
;
1193 decoder
->ip
= decoder
->last_ip
;
1199 if (intel_pt_insn
.branch
== INTEL_PT_BR_CONDITIONAL
) {
1200 uint64_t to_ip
= decoder
->ip
+ intel_pt_insn
.length
+
1203 if (decoder
->pgd_ip
&&
1204 decoder
->pkt_state
== INTEL_PT_STATE_TIP_PGD
&&
1205 decoder
->pgd_ip(to_ip
, decoder
->data
)) {
1206 /* Conditional branch leaving filter region */
1207 decoder
->pge
= false;
1208 decoder
->continuous_period
= false;
1209 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1210 decoder
->ip
= to_ip
;
1211 decoder
->state
.from_ip
= decoder
->ip
;
1212 decoder
->state
.to_ip
= to_ip
;
1213 decoder
->state
.type
|= INTEL_PT_TRACE_END
;
1216 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1218 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1222 return intel_pt_bug(decoder
);
1225 static int intel_pt_walk_tnt(struct intel_pt_decoder
*decoder
)
1227 struct intel_pt_insn intel_pt_insn
;
1231 err
= intel_pt_walk_insn(decoder
, &intel_pt_insn
, 0);
1232 if (err
== INTEL_PT_RETURN
)
1237 if (intel_pt_insn
.op
== INTEL_PT_OP_RET
) {
1238 if (!decoder
->return_compression
) {
1239 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1241 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1244 if (!decoder
->ret_addr
) {
1245 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1247 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1250 if (!(decoder
->tnt
.payload
& BIT63
)) {
1251 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1253 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1256 decoder
->tnt
.count
-= 1;
1257 if (!decoder
->tnt
.count
)
1258 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1259 decoder
->tnt
.payload
<<= 1;
1260 decoder
->state
.from_ip
= decoder
->ip
;
1261 decoder
->ip
= decoder
->ret_addr
;
1262 decoder
->state
.to_ip
= decoder
->ip
;
1266 if (intel_pt_insn
.branch
== INTEL_PT_BR_INDIRECT
) {
1267 /* Handle deferred TIPs */
1268 err
= intel_pt_get_next_packet(decoder
);
1271 if (decoder
->packet
.type
!= INTEL_PT_TIP
||
1272 decoder
->packet
.count
== 0) {
1273 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1275 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1276 decoder
->pkt_step
= 0;
1279 intel_pt_set_last_ip(decoder
);
1280 decoder
->state
.from_ip
= decoder
->ip
;
1281 decoder
->state
.to_ip
= decoder
->last_ip
;
1282 decoder
->ip
= decoder
->last_ip
;
1286 if (intel_pt_insn
.branch
== INTEL_PT_BR_CONDITIONAL
) {
1287 decoder
->tnt
.count
-= 1;
1288 if (!decoder
->tnt
.count
)
1289 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
1290 if (decoder
->tnt
.payload
& BIT63
) {
1291 decoder
->tnt
.payload
<<= 1;
1292 decoder
->state
.from_ip
= decoder
->ip
;
1293 decoder
->ip
+= intel_pt_insn
.length
+
1295 decoder
->state
.to_ip
= decoder
->ip
;
1298 /* Instruction sample for a non-taken branch */
1299 if (decoder
->state
.type
& INTEL_PT_INSTRUCTION
) {
1300 decoder
->tnt
.payload
<<= 1;
1301 decoder
->state
.type
= INTEL_PT_INSTRUCTION
;
1302 decoder
->state
.from_ip
= decoder
->ip
;
1303 decoder
->state
.to_ip
= 0;
1304 decoder
->ip
+= intel_pt_insn
.length
;
1307 decoder
->ip
+= intel_pt_insn
.length
;
1308 if (!decoder
->tnt
.count
)
1310 decoder
->tnt
.payload
<<= 1;
1314 return intel_pt_bug(decoder
);
1318 static int intel_pt_mode_tsx(struct intel_pt_decoder
*decoder
, bool *no_tip
)
1320 unsigned int fup_tx_flags
;
1323 fup_tx_flags
= decoder
->packet
.payload
&
1324 (INTEL_PT_IN_TX
| INTEL_PT_ABORT_TX
);
1325 err
= intel_pt_get_next_packet(decoder
);
1328 if (decoder
->packet
.type
== INTEL_PT_FUP
) {
1329 decoder
->fup_tx_flags
= fup_tx_flags
;
1330 decoder
->set_fup_tx_flags
= true;
1331 if (!(decoder
->fup_tx_flags
& INTEL_PT_ABORT_TX
))
1334 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1336 intel_pt_update_in_tx(decoder
);
1341 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder
*decoder
)
1345 decoder
->have_tma
= false;
1347 if (decoder
->ref_timestamp
) {
1348 timestamp
= decoder
->packet
.payload
|
1349 (decoder
->ref_timestamp
& (0xffULL
<< 56));
1350 if (timestamp
< decoder
->ref_timestamp
) {
1351 if (decoder
->ref_timestamp
- timestamp
> (1ULL << 55))
1352 timestamp
+= (1ULL << 56);
1354 if (timestamp
- decoder
->ref_timestamp
> (1ULL << 55))
1355 timestamp
-= (1ULL << 56);
1357 decoder
->tsc_timestamp
= timestamp
;
1358 decoder
->timestamp
= timestamp
;
1359 decoder
->ref_timestamp
= 0;
1360 decoder
->timestamp_insn_cnt
= 0;
1361 } else if (decoder
->timestamp
) {
1362 timestamp
= decoder
->packet
.payload
|
1363 (decoder
->timestamp
& (0xffULL
<< 56));
1364 decoder
->tsc_timestamp
= timestamp
;
1365 if (timestamp
< decoder
->timestamp
&&
1366 decoder
->timestamp
- timestamp
< decoder
->tsc_slip
) {
1367 intel_pt_log_to("Suppressing backwards timestamp",
1369 timestamp
= decoder
->timestamp
;
1371 if (timestamp
< decoder
->timestamp
) {
1372 intel_pt_log_to("Wraparound timestamp", timestamp
);
1373 timestamp
+= (1ULL << 56);
1374 decoder
->tsc_timestamp
= timestamp
;
1376 decoder
->timestamp
= timestamp
;
1377 decoder
->timestamp_insn_cnt
= 0;
1380 if (decoder
->last_packet_type
== INTEL_PT_CYC
) {
1381 decoder
->cyc_ref_timestamp
= decoder
->timestamp
;
1382 decoder
->cycle_cnt
= 0;
1383 decoder
->have_calc_cyc_to_tsc
= false;
1384 intel_pt_calc_cyc_to_tsc(decoder
, false);
1387 intel_pt_log_to("Setting timestamp", decoder
->timestamp
);
1390 static int intel_pt_overflow(struct intel_pt_decoder
*decoder
)
1392 intel_pt_log("ERROR: Buffer overflow\n");
1393 intel_pt_clear_tx_flags(decoder
);
1394 decoder
->timestamp_insn_cnt
= 0;
1395 decoder
->pkt_state
= INTEL_PT_STATE_ERR_RESYNC
;
1396 decoder
->overflow
= true;
1400 static void intel_pt_calc_tma(struct intel_pt_decoder
*decoder
)
1402 uint32_t ctc
= decoder
->packet
.payload
;
1403 uint32_t fc
= decoder
->packet
.count
;
1404 uint32_t ctc_rem
= ctc
& decoder
->ctc_rem_mask
;
1406 if (!decoder
->tsc_ctc_ratio_d
)
1409 decoder
->last_mtc
= (ctc
>> decoder
->mtc_shift
) & 0xff;
1410 decoder
->ctc_timestamp
= decoder
->tsc_timestamp
- fc
;
1411 if (decoder
->tsc_ctc_mult
) {
1412 decoder
->ctc_timestamp
-= ctc_rem
* decoder
->tsc_ctc_mult
;
1414 decoder
->ctc_timestamp
-= multdiv(ctc_rem
,
1415 decoder
->tsc_ctc_ratio_n
,
1416 decoder
->tsc_ctc_ratio_d
);
1418 decoder
->ctc_delta
= 0;
1419 decoder
->have_tma
= true;
1420 decoder
->fixup_last_mtc
= true;
1421 intel_pt_log("CTC timestamp " x64_fmt
" last MTC %#x CTC rem %#x\n",
1422 decoder
->ctc_timestamp
, decoder
->last_mtc
, ctc_rem
);
1425 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder
*decoder
)
1428 uint32_t mtc
, mtc_delta
;
1430 if (!decoder
->have_tma
)
1433 mtc
= decoder
->packet
.payload
;
1435 if (decoder
->mtc_shift
> 8 && decoder
->fixup_last_mtc
) {
1436 decoder
->fixup_last_mtc
= false;
1437 intel_pt_fixup_last_mtc(mtc
, decoder
->mtc_shift
,
1438 &decoder
->last_mtc
);
1441 if (mtc
> decoder
->last_mtc
)
1442 mtc_delta
= mtc
- decoder
->last_mtc
;
1444 mtc_delta
= mtc
+ 256 - decoder
->last_mtc
;
1446 decoder
->ctc_delta
+= mtc_delta
<< decoder
->mtc_shift
;
1448 if (decoder
->tsc_ctc_mult
) {
1449 timestamp
= decoder
->ctc_timestamp
+
1450 decoder
->ctc_delta
* decoder
->tsc_ctc_mult
;
1452 timestamp
= decoder
->ctc_timestamp
+
1453 multdiv(decoder
->ctc_delta
,
1454 decoder
->tsc_ctc_ratio_n
,
1455 decoder
->tsc_ctc_ratio_d
);
1458 if (timestamp
< decoder
->timestamp
)
1459 intel_pt_log("Suppressing MTC timestamp " x64_fmt
" less than current timestamp " x64_fmt
"\n",
1460 timestamp
, decoder
->timestamp
);
1462 decoder
->timestamp
= timestamp
;
1464 decoder
->timestamp_insn_cnt
= 0;
1465 decoder
->last_mtc
= mtc
;
1467 if (decoder
->last_packet_type
== INTEL_PT_CYC
) {
1468 decoder
->cyc_ref_timestamp
= decoder
->timestamp
;
1469 decoder
->cycle_cnt
= 0;
1470 decoder
->have_calc_cyc_to_tsc
= false;
1471 intel_pt_calc_cyc_to_tsc(decoder
, true);
1474 intel_pt_log_to("Setting timestamp", decoder
->timestamp
);
1477 static void intel_pt_calc_cbr(struct intel_pt_decoder
*decoder
)
1479 unsigned int cbr
= decoder
->packet
.payload
& 0xff;
1481 decoder
->cbr_payload
= decoder
->packet
.payload
;
1483 if (decoder
->cbr
== cbr
)
1487 decoder
->cbr_cyc_to_tsc
= decoder
->max_non_turbo_ratio_fp
/ cbr
;
1490 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder
*decoder
)
1492 uint64_t timestamp
= decoder
->cyc_ref_timestamp
;
1494 decoder
->have_cyc
= true;
1496 decoder
->cycle_cnt
+= decoder
->packet
.payload
;
1498 if (!decoder
->cyc_ref_timestamp
)
1501 if (decoder
->have_calc_cyc_to_tsc
)
1502 timestamp
+= decoder
->cycle_cnt
* decoder
->calc_cyc_to_tsc
;
1503 else if (decoder
->cbr
)
1504 timestamp
+= decoder
->cycle_cnt
* decoder
->cbr_cyc_to_tsc
;
1508 if (timestamp
< decoder
->timestamp
)
1509 intel_pt_log("Suppressing CYC timestamp " x64_fmt
" less than current timestamp " x64_fmt
"\n",
1510 timestamp
, decoder
->timestamp
);
1512 decoder
->timestamp
= timestamp
;
1514 decoder
->timestamp_insn_cnt
= 0;
1516 intel_pt_log_to("Setting timestamp", decoder
->timestamp
);
1519 /* Walk PSB+ packets when already in sync. */
1520 static int intel_pt_walk_psbend(struct intel_pt_decoder
*decoder
)
1525 err
= intel_pt_get_next_packet(decoder
);
1529 switch (decoder
->packet
.type
) {
1530 case INTEL_PT_PSBEND
:
1533 case INTEL_PT_TIP_PGD
:
1534 case INTEL_PT_TIP_PGE
:
1537 case INTEL_PT_TRACESTOP
:
1540 case INTEL_PT_PTWRITE
:
1541 case INTEL_PT_PTWRITE_IP
:
1542 case INTEL_PT_EXSTOP
:
1543 case INTEL_PT_EXSTOP_IP
:
1544 case INTEL_PT_MWAIT
:
1547 decoder
->have_tma
= false;
1548 intel_pt_log("ERROR: Unexpected packet\n");
1552 return intel_pt_overflow(decoder
);
1555 intel_pt_calc_tsc_timestamp(decoder
);
1559 intel_pt_calc_tma(decoder
);
1563 intel_pt_calc_cbr(decoder
);
1566 case INTEL_PT_MODE_EXEC
:
1567 decoder
->exec_mode
= decoder
->packet
.payload
;
1571 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
1575 decoder
->pge
= true;
1576 if (decoder
->packet
.count
)
1577 intel_pt_set_last_ip(decoder
);
1580 case INTEL_PT_MODE_TSX
:
1581 intel_pt_update_in_tx(decoder
);
1585 intel_pt_calc_mtc_timestamp(decoder
);
1586 if (decoder
->period_type
== INTEL_PT_PERIOD_MTC
)
1587 decoder
->state
.type
|= INTEL_PT_INSTRUCTION
;
1600 static int intel_pt_walk_fup_tip(struct intel_pt_decoder
*decoder
)
1604 if (decoder
->tx_flags
& INTEL_PT_ABORT_TX
) {
1605 decoder
->tx_flags
= 0;
1606 decoder
->state
.flags
&= ~INTEL_PT_IN_TX
;
1607 decoder
->state
.flags
|= INTEL_PT_ABORT_TX
;
1609 decoder
->state
.flags
|= INTEL_PT_ASYNC
;
1613 err
= intel_pt_get_next_packet(decoder
);
1617 switch (decoder
->packet
.type
) {
1620 case INTEL_PT_TRACESTOP
:
1624 case INTEL_PT_MODE_TSX
:
1626 case INTEL_PT_PSBEND
:
1627 case INTEL_PT_PTWRITE
:
1628 case INTEL_PT_PTWRITE_IP
:
1629 case INTEL_PT_EXSTOP
:
1630 case INTEL_PT_EXSTOP_IP
:
1631 case INTEL_PT_MWAIT
:
1634 intel_pt_log("ERROR: Missing TIP after FUP\n");
1635 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
1636 decoder
->pkt_step
= 0;
1640 intel_pt_calc_cbr(decoder
);
1644 return intel_pt_overflow(decoder
);
1646 case INTEL_PT_TIP_PGD
:
1647 decoder
->state
.from_ip
= decoder
->ip
;
1648 if (decoder
->packet
.count
== 0) {
1649 decoder
->state
.to_ip
= 0;
1651 intel_pt_set_ip(decoder
);
1652 decoder
->state
.to_ip
= decoder
->ip
;
1654 decoder
->pge
= false;
1655 decoder
->continuous_period
= false;
1656 decoder
->state
.type
|= INTEL_PT_TRACE_END
;
1659 case INTEL_PT_TIP_PGE
:
1660 decoder
->pge
= true;
1661 intel_pt_log("Omitting PGE ip " x64_fmt
"\n",
1663 decoder
->state
.from_ip
= 0;
1664 if (decoder
->packet
.count
== 0) {
1665 decoder
->state
.to_ip
= 0;
1667 intel_pt_set_ip(decoder
);
1668 decoder
->state
.to_ip
= decoder
->ip
;
1670 decoder
->state
.type
|= INTEL_PT_TRACE_BEGIN
;
1674 decoder
->state
.from_ip
= decoder
->ip
;
1675 if (decoder
->packet
.count
== 0) {
1676 decoder
->state
.to_ip
= 0;
1678 intel_pt_set_ip(decoder
);
1679 decoder
->state
.to_ip
= decoder
->ip
;
1684 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
1688 intel_pt_calc_mtc_timestamp(decoder
);
1689 if (decoder
->period_type
== INTEL_PT_PERIOD_MTC
)
1690 decoder
->state
.type
|= INTEL_PT_INSTRUCTION
;
1694 intel_pt_calc_cyc_timestamp(decoder
);
1697 case INTEL_PT_MODE_EXEC
:
1698 decoder
->exec_mode
= decoder
->packet
.payload
;
1707 return intel_pt_bug(decoder
);
1712 static int intel_pt_walk_trace(struct intel_pt_decoder
*decoder
)
1714 bool no_tip
= false;
1718 err
= intel_pt_get_next_packet(decoder
);
1722 switch (decoder
->packet
.type
) {
1724 if (!decoder
->packet
.count
)
1726 decoder
->tnt
= decoder
->packet
;
1727 decoder
->pkt_state
= INTEL_PT_STATE_TNT
;
1728 err
= intel_pt_walk_tnt(decoder
);
1733 case INTEL_PT_TIP_PGD
:
1734 if (decoder
->packet
.count
!= 0)
1735 intel_pt_set_last_ip(decoder
);
1736 decoder
->pkt_state
= INTEL_PT_STATE_TIP_PGD
;
1737 return intel_pt_walk_tip(decoder
);
1739 case INTEL_PT_TIP_PGE
: {
1740 decoder
->pge
= true;
1741 if (decoder
->packet
.count
== 0) {
1742 intel_pt_log_at("Skipping zero TIP.PGE",
1746 intel_pt_set_ip(decoder
);
1747 decoder
->state
.from_ip
= 0;
1748 decoder
->state
.to_ip
= decoder
->ip
;
1749 decoder
->state
.type
|= INTEL_PT_TRACE_BEGIN
;
1754 return intel_pt_overflow(decoder
);
1757 if (decoder
->packet
.count
!= 0)
1758 intel_pt_set_last_ip(decoder
);
1759 decoder
->pkt_state
= INTEL_PT_STATE_TIP
;
1760 return intel_pt_walk_tip(decoder
);
1763 if (decoder
->packet
.count
== 0) {
1764 intel_pt_log_at("Skipping zero FUP",
1769 intel_pt_set_last_ip(decoder
);
1770 if (!decoder
->branch_enable
) {
1771 decoder
->ip
= decoder
->last_ip
;
1772 if (intel_pt_fup_event(decoder
))
1777 if (decoder
->set_fup_mwait
)
1779 err
= intel_pt_walk_fup(decoder
);
1780 if (err
!= -EAGAIN
) {
1784 decoder
->pkt_state
=
1785 INTEL_PT_STATE_FUP_NO_TIP
;
1787 decoder
->pkt_state
= INTEL_PT_STATE_FUP
;
1794 return intel_pt_walk_fup_tip(decoder
);
1796 case INTEL_PT_TRACESTOP
:
1797 decoder
->pge
= false;
1798 decoder
->continuous_period
= false;
1799 intel_pt_clear_tx_flags(decoder
);
1800 decoder
->have_tma
= false;
1804 decoder
->last_ip
= 0;
1805 decoder
->have_last_ip
= true;
1806 intel_pt_clear_stack(&decoder
->stack
);
1807 err
= intel_pt_walk_psbend(decoder
);
1815 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
1819 intel_pt_calc_mtc_timestamp(decoder
);
1820 if (decoder
->period_type
!= INTEL_PT_PERIOD_MTC
)
1823 * Ensure that there has been an instruction since the
1826 if (!decoder
->mtc_insn
)
1828 decoder
->mtc_insn
= false;
1829 /* Ensure that there is a timestamp */
1830 if (!decoder
->timestamp
)
1832 decoder
->state
.type
= INTEL_PT_INSTRUCTION
;
1833 decoder
->state
.from_ip
= decoder
->ip
;
1834 decoder
->state
.to_ip
= 0;
1835 decoder
->mtc_insn
= false;
1839 intel_pt_calc_tsc_timestamp(decoder
);
1843 intel_pt_calc_tma(decoder
);
1847 intel_pt_calc_cyc_timestamp(decoder
);
1851 intel_pt_calc_cbr(decoder
);
1852 if (!decoder
->branch_enable
&&
1853 decoder
->cbr
!= decoder
->cbr_seen
) {
1854 decoder
->cbr_seen
= decoder
->cbr
;
1855 decoder
->state
.type
= INTEL_PT_CBR_CHG
;
1856 decoder
->state
.from_ip
= decoder
->ip
;
1857 decoder
->state
.to_ip
= 0;
1858 decoder
->state
.cbr_payload
=
1859 decoder
->packet
.payload
;
1864 case INTEL_PT_MODE_EXEC
:
1865 decoder
->exec_mode
= decoder
->packet
.payload
;
1868 case INTEL_PT_MODE_TSX
:
1869 /* MODE_TSX need not be followed by FUP */
1870 if (!decoder
->pge
) {
1871 intel_pt_update_in_tx(decoder
);
1874 err
= intel_pt_mode_tsx(decoder
, &no_tip
);
1879 case INTEL_PT_BAD
: /* Does not happen */
1880 return intel_pt_bug(decoder
);
1882 case INTEL_PT_PSBEND
:
1888 case INTEL_PT_PTWRITE_IP
:
1889 decoder
->fup_ptw_payload
= decoder
->packet
.payload
;
1890 err
= intel_pt_get_next_packet(decoder
);
1893 if (decoder
->packet
.type
== INTEL_PT_FUP
) {
1894 decoder
->set_fup_ptw
= true;
1897 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1902 case INTEL_PT_PTWRITE
:
1903 decoder
->state
.type
= INTEL_PT_PTW
;
1904 decoder
->state
.from_ip
= decoder
->ip
;
1905 decoder
->state
.to_ip
= 0;
1906 decoder
->state
.ptw_payload
= decoder
->packet
.payload
;
1909 case INTEL_PT_MWAIT
:
1910 decoder
->fup_mwait_payload
= decoder
->packet
.payload
;
1911 decoder
->set_fup_mwait
= true;
1915 if (decoder
->set_fup_mwait
) {
1916 decoder
->fup_pwre_payload
=
1917 decoder
->packet
.payload
;
1918 decoder
->set_fup_pwre
= true;
1921 decoder
->state
.type
= INTEL_PT_PWR_ENTRY
;
1922 decoder
->state
.from_ip
= decoder
->ip
;
1923 decoder
->state
.to_ip
= 0;
1924 decoder
->state
.pwrx_payload
= decoder
->packet
.payload
;
1927 case INTEL_PT_EXSTOP_IP
:
1928 err
= intel_pt_get_next_packet(decoder
);
1931 if (decoder
->packet
.type
== INTEL_PT_FUP
) {
1932 decoder
->set_fup_exstop
= true;
1935 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1940 case INTEL_PT_EXSTOP
:
1941 decoder
->state
.type
= INTEL_PT_EX_STOP
;
1942 decoder
->state
.from_ip
= decoder
->ip
;
1943 decoder
->state
.to_ip
= 0;
1947 decoder
->state
.type
= INTEL_PT_PWR_EXIT
;
1948 decoder
->state
.from_ip
= decoder
->ip
;
1949 decoder
->state
.to_ip
= 0;
1950 decoder
->state
.pwrx_payload
= decoder
->packet
.payload
;
1954 return intel_pt_bug(decoder
);
1959 static inline bool intel_pt_have_ip(struct intel_pt_decoder
*decoder
)
1961 return decoder
->packet
.count
&&
1962 (decoder
->have_last_ip
|| decoder
->packet
.count
== 3 ||
1963 decoder
->packet
.count
== 6);
1966 /* Walk PSB+ packets to get in sync. */
1967 static int intel_pt_walk_psb(struct intel_pt_decoder
*decoder
)
1972 err
= intel_pt_get_next_packet(decoder
);
1976 switch (decoder
->packet
.type
) {
1977 case INTEL_PT_TIP_PGD
:
1978 decoder
->continuous_period
= false;
1980 case INTEL_PT_TIP_PGE
:
1982 case INTEL_PT_PTWRITE
:
1983 case INTEL_PT_PTWRITE_IP
:
1984 case INTEL_PT_EXSTOP
:
1985 case INTEL_PT_EXSTOP_IP
:
1986 case INTEL_PT_MWAIT
:
1989 intel_pt_log("ERROR: Unexpected packet\n");
1993 decoder
->pge
= true;
1994 if (intel_pt_have_ip(decoder
)) {
1995 uint64_t current_ip
= decoder
->ip
;
1997 intel_pt_set_ip(decoder
);
1999 intel_pt_log_to("Setting IP",
2005 intel_pt_calc_mtc_timestamp(decoder
);
2009 intel_pt_calc_tsc_timestamp(decoder
);
2013 intel_pt_calc_tma(decoder
);
2017 intel_pt_calc_cyc_timestamp(decoder
);
2021 intel_pt_calc_cbr(decoder
);
2025 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
2028 case INTEL_PT_MODE_EXEC
:
2029 decoder
->exec_mode
= decoder
->packet
.payload
;
2032 case INTEL_PT_MODE_TSX
:
2033 intel_pt_update_in_tx(decoder
);
2036 case INTEL_PT_TRACESTOP
:
2037 decoder
->pge
= false;
2038 decoder
->continuous_period
= false;
2039 intel_pt_clear_tx_flags(decoder
);
2043 decoder
->have_tma
= false;
2044 intel_pt_log("ERROR: Unexpected packet\n");
2046 decoder
->pkt_state
= INTEL_PT_STATE_ERR4
;
2048 decoder
->pkt_state
= INTEL_PT_STATE_ERR3
;
2051 case INTEL_PT_BAD
: /* Does not happen */
2052 return intel_pt_bug(decoder
);
2055 return intel_pt_overflow(decoder
);
2057 case INTEL_PT_PSBEND
:
2070 static int intel_pt_walk_to_ip(struct intel_pt_decoder
*decoder
)
2075 err
= intel_pt_get_next_packet(decoder
);
2079 switch (decoder
->packet
.type
) {
2080 case INTEL_PT_TIP_PGD
:
2081 decoder
->continuous_period
= false;
2083 case INTEL_PT_TIP_PGE
:
2085 decoder
->pge
= decoder
->packet
.type
!= INTEL_PT_TIP_PGD
;
2086 if (intel_pt_have_ip(decoder
))
2087 intel_pt_set_ip(decoder
);
2090 if (decoder
->packet
.type
== INTEL_PT_TIP_PGE
)
2091 decoder
->state
.type
|= INTEL_PT_TRACE_BEGIN
;
2092 if (decoder
->packet
.type
== INTEL_PT_TIP_PGD
)
2093 decoder
->state
.type
|= INTEL_PT_TRACE_END
;
2097 if (intel_pt_have_ip(decoder
))
2098 intel_pt_set_ip(decoder
);
2104 intel_pt_calc_mtc_timestamp(decoder
);
2108 intel_pt_calc_tsc_timestamp(decoder
);
2112 intel_pt_calc_tma(decoder
);
2116 intel_pt_calc_cyc_timestamp(decoder
);
2120 intel_pt_calc_cbr(decoder
);
2124 decoder
->cr3
= decoder
->packet
.payload
& (BIT63
- 1);
2127 case INTEL_PT_MODE_EXEC
:
2128 decoder
->exec_mode
= decoder
->packet
.payload
;
2131 case INTEL_PT_MODE_TSX
:
2132 intel_pt_update_in_tx(decoder
);
2136 return intel_pt_overflow(decoder
);
2138 case INTEL_PT_BAD
: /* Does not happen */
2139 return intel_pt_bug(decoder
);
2141 case INTEL_PT_TRACESTOP
:
2142 decoder
->pge
= false;
2143 decoder
->continuous_period
= false;
2144 intel_pt_clear_tx_flags(decoder
);
2145 decoder
->have_tma
= false;
2149 decoder
->last_ip
= 0;
2150 decoder
->have_last_ip
= true;
2151 intel_pt_clear_stack(&decoder
->stack
);
2152 err
= intel_pt_walk_psb(decoder
);
2156 /* Do not have a sample */
2157 decoder
->state
.type
= 0;
2163 case INTEL_PT_PSBEND
:
2167 case INTEL_PT_PTWRITE
:
2168 case INTEL_PT_PTWRITE_IP
:
2169 case INTEL_PT_EXSTOP
:
2170 case INTEL_PT_EXSTOP_IP
:
2171 case INTEL_PT_MWAIT
:
2180 static int intel_pt_sync_ip(struct intel_pt_decoder
*decoder
)
2184 decoder
->set_fup_tx_flags
= false;
2185 decoder
->set_fup_ptw
= false;
2186 decoder
->set_fup_mwait
= false;
2187 decoder
->set_fup_pwre
= false;
2188 decoder
->set_fup_exstop
= false;
2190 if (!decoder
->branch_enable
) {
2191 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
2192 decoder
->overflow
= false;
2193 decoder
->state
.type
= 0; /* Do not have a sample */
2197 intel_pt_log("Scanning for full IP\n");
2198 err
= intel_pt_walk_to_ip(decoder
);
2202 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
2203 decoder
->overflow
= false;
2205 decoder
->state
.from_ip
= 0;
2206 decoder
->state
.to_ip
= decoder
->ip
;
2207 intel_pt_log_to("Setting IP", decoder
->ip
);
2212 static int intel_pt_part_psb(struct intel_pt_decoder
*decoder
)
2214 const unsigned char *end
= decoder
->buf
+ decoder
->len
;
2217 for (i
= INTEL_PT_PSB_LEN
- 1; i
; i
--) {
2218 if (i
> decoder
->len
)
2220 if (!memcmp(end
- i
, INTEL_PT_PSB_STR
, i
))
2226 static int intel_pt_rest_psb(struct intel_pt_decoder
*decoder
, int part_psb
)
2228 size_t rest_psb
= INTEL_PT_PSB_LEN
- part_psb
;
2229 const char *psb
= INTEL_PT_PSB_STR
;
2231 if (rest_psb
> decoder
->len
||
2232 memcmp(decoder
->buf
, psb
+ part_psb
, rest_psb
))
2238 static int intel_pt_get_split_psb(struct intel_pt_decoder
*decoder
,
2243 decoder
->pos
+= decoder
->len
;
2246 ret
= intel_pt_get_next_data(decoder
);
2250 rest_psb
= intel_pt_rest_psb(decoder
, part_psb
);
2254 decoder
->pos
-= part_psb
;
2255 decoder
->next_buf
= decoder
->buf
+ rest_psb
;
2256 decoder
->next_len
= decoder
->len
- rest_psb
;
2257 memcpy(decoder
->temp_buf
, INTEL_PT_PSB_STR
, INTEL_PT_PSB_LEN
);
2258 decoder
->buf
= decoder
->temp_buf
;
2259 decoder
->len
= INTEL_PT_PSB_LEN
;
2264 static int intel_pt_scan_for_psb(struct intel_pt_decoder
*decoder
)
2266 unsigned char *next
;
2269 intel_pt_log("Scanning for PSB\n");
2271 if (!decoder
->len
) {
2272 ret
= intel_pt_get_next_data(decoder
);
2277 next
= memmem(decoder
->buf
, decoder
->len
, INTEL_PT_PSB_STR
,
2282 part_psb
= intel_pt_part_psb(decoder
);
2284 ret
= intel_pt_get_split_psb(decoder
, part_psb
);
2288 decoder
->pos
+= decoder
->len
;
2294 decoder
->pkt_step
= next
- decoder
->buf
;
2295 return intel_pt_get_next_packet(decoder
);
2299 static int intel_pt_sync(struct intel_pt_decoder
*decoder
)
2303 decoder
->pge
= false;
2304 decoder
->continuous_period
= false;
2305 decoder
->have_last_ip
= false;
2306 decoder
->last_ip
= 0;
2308 intel_pt_clear_stack(&decoder
->stack
);
2310 err
= intel_pt_scan_for_psb(decoder
);
2314 decoder
->have_last_ip
= true;
2315 decoder
->pkt_state
= INTEL_PT_STATE_NO_IP
;
2317 err
= intel_pt_walk_psb(decoder
);
2322 decoder
->state
.type
= 0; /* Do not have a sample */
2323 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
2325 return intel_pt_sync_ip(decoder
);
2331 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder
*decoder
)
2333 uint64_t est
= decoder
->sample_insn_cnt
<< 1;
2335 if (!decoder
->cbr
|| !decoder
->max_non_turbo_ratio
)
2338 est
*= decoder
->max_non_turbo_ratio
;
2339 est
/= decoder
->cbr
;
2341 return decoder
->sample_timestamp
+ est
;
2344 const struct intel_pt_state
*intel_pt_decode(struct intel_pt_decoder
*decoder
)
2349 decoder
->state
.type
= INTEL_PT_BRANCH
;
2350 decoder
->state
.flags
= 0;
2352 switch (decoder
->pkt_state
) {
2353 case INTEL_PT_STATE_NO_PSB
:
2354 err
= intel_pt_sync(decoder
);
2356 case INTEL_PT_STATE_NO_IP
:
2357 decoder
->have_last_ip
= false;
2358 decoder
->last_ip
= 0;
2361 case INTEL_PT_STATE_ERR_RESYNC
:
2362 err
= intel_pt_sync_ip(decoder
);
2364 case INTEL_PT_STATE_IN_SYNC
:
2365 err
= intel_pt_walk_trace(decoder
);
2367 case INTEL_PT_STATE_TNT
:
2368 err
= intel_pt_walk_tnt(decoder
);
2370 err
= intel_pt_walk_trace(decoder
);
2372 case INTEL_PT_STATE_TIP
:
2373 case INTEL_PT_STATE_TIP_PGD
:
2374 err
= intel_pt_walk_tip(decoder
);
2376 case INTEL_PT_STATE_FUP
:
2377 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
2378 err
= intel_pt_walk_fup(decoder
);
2380 err
= intel_pt_walk_fup_tip(decoder
);
2382 decoder
->pkt_state
= INTEL_PT_STATE_FUP
;
2384 case INTEL_PT_STATE_FUP_NO_TIP
:
2385 decoder
->pkt_state
= INTEL_PT_STATE_IN_SYNC
;
2386 err
= intel_pt_walk_fup(decoder
);
2388 err
= intel_pt_walk_trace(decoder
);
2391 err
= intel_pt_bug(decoder
);
2394 } while (err
== -ENOLINK
);
2397 decoder
->state
.err
= intel_pt_ext_err(err
);
2398 decoder
->state
.from_ip
= decoder
->ip
;
2399 decoder
->sample_timestamp
= decoder
->timestamp
;
2400 decoder
->sample_insn_cnt
= decoder
->timestamp_insn_cnt
;
2402 decoder
->state
.err
= 0;
2403 if (decoder
->cbr
!= decoder
->cbr_seen
&& decoder
->state
.type
) {
2404 decoder
->cbr_seen
= decoder
->cbr
;
2405 decoder
->state
.type
|= INTEL_PT_CBR_CHG
;
2406 decoder
->state
.cbr_payload
= decoder
->cbr_payload
;
2408 if (intel_pt_sample_time(decoder
->pkt_state
)) {
2409 decoder
->sample_timestamp
= decoder
->timestamp
;
2410 decoder
->sample_insn_cnt
= decoder
->timestamp_insn_cnt
;
2414 decoder
->state
.timestamp
= decoder
->sample_timestamp
;
2415 decoder
->state
.est_timestamp
= intel_pt_est_timestamp(decoder
);
2416 decoder
->state
.cr3
= decoder
->cr3
;
2417 decoder
->state
.tot_insn_cnt
= decoder
->tot_insn_cnt
;
2419 return &decoder
->state
;
2423 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2424 * @buf: pointer to buffer pointer
2425 * @len: size of buffer
2427 * Updates the buffer pointer to point to the start of the next PSB packet if
2428 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2429 * @len is adjusted accordingly.
2431 * Return: %true if a PSB packet is found, %false otherwise.
2433 static bool intel_pt_next_psb(unsigned char **buf
, size_t *len
)
2435 unsigned char *next
;
2437 next
= memmem(*buf
, *len
, INTEL_PT_PSB_STR
, INTEL_PT_PSB_LEN
);
2439 *len
-= next
- *buf
;
2447 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2449 * @buf: pointer to buffer pointer
2450 * @len: size of buffer
2452 * Updates the buffer pointer to point to the start of the following PSB packet
2453 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2454 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2456 * Return: %true if a PSB packet is found, %false otherwise.
2458 static bool intel_pt_step_psb(unsigned char **buf
, size_t *len
)
2460 unsigned char *next
;
2465 next
= memmem(*buf
+ 1, *len
- 1, INTEL_PT_PSB_STR
, INTEL_PT_PSB_LEN
);
2467 *len
-= next
- *buf
;
2475 * intel_pt_last_psb - find the last PSB packet in a buffer.
2477 * @len: size of buffer
2479 * This function finds the last PSB in a buffer.
2481 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2483 static unsigned char *intel_pt_last_psb(unsigned char *buf
, size_t len
)
2485 const char *n
= INTEL_PT_PSB_STR
;
2489 if (len
< INTEL_PT_PSB_LEN
)
2492 k
= len
- INTEL_PT_PSB_LEN
+ 1;
2494 p
= memrchr(buf
, n
[0], k
);
2497 if (!memcmp(p
+ 1, n
+ 1, INTEL_PT_PSB_LEN
- 1))
2506 * intel_pt_next_tsc - find and return next TSC.
2508 * @len: size of buffer
2509 * @tsc: TSC value returned
2510 * @rem: returns remaining size when TSC is found
2512 * Find a TSC packet in @buf and return the TSC value. This function assumes
2513 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2514 * PSBEND packet is found.
2516 * Return: %true if TSC is found, false otherwise.
2518 static bool intel_pt_next_tsc(unsigned char *buf
, size_t len
, uint64_t *tsc
,
2521 struct intel_pt_pkt packet
;
2525 ret
= intel_pt_get_packet(buf
, len
, &packet
);
2528 if (packet
.type
== INTEL_PT_TSC
) {
2529 *tsc
= packet
.payload
;
2533 if (packet
.type
== INTEL_PT_PSBEND
)
2542 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2543 * @tsc1: first TSC to compare
2544 * @tsc2: second TSC to compare
2546 * This function compares 7-byte TSC values allowing for the possibility that
2547 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2548 * around so for that purpose this function assumes the absolute difference is
2549 * less than half the maximum difference.
2551 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2554 static int intel_pt_tsc_cmp(uint64_t tsc1
, uint64_t tsc2
)
2556 const uint64_t halfway
= (1ULL << 55);
2562 if (tsc2
- tsc1
< halfway
)
2567 if (tsc1
- tsc2
< halfway
)
2574 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2577 * adj_for_padding - adjust overlap to account for padding.
2578 * @buf_b: second buffer
2579 * @buf_a: first buffer
2580 * @len_a: size of first buffer
2582 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2585 * Return: A pointer into @buf_b from where non-overlapped data starts
2587 static unsigned char *adj_for_padding(unsigned char *buf_b
,
2588 unsigned char *buf_a
, size_t len_a
)
2590 unsigned char *p
= buf_b
- MAX_PADDING
;
2591 unsigned char *q
= buf_a
+ len_a
- MAX_PADDING
;
2594 for (i
= MAX_PADDING
; i
; i
--, p
++, q
++) {
2603 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2605 * @buf_a: first buffer
2606 * @len_a: size of first buffer
2607 * @buf_b: second buffer
2608 * @len_b: size of second buffer
2609 * @consecutive: returns true if there is data in buf_b that is consecutive
2612 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2613 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2614 * walk forward in @buf_b until a later TSC is found. A precondition is that
2615 * @buf_a and @buf_b are positioned at a PSB.
2617 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2618 * @buf_b + @len_b if there is no non-overlapped data.
2620 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a
,
2622 unsigned char *buf_b
,
2623 size_t len_b
, bool *consecutive
)
2625 uint64_t tsc_a
, tsc_b
;
2627 size_t len
, rem_a
, rem_b
;
2629 p
= intel_pt_last_psb(buf_a
, len_a
);
2631 return buf_b
; /* No PSB in buf_a => no overlap */
2633 len
= len_a
- (p
- buf_a
);
2634 if (!intel_pt_next_tsc(p
, len
, &tsc_a
, &rem_a
)) {
2635 /* The last PSB+ in buf_a is incomplete, so go back one more */
2637 p
= intel_pt_last_psb(buf_a
, len_a
);
2639 return buf_b
; /* No full PSB+ => assume no overlap */
2640 len
= len_a
- (p
- buf_a
);
2641 if (!intel_pt_next_tsc(p
, len
, &tsc_a
, &rem_a
))
2642 return buf_b
; /* No TSC in buf_a => assume no overlap */
2646 /* Ignore PSB+ with no TSC */
2647 if (intel_pt_next_tsc(buf_b
, len_b
, &tsc_b
, &rem_b
)) {
2648 int cmp
= intel_pt_tsc_cmp(tsc_a
, tsc_b
);
2650 /* Same TSC, so buffers are consecutive */
2651 if (!cmp
&& rem_b
>= rem_a
) {
2652 unsigned char *start
;
2654 *consecutive
= true;
2655 start
= buf_b
+ len_b
- (rem_b
- rem_a
);
2656 return adj_for_padding(start
, buf_a
, len_a
);
2659 return buf_b
; /* tsc_a < tsc_b => no overlap */
2662 if (!intel_pt_step_psb(&buf_b
, &len_b
))
2663 return buf_b
+ len_b
; /* No PSB in buf_b => no data */
2668 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2669 * @buf_a: first buffer
2670 * @len_a: size of first buffer
2671 * @buf_b: second buffer
2672 * @len_b: size of second buffer
2673 * @have_tsc: can use TSC packets to detect overlap
2674 * @consecutive: returns true if there is data in buf_b that is consecutive
2677 * When trace samples or snapshots are recorded there is the possibility that
2678 * the data overlaps. Note that, for the purposes of decoding, data is only
2679 * useful if it begins with a PSB packet.
2681 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2682 * @buf_b + @len_b if there is no non-overlapped data.
2684 unsigned char *intel_pt_find_overlap(unsigned char *buf_a
, size_t len_a
,
2685 unsigned char *buf_b
, size_t len_b
,
2686 bool have_tsc
, bool *consecutive
)
2688 unsigned char *found
;
2690 /* Buffer 'b' must start at PSB so throw away everything before that */
2691 if (!intel_pt_next_psb(&buf_b
, &len_b
))
2692 return buf_b
+ len_b
; /* No PSB */
2694 if (!intel_pt_next_psb(&buf_a
, &len_a
))
2695 return buf_b
; /* No overlap */
2698 found
= intel_pt_find_overlap_tsc(buf_a
, len_a
, buf_b
, len_b
,
2705 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2706 * we can ignore the first part of buffer 'a'.
2708 while (len_b
< len_a
) {
2709 if (!intel_pt_step_psb(&buf_a
, &len_a
))
2710 return buf_b
; /* No overlap */
2713 /* Now len_b >= len_a */
2715 /* Potential overlap so check the bytes */
2716 found
= memmem(buf_a
, len_a
, buf_b
, len_a
);
2718 *consecutive
= true;
2719 return adj_for_padding(buf_b
+ len_a
, buf_a
, len_a
);
2722 /* Try again at next PSB in buffer 'a' */
2723 if (!intel_pt_step_psb(&buf_a
, &len_a
))
2724 return buf_b
; /* No overlap */