]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - tools/perf/util/intel-pt-decoder/intel-pt-decoder.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:


f9cd3aaef488f0c1db0733ca37d0f57134c5fdaa
[mirror_ubuntu-bionic-kernel.git] / tools / perf / util / intel-pt-decoder / intel-pt-decoder.c
1 /*
2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15
16 #ifndef _GNU_SOURCE
17 #define _GNU_SOURCE
18 #endif
19 #include <stdlib.h>
20 #include <stdbool.h>
21 #include <string.h>
22 #include <errno.h>
23 #include <stdint.h>
24 #include <inttypes.h>
25 #include <linux/compiler.h>
26
27 #include "../cache.h"
28 #include "../util.h"
29
30 #include "intel-pt-insn-decoder.h"
31 #include "intel-pt-pkt-decoder.h"
32 #include "intel-pt-decoder.h"
33 #include "intel-pt-log.h"
34
35 #define INTEL_PT_BLK_SIZE 1024
36
37 #define BIT63 (((uint64_t)1 << 63))
38
39 #define INTEL_PT_RETURN 1
40
41 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
42 #define INTEL_PT_MAX_LOOPS 10000
43
44 struct intel_pt_blk {
45 struct intel_pt_blk *prev;
46 uint64_t ip[INTEL_PT_BLK_SIZE];
47 };
48
49 struct intel_pt_stack {
50 struct intel_pt_blk *blk;
51 struct intel_pt_blk *spare;
52 int pos;
53 };
54
55 enum intel_pt_pkt_state {
56 INTEL_PT_STATE_NO_PSB,
57 INTEL_PT_STATE_NO_IP,
58 INTEL_PT_STATE_ERR_RESYNC,
59 INTEL_PT_STATE_IN_SYNC,
60 INTEL_PT_STATE_TNT,
61 INTEL_PT_STATE_TIP,
62 INTEL_PT_STATE_TIP_PGD,
63 INTEL_PT_STATE_FUP,
64 INTEL_PT_STATE_FUP_NO_TIP,
65 };
66
67 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
68 {
69 switch (pkt_state) {
70 case INTEL_PT_STATE_NO_PSB:
71 case INTEL_PT_STATE_NO_IP:
72 case INTEL_PT_STATE_ERR_RESYNC:
73 case INTEL_PT_STATE_IN_SYNC:
74 case INTEL_PT_STATE_TNT:
75 return true;
76 case INTEL_PT_STATE_TIP:
77 case INTEL_PT_STATE_TIP_PGD:
78 case INTEL_PT_STATE_FUP:
79 case INTEL_PT_STATE_FUP_NO_TIP:
80 return false;
81 default:
82 return true;
83 };
84 }
85
86 #ifdef INTEL_PT_STRICT
87 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
88 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
89 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
90 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
91 #else
92 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
93 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
94 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
95 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
96 #endif
97
98 struct intel_pt_decoder {
99 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
100 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
101 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
102 uint64_t max_insn_cnt, void *data);
103 bool (*pgd_ip)(uint64_t ip, void *data);
104 void *data;
105 struct intel_pt_state state;
106 const unsigned char *buf;
107 size_t len;
108 bool return_compression;
109 bool mtc_insn;
110 bool pge;
111 bool have_tma;
112 bool have_cyc;
113 bool fixup_last_mtc;
114 uint64_t pos;
115 uint64_t last_ip;
116 uint64_t ip;
117 uint64_t cr3;
118 uint64_t timestamp;
119 uint64_t tsc_timestamp;
120 uint64_t ref_timestamp;
121 uint64_t sample_timestamp;
122 uint64_t ret_addr;
123 uint64_t ctc_timestamp;
124 uint64_t ctc_delta;
125 uint64_t cycle_cnt;
126 uint64_t cyc_ref_timestamp;
127 uint32_t last_mtc;
128 uint32_t tsc_ctc_ratio_n;
129 uint32_t tsc_ctc_ratio_d;
130 uint32_t tsc_ctc_mult;
131 uint32_t tsc_slip;
132 uint32_t ctc_rem_mask;
133 int mtc_shift;
134 struct intel_pt_stack stack;
135 enum intel_pt_pkt_state pkt_state;
136 struct intel_pt_pkt packet;
137 struct intel_pt_pkt tnt;
138 int pkt_step;
139 int pkt_len;
140 int last_packet_type;
141 unsigned int cbr;
142 unsigned int max_non_turbo_ratio;
143 double max_non_turbo_ratio_fp;
144 double cbr_cyc_to_tsc;
145 double calc_cyc_to_tsc;
146 bool have_calc_cyc_to_tsc;
147 int exec_mode;
148 unsigned int insn_bytes;
149 uint64_t period;
150 enum intel_pt_period_type period_type;
151 uint64_t tot_insn_cnt;
152 uint64_t period_insn_cnt;
153 uint64_t period_mask;
154 uint64_t period_ticks;
155 uint64_t last_masked_timestamp;
156 bool continuous_period;
157 bool overflow;
158 bool set_fup_tx_flags;
159 unsigned int fup_tx_flags;
160 unsigned int tx_flags;
161 uint64_t timestamp_insn_cnt;
162 uint64_t sample_insn_cnt;
163 uint64_t stuck_ip;
164 int no_progress;
165 int stuck_ip_prd;
166 int stuck_ip_cnt;
167 const unsigned char *next_buf;
168 size_t next_len;
169 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
170 };
171
172 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
173 {
174 int i;
175
176 for (i = 0; x != 1; i++)
177 x >>= 1;
178
179 return x << i;
180 }
181
182 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
183 {
184 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
185 uint64_t period;
186
187 period = intel_pt_lower_power_of_2(decoder->period);
188 decoder->period_mask = ~(period - 1);
189 decoder->period_ticks = period;
190 }
191 }
192
193 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
194 {
195 if (!d)
196 return 0;
197 return (t / d) * n + ((t % d) * n) / d;
198 }
199
200 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
201 {
202 struct intel_pt_decoder *decoder;
203
204 if (!params->get_trace || !params->walk_insn)
205 return NULL;
206
207 decoder = zalloc(sizeof(struct intel_pt_decoder));
208 if (!decoder)
209 return NULL;
210
211 decoder->get_trace = params->get_trace;
212 decoder->walk_insn = params->walk_insn;
213 decoder->pgd_ip = params->pgd_ip;
214 decoder->data = params->data;
215 decoder->return_compression = params->return_compression;
216
217 decoder->period = params->period;
218 decoder->period_type = params->period_type;
219
220 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
221 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
222
223 intel_pt_setup_period(decoder);
224
225 decoder->mtc_shift = params->mtc_period;
226 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
227
228 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
229 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
230
231 if (!decoder->tsc_ctc_ratio_n)
232 decoder->tsc_ctc_ratio_d = 0;
233
234 if (decoder->tsc_ctc_ratio_d) {
235 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
236 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
237 decoder->tsc_ctc_ratio_d;
238
239 /*
240 * Allow for timestamps appearing to backwards because a TSC
241 * packet has slipped past a MTC packet, so allow 2 MTC ticks
242 * or ...
243 */
244 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
245 decoder->tsc_ctc_ratio_n,
246 decoder->tsc_ctc_ratio_d);
247 }
248 /* ... or 0x100 paranoia */
249 if (decoder->tsc_slip < 0x100)
250 decoder->tsc_slip = 0x100;
251
252 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
253 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
254 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
255 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
256 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
257
258 return decoder;
259 }
260
261 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
262 {
263 struct intel_pt_blk *blk = stack->blk;
264
265 stack->blk = blk->prev;
266 if (!stack->spare)
267 stack->spare = blk;
268 else
269 free(blk);
270 }
271
272 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
273 {
274 if (!stack->pos) {
275 if (!stack->blk)
276 return 0;
277 intel_pt_pop_blk(stack);
278 if (!stack->blk)
279 return 0;
280 stack->pos = INTEL_PT_BLK_SIZE;
281 }
282 return stack->blk->ip[--stack->pos];
283 }
284
285 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
286 {
287 struct intel_pt_blk *blk;
288
289 if (stack->spare) {
290 blk = stack->spare;
291 stack->spare = NULL;
292 } else {
293 blk = malloc(sizeof(struct intel_pt_blk));
294 if (!blk)
295 return -ENOMEM;
296 }
297
298 blk->prev = stack->blk;
299 stack->blk = blk;
300 stack->pos = 0;
301 return 0;
302 }
303
304 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
305 {
306 int err;
307
308 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
309 err = intel_pt_alloc_blk(stack);
310 if (err)
311 return err;
312 }
313
314 stack->blk->ip[stack->pos++] = ip;
315 return 0;
316 }
317
318 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
319 {
320 while (stack->blk)
321 intel_pt_pop_blk(stack);
322 stack->pos = 0;
323 }
324
325 static void intel_pt_free_stack(struct intel_pt_stack *stack)
326 {
327 intel_pt_clear_stack(stack);
328 zfree(&stack->blk);
329 zfree(&stack->spare);
330 }
331
332 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
333 {
334 intel_pt_free_stack(&decoder->stack);
335 free(decoder);
336 }
337
338 static int intel_pt_ext_err(int code)
339 {
340 switch (code) {
341 case -ENOMEM:
342 return INTEL_PT_ERR_NOMEM;
343 case -ENOSYS:
344 return INTEL_PT_ERR_INTERN;
345 case -EBADMSG:
346 return INTEL_PT_ERR_BADPKT;
347 case -ENODATA:
348 return INTEL_PT_ERR_NODATA;
349 case -EILSEQ:
350 return INTEL_PT_ERR_NOINSN;
351 case -ENOENT:
352 return INTEL_PT_ERR_MISMAT;
353 case -EOVERFLOW:
354 return INTEL_PT_ERR_OVR;
355 case -ENOSPC:
356 return INTEL_PT_ERR_LOST;
357 case -ELOOP:
358 return INTEL_PT_ERR_NELOOP;
359 default:
360 return INTEL_PT_ERR_UNK;
361 }
362 }
363
364 static const char *intel_pt_err_msgs[] = {
365 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
366 [INTEL_PT_ERR_INTERN] = "Internal error",
367 [INTEL_PT_ERR_BADPKT] = "Bad packet",
368 [INTEL_PT_ERR_NODATA] = "No more data",
369 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
370 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
371 [INTEL_PT_ERR_OVR] = "Overflow packet",
372 [INTEL_PT_ERR_LOST] = "Lost trace data",
373 [INTEL_PT_ERR_UNK] = "Unknown error!",
374 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
375 };
376
377 int intel_pt__strerror(int code, char *buf, size_t buflen)
378 {
379 if (code < 1 || code >= INTEL_PT_ERR_MAX)
380 code = INTEL_PT_ERR_UNK;
381 strlcpy(buf, intel_pt_err_msgs[code], buflen);
382 return 0;
383 }
384
385 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
386 uint64_t last_ip)
387 {
388 uint64_t ip;
389
390 switch (packet->count) {
391 case 1:
392 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
393 packet->payload;
394 break;
395 case 2:
396 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
397 packet->payload;
398 break;
399 case 3:
400 ip = packet->payload;
401 /* Sign-extend 6-byte ip */
402 if (ip & (uint64_t)0x800000000000ULL)
403 ip |= (uint64_t)0xffff000000000000ULL;
404 break;
405 case 4:
406 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
407 packet->payload;
408 break;
409 case 6:
410 ip = packet->payload;
411 break;
412 default:
413 return 0;
414 }
415
416 return ip;
417 }
418
419 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
420 {
421 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
422 }
423
424 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
425 {
426 intel_pt_set_last_ip(decoder);
427 decoder->ip = decoder->last_ip;
428 }
429
430 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
431 {
432 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
433 decoder->buf);
434 }
435
436 static int intel_pt_bug(struct intel_pt_decoder *decoder)
437 {
438 intel_pt_log("ERROR: Internal error\n");
439 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
440 return -ENOSYS;
441 }
442
443 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
444 {
445 decoder->tx_flags = 0;
446 }
447
448 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
449 {
450 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
451 }
452
453 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
454 {
455 intel_pt_clear_tx_flags(decoder);
456 decoder->have_tma = false;
457 decoder->pkt_len = 1;
458 decoder->pkt_step = 1;
459 intel_pt_decoder_log_packet(decoder);
460 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
461 intel_pt_log("ERROR: Bad packet\n");
462 decoder->pkt_state = INTEL_PT_STATE_ERR1;
463 }
464 return -EBADMSG;
465 }
466
467 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
468 {
469 struct intel_pt_buffer buffer = { .buf = 0, };
470 int ret;
471
472 decoder->pkt_step = 0;
473
474 intel_pt_log("Getting more data\n");
475 ret = decoder->get_trace(&buffer, decoder->data);
476 if (ret)
477 return ret;
478 decoder->buf = buffer.buf;
479 decoder->len = buffer.len;
480 if (!decoder->len) {
481 intel_pt_log("No more data\n");
482 return -ENODATA;
483 }
484 if (!buffer.consecutive) {
485 decoder->ip = 0;
486 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
487 decoder->ref_timestamp = buffer.ref_timestamp;
488 decoder->timestamp = 0;
489 decoder->have_tma = false;
490 decoder->state.trace_nr = buffer.trace_nr;
491 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
492 decoder->ref_timestamp);
493 return -ENOLINK;
494 }
495
496 return 0;
497 }
498
499 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
500 {
501 if (!decoder->next_buf)
502 return intel_pt_get_data(decoder);
503
504 decoder->buf = decoder->next_buf;
505 decoder->len = decoder->next_len;
506 decoder->next_buf = 0;
507 decoder->next_len = 0;
508 return 0;
509 }
510
511 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
512 {
513 unsigned char *buf = decoder->temp_buf;
514 size_t old_len, len, n;
515 int ret;
516
517 old_len = decoder->len;
518 len = decoder->len;
519 memcpy(buf, decoder->buf, len);
520
521 ret = intel_pt_get_data(decoder);
522 if (ret) {
523 decoder->pos += old_len;
524 return ret < 0 ? ret : -EINVAL;
525 }
526
527 n = INTEL_PT_PKT_MAX_SZ - len;
528 if (n > decoder->len)
529 n = decoder->len;
530 memcpy(buf + len, decoder->buf, n);
531 len += n;
532
533 ret = intel_pt_get_packet(buf, len, &decoder->packet);
534 if (ret < (int)old_len) {
535 decoder->next_buf = decoder->buf;
536 decoder->next_len = decoder->len;
537 decoder->buf = buf;
538 decoder->len = old_len;
539 return intel_pt_bad_packet(decoder);
540 }
541
542 decoder->next_buf = decoder->buf + (ret - old_len);
543 decoder->next_len = decoder->len - (ret - old_len);
544
545 decoder->buf = buf;
546 decoder->len = ret;
547
548 return ret;
549 }
550
551 struct intel_pt_pkt_info {
552 struct intel_pt_decoder *decoder;
553 struct intel_pt_pkt packet;
554 uint64_t pos;
555 int pkt_len;
556 int last_packet_type;
557 void *data;
558 };
559
560 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
561
562 /* Lookahead packets in current buffer */
563 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
564 intel_pt_pkt_cb_t cb, void *data)
565 {
566 struct intel_pt_pkt_info pkt_info;
567 const unsigned char *buf = decoder->buf;
568 size_t len = decoder->len;
569 int ret;
570
571 pkt_info.decoder = decoder;
572 pkt_info.pos = decoder->pos;
573 pkt_info.pkt_len = decoder->pkt_step;
574 pkt_info.last_packet_type = decoder->last_packet_type;
575 pkt_info.data = data;
576
577 while (1) {
578 do {
579 pkt_info.pos += pkt_info.pkt_len;
580 buf += pkt_info.pkt_len;
581 len -= pkt_info.pkt_len;
582
583 if (!len)
584 return INTEL_PT_NEED_MORE_BYTES;
585
586 ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
587 if (!ret)
588 return INTEL_PT_NEED_MORE_BYTES;
589 if (ret < 0)
590 return ret;
591
592 pkt_info.pkt_len = ret;
593 } while (pkt_info.packet.type == INTEL_PT_PAD);
594
595 ret = cb(&pkt_info);
596 if (ret)
597 return 0;
598
599 pkt_info.last_packet_type = pkt_info.packet.type;
600 }
601 }
602
603 struct intel_pt_calc_cyc_to_tsc_info {
604 uint64_t cycle_cnt;
605 unsigned int cbr;
606 uint32_t last_mtc;
607 uint64_t ctc_timestamp;
608 uint64_t ctc_delta;
609 uint64_t tsc_timestamp;
610 uint64_t timestamp;
611 bool have_tma;
612 bool fixup_last_mtc;
613 bool from_mtc;
614 double cbr_cyc_to_tsc;
615 };
616
617 /*
618 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
619 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
620 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
621 * packet by copying the missing bits from the current MTC assuming the least
622 * difference between the two, and that the current MTC comes after last_mtc.
623 */
624 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
625 uint32_t *last_mtc)
626 {
627 uint32_t first_missing_bit = 1U << (16 - mtc_shift);
628 uint32_t mask = ~(first_missing_bit - 1);
629
630 *last_mtc |= mtc & mask;
631 if (*last_mtc >= mtc) {
632 *last_mtc -= first_missing_bit;
633 *last_mtc &= 0xff;
634 }
635 }
636
637 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
638 {
639 struct intel_pt_decoder *decoder = pkt_info->decoder;
640 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
641 uint64_t timestamp;
642 double cyc_to_tsc;
643 unsigned int cbr;
644 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
645
646 switch (pkt_info->packet.type) {
647 case INTEL_PT_TNT:
648 case INTEL_PT_TIP_PGE:
649 case INTEL_PT_TIP:
650 case INTEL_PT_FUP:
651 case INTEL_PT_PSB:
652 case INTEL_PT_PIP:
653 case INTEL_PT_MODE_EXEC:
654 case INTEL_PT_MODE_TSX:
655 case INTEL_PT_PSBEND:
656 case INTEL_PT_PAD:
657 case INTEL_PT_VMCS:
658 case INTEL_PT_MNT:
659 return 0;
660
661 case INTEL_PT_MTC:
662 if (!data->have_tma)
663 return 0;
664
665 mtc = pkt_info->packet.payload;
666 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
667 data->fixup_last_mtc = false;
668 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
669 &data->last_mtc);
670 }
671 if (mtc > data->last_mtc)
672 mtc_delta = mtc - data->last_mtc;
673 else
674 mtc_delta = mtc + 256 - data->last_mtc;
675 data->ctc_delta += mtc_delta << decoder->mtc_shift;
676 data->last_mtc = mtc;
677
678 if (decoder->tsc_ctc_mult) {
679 timestamp = data->ctc_timestamp +
680 data->ctc_delta * decoder->tsc_ctc_mult;
681 } else {
682 timestamp = data->ctc_timestamp +
683 multdiv(data->ctc_delta,
684 decoder->tsc_ctc_ratio_n,
685 decoder->tsc_ctc_ratio_d);
686 }
687
688 if (timestamp < data->timestamp)
689 return 1;
690
691 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
692 data->timestamp = timestamp;
693 return 0;
694 }
695
696 break;
697
698 case INTEL_PT_TSC:
699 timestamp = pkt_info->packet.payload |
700 (data->timestamp & (0xffULL << 56));
701 if (data->from_mtc && timestamp < data->timestamp &&
702 data->timestamp - timestamp < decoder->tsc_slip)
703 return 1;
704 if (timestamp < data->timestamp)
705 timestamp += (1ULL << 56);
706 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
707 if (data->from_mtc)
708 return 1;
709 data->tsc_timestamp = timestamp;
710 data->timestamp = timestamp;
711 return 0;
712 }
713 break;
714
715 case INTEL_PT_TMA:
716 if (data->from_mtc)
717 return 1;
718
719 if (!decoder->tsc_ctc_ratio_d)
720 return 0;
721
722 ctc = pkt_info->packet.payload;
723 fc = pkt_info->packet.count;
724 ctc_rem = ctc & decoder->ctc_rem_mask;
725
726 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
727
728 data->ctc_timestamp = data->tsc_timestamp - fc;
729 if (decoder->tsc_ctc_mult) {
730 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
731 } else {
732 data->ctc_timestamp -=
733 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
734 decoder->tsc_ctc_ratio_d);
735 }
736
737 data->ctc_delta = 0;
738 data->have_tma = true;
739 data->fixup_last_mtc = true;
740
741 return 0;
742
743 case INTEL_PT_CYC:
744 data->cycle_cnt += pkt_info->packet.payload;
745 return 0;
746
747 case INTEL_PT_CBR:
748 cbr = pkt_info->packet.payload;
749 if (data->cbr && data->cbr != cbr)
750 return 1;
751 data->cbr = cbr;
752 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
753 return 0;
754
755 case INTEL_PT_TIP_PGD:
756 case INTEL_PT_TRACESTOP:
757 case INTEL_PT_OVF:
758 case INTEL_PT_BAD: /* Does not happen */
759 default:
760 return 1;
761 }
762
763 if (!data->cbr && decoder->cbr) {
764 data->cbr = decoder->cbr;
765 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
766 }
767
768 if (!data->cycle_cnt)
769 return 1;
770
771 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
772
773 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
774 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
775 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
776 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
777 return 1;
778 }
779
780 decoder->calc_cyc_to_tsc = cyc_to_tsc;
781 decoder->have_calc_cyc_to_tsc = true;
782
783 if (data->cbr) {
784 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
785 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
786 } else {
787 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
788 cyc_to_tsc, pkt_info->pos);
789 }
790
791 return 1;
792 }
793
794 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
795 bool from_mtc)
796 {
797 struct intel_pt_calc_cyc_to_tsc_info data = {
798 .cycle_cnt = 0,
799 .cbr = 0,
800 .last_mtc = decoder->last_mtc,
801 .ctc_timestamp = decoder->ctc_timestamp,
802 .ctc_delta = decoder->ctc_delta,
803 .tsc_timestamp = decoder->tsc_timestamp,
804 .timestamp = decoder->timestamp,
805 .have_tma = decoder->have_tma,
806 .fixup_last_mtc = decoder->fixup_last_mtc,
807 .from_mtc = from_mtc,
808 .cbr_cyc_to_tsc = 0,
809 };
810
811 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
812 }
813
814 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
815 {
816 int ret;
817
818 decoder->last_packet_type = decoder->packet.type;
819
820 do {
821 decoder->pos += decoder->pkt_step;
822 decoder->buf += decoder->pkt_step;
823 decoder->len -= decoder->pkt_step;
824
825 if (!decoder->len) {
826 ret = intel_pt_get_next_data(decoder);
827 if (ret)
828 return ret;
829 }
830
831 ret = intel_pt_get_packet(decoder->buf, decoder->len,
832 &decoder->packet);
833 if (ret == INTEL_PT_NEED_MORE_BYTES &&
834 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
835 ret = intel_pt_get_split_packet(decoder);
836 if (ret < 0)
837 return ret;
838 }
839 if (ret <= 0)
840 return intel_pt_bad_packet(decoder);
841
842 decoder->pkt_len = ret;
843 decoder->pkt_step = ret;
844 intel_pt_decoder_log_packet(decoder);
845 } while (decoder->packet.type == INTEL_PT_PAD);
846
847 return 0;
848 }
849
850 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
851 {
852 uint64_t timestamp, masked_timestamp;
853
854 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
855 masked_timestamp = timestamp & decoder->period_mask;
856 if (decoder->continuous_period) {
857 if (masked_timestamp != decoder->last_masked_timestamp)
858 return 1;
859 } else {
860 timestamp += 1;
861 masked_timestamp = timestamp & decoder->period_mask;
862 if (masked_timestamp != decoder->last_masked_timestamp) {
863 decoder->last_masked_timestamp = masked_timestamp;
864 decoder->continuous_period = true;
865 }
866 }
867 return decoder->period_ticks - (timestamp - masked_timestamp);
868 }
869
870 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
871 {
872 switch (decoder->period_type) {
873 case INTEL_PT_PERIOD_INSTRUCTIONS:
874 return decoder->period - decoder->period_insn_cnt;
875 case INTEL_PT_PERIOD_TICKS:
876 return intel_pt_next_period(decoder);
877 case INTEL_PT_PERIOD_NONE:
878 case INTEL_PT_PERIOD_MTC:
879 default:
880 return 0;
881 }
882 }
883
884 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
885 {
886 uint64_t timestamp, masked_timestamp;
887
888 switch (decoder->period_type) {
889 case INTEL_PT_PERIOD_INSTRUCTIONS:
890 decoder->period_insn_cnt = 0;
891 break;
892 case INTEL_PT_PERIOD_TICKS:
893 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
894 masked_timestamp = timestamp & decoder->period_mask;
895 decoder->last_masked_timestamp = masked_timestamp;
896 break;
897 case INTEL_PT_PERIOD_NONE:
898 case INTEL_PT_PERIOD_MTC:
899 default:
900 break;
901 }
902
903 decoder->state.type |= INTEL_PT_INSTRUCTION;
904 }
905
906 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
907 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
908 {
909 uint64_t max_insn_cnt, insn_cnt = 0;
910 int err;
911
912 if (!decoder->mtc_insn)
913 decoder->mtc_insn = true;
914
915 max_insn_cnt = intel_pt_next_sample(decoder);
916
917 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
918 max_insn_cnt, decoder->data);
919
920 decoder->tot_insn_cnt += insn_cnt;
921 decoder->timestamp_insn_cnt += insn_cnt;
922 decoder->sample_insn_cnt += insn_cnt;
923 decoder->period_insn_cnt += insn_cnt;
924
925 if (err) {
926 decoder->no_progress = 0;
927 decoder->pkt_state = INTEL_PT_STATE_ERR2;
928 intel_pt_log_at("ERROR: Failed to get instruction",
929 decoder->ip);
930 if (err == -ENOENT)
931 return -ENOLINK;
932 return -EILSEQ;
933 }
934
935 if (ip && decoder->ip == ip) {
936 err = -EAGAIN;
937 goto out;
938 }
939
940 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
941 intel_pt_sample_insn(decoder);
942
943 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
944 decoder->state.type = INTEL_PT_INSTRUCTION;
945 decoder->state.from_ip = decoder->ip;
946 decoder->state.to_ip = 0;
947 decoder->ip += intel_pt_insn->length;
948 err = INTEL_PT_RETURN;
949 goto out;
950 }
951
952 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
953 /* Zero-length calls are excluded */
954 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
955 intel_pt_insn->rel) {
956 err = intel_pt_push(&decoder->stack, decoder->ip +
957 intel_pt_insn->length);
958 if (err)
959 goto out;
960 }
961 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
962 decoder->ret_addr = intel_pt_pop(&decoder->stack);
963 }
964
965 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
966 int cnt = decoder->no_progress++;
967
968 decoder->state.from_ip = decoder->ip;
969 decoder->ip += intel_pt_insn->length +
970 intel_pt_insn->rel;
971 decoder->state.to_ip = decoder->ip;
972 err = INTEL_PT_RETURN;
973
974 /*
975 * Check for being stuck in a loop. This can happen if a
976 * decoder error results in the decoder erroneously setting the
977 * ip to an address that is itself in an infinite loop that
978 * consumes no packets. When that happens, there must be an
979 * unconditional branch.
980 */
981 if (cnt) {
982 if (cnt == 1) {
983 decoder->stuck_ip = decoder->state.to_ip;
984 decoder->stuck_ip_prd = 1;
985 decoder->stuck_ip_cnt = 1;
986 } else if (cnt > INTEL_PT_MAX_LOOPS ||
987 decoder->state.to_ip == decoder->stuck_ip) {
988 intel_pt_log_at("ERROR: Never-ending loop",
989 decoder->state.to_ip);
990 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
991 err = -ELOOP;
992 goto out;
993 } else if (!--decoder->stuck_ip_cnt) {
994 decoder->stuck_ip_prd += 1;
995 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
996 decoder->stuck_ip = decoder->state.to_ip;
997 }
998 }
999 goto out_no_progress;
1000 }
1001 out:
1002 decoder->no_progress = 0;
1003 out_no_progress:
1004 decoder->state.insn_op = intel_pt_insn->op;
1005 decoder->state.insn_len = intel_pt_insn->length;
1006 memcpy(decoder->state.insn, intel_pt_insn->buf,
1007 INTEL_PT_INSN_BUF_SZ);
1008
1009 if (decoder->tx_flags & INTEL_PT_IN_TX)
1010 decoder->state.flags |= INTEL_PT_IN_TX;
1011
1012 return err;
1013 }
1014
1015 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1016 {
1017 struct intel_pt_insn intel_pt_insn;
1018 uint64_t ip;
1019 int err;
1020
1021 ip = decoder->last_ip;
1022
1023 while (1) {
1024 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1025 if (err == INTEL_PT_RETURN)
1026 return 0;
1027 if (err == -EAGAIN) {
1028 if (decoder->set_fup_tx_flags) {
1029 decoder->set_fup_tx_flags = false;
1030 decoder->tx_flags = decoder->fup_tx_flags;
1031 decoder->state.type = INTEL_PT_TRANSACTION;
1032 decoder->state.from_ip = decoder->ip;
1033 decoder->state.to_ip = 0;
1034 decoder->state.flags = decoder->fup_tx_flags;
1035 return 0;
1036 }
1037 return err;
1038 }
1039 decoder->set_fup_tx_flags = false;
1040 if (err)
1041 return err;
1042
1043 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1044 intel_pt_log_at("ERROR: Unexpected indirect branch",
1045 decoder->ip);
1046 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1047 return -ENOENT;
1048 }
1049
1050 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1051 intel_pt_log_at("ERROR: Unexpected conditional branch",
1052 decoder->ip);
1053 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1054 return -ENOENT;
1055 }
1056
1057 intel_pt_bug(decoder);
1058 }
1059 }
1060
1061 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1062 {
1063 struct intel_pt_insn intel_pt_insn;
1064 int err;
1065
1066 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1067 if (err == INTEL_PT_RETURN &&
1068 decoder->pgd_ip &&
1069 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1070 (decoder->state.type & INTEL_PT_BRANCH) &&
1071 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1072 /* Unconditional branch leaving filter region */
1073 decoder->no_progress = 0;
1074 decoder->pge = false;
1075 decoder->continuous_period = false;
1076 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1077 decoder->state.to_ip = 0;
1078 return 0;
1079 }
1080 if (err == INTEL_PT_RETURN)
1081 return 0;
1082 if (err)
1083 return err;
1084
1085 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1086 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1087 decoder->pge = false;
1088 decoder->continuous_period = false;
1089 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1090 decoder->state.from_ip = decoder->ip;
1091 decoder->state.to_ip = 0;
1092 if (decoder->packet.count != 0)
1093 decoder->ip = decoder->last_ip;
1094 } else {
1095 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1096 decoder->state.from_ip = decoder->ip;
1097 if (decoder->packet.count == 0) {
1098 decoder->state.to_ip = 0;
1099 } else {
1100 decoder->state.to_ip = decoder->last_ip;
1101 decoder->ip = decoder->last_ip;
1102 }
1103 }
1104 return 0;
1105 }
1106
1107 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1108 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1109 intel_pt_insn.rel;
1110
1111 if (decoder->pgd_ip &&
1112 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1113 decoder->pgd_ip(to_ip, decoder->data)) {
1114 /* Conditional branch leaving filter region */
1115 decoder->pge = false;
1116 decoder->continuous_period = false;
1117 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1118 decoder->ip = to_ip;
1119 decoder->state.from_ip = decoder->ip;
1120 decoder->state.to_ip = 0;
1121 return 0;
1122 }
1123 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1124 decoder->ip);
1125 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1126 return -ENOENT;
1127 }
1128
1129 return intel_pt_bug(decoder);
1130 }
1131
1132 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1133 {
1134 struct intel_pt_insn intel_pt_insn;
1135 int err;
1136
1137 while (1) {
1138 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1139 if (err == INTEL_PT_RETURN)
1140 return 0;
1141 if (err)
1142 return err;
1143
1144 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1145 if (!decoder->return_compression) {
1146 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1147 decoder->ip);
1148 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1149 return -ENOENT;
1150 }
1151 if (!decoder->ret_addr) {
1152 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1153 decoder->ip);
1154 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1155 return -ENOENT;
1156 }
1157 if (!(decoder->tnt.payload & BIT63)) {
1158 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1159 decoder->ip);
1160 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1161 return -ENOENT;
1162 }
1163 decoder->tnt.count -= 1;
1164 if (!decoder->tnt.count)
1165 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1166 decoder->tnt.payload <<= 1;
1167 decoder->state.from_ip = decoder->ip;
1168 decoder->ip = decoder->ret_addr;
1169 decoder->state.to_ip = decoder->ip;
1170 return 0;
1171 }
1172
1173 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1174 /* Handle deferred TIPs */
1175 err = intel_pt_get_next_packet(decoder);
1176 if (err)
1177 return err;
1178 if (decoder->packet.type != INTEL_PT_TIP ||
1179 decoder->packet.count == 0) {
1180 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1181 decoder->ip);
1182 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1183 decoder->pkt_step = 0;
1184 return -ENOENT;
1185 }
1186 intel_pt_set_last_ip(decoder);
1187 decoder->state.from_ip = decoder->ip;
1188 decoder->state.to_ip = decoder->last_ip;
1189 decoder->ip = decoder->last_ip;
1190 return 0;
1191 }
1192
1193 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1194 decoder->tnt.count -= 1;
1195 if (!decoder->tnt.count)
1196 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1197 if (decoder->tnt.payload & BIT63) {
1198 decoder->tnt.payload <<= 1;
1199 decoder->state.from_ip = decoder->ip;
1200 decoder->ip += intel_pt_insn.length +
1201 intel_pt_insn.rel;
1202 decoder->state.to_ip = decoder->ip;
1203 return 0;
1204 }
1205 /* Instruction sample for a non-taken branch */
1206 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1207 decoder->tnt.payload <<= 1;
1208 decoder->state.type = INTEL_PT_INSTRUCTION;
1209 decoder->state.from_ip = decoder->ip;
1210 decoder->state.to_ip = 0;
1211 decoder->ip += intel_pt_insn.length;
1212 return 0;
1213 }
1214 decoder->ip += intel_pt_insn.length;
1215 if (!decoder->tnt.count)
1216 return -EAGAIN;
1217 decoder->tnt.payload <<= 1;
1218 continue;
1219 }
1220
1221 return intel_pt_bug(decoder);
1222 }
1223 }
1224
1225 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1226 {
1227 unsigned int fup_tx_flags;
1228 int err;
1229
1230 fup_tx_flags = decoder->packet.payload &
1231 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1232 err = intel_pt_get_next_packet(decoder);
1233 if (err)
1234 return err;
1235 if (decoder->packet.type == INTEL_PT_FUP) {
1236 decoder->fup_tx_flags = fup_tx_flags;
1237 decoder->set_fup_tx_flags = true;
1238 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1239 *no_tip = true;
1240 } else {
1241 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1242 decoder->pos);
1243 intel_pt_update_in_tx(decoder);
1244 }
1245 return 0;
1246 }
1247
1248 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1249 {
1250 uint64_t timestamp;
1251
1252 decoder->have_tma = false;
1253
1254 if (decoder->ref_timestamp) {
1255 timestamp = decoder->packet.payload |
1256 (decoder->ref_timestamp & (0xffULL << 56));
1257 if (timestamp < decoder->ref_timestamp) {
1258 if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1259 timestamp += (1ULL << 56);
1260 } else {
1261 if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1262 timestamp -= (1ULL << 56);
1263 }
1264 decoder->tsc_timestamp = timestamp;
1265 decoder->timestamp = timestamp;
1266 decoder->ref_timestamp = 0;
1267 decoder->timestamp_insn_cnt = 0;
1268 } else if (decoder->timestamp) {
1269 timestamp = decoder->packet.payload |
1270 (decoder->timestamp & (0xffULL << 56));
1271 decoder->tsc_timestamp = timestamp;
1272 if (timestamp < decoder->timestamp &&
1273 decoder->timestamp - timestamp < decoder->tsc_slip) {
1274 intel_pt_log_to("Suppressing backwards timestamp",
1275 timestamp);
1276 timestamp = decoder->timestamp;
1277 }
1278 if (timestamp < decoder->timestamp) {
1279 intel_pt_log_to("Wraparound timestamp", timestamp);
1280 timestamp += (1ULL << 56);
1281 decoder->tsc_timestamp = timestamp;
1282 }
1283 decoder->timestamp = timestamp;
1284 decoder->timestamp_insn_cnt = 0;
1285 }
1286
1287 if (decoder->last_packet_type == INTEL_PT_CYC) {
1288 decoder->cyc_ref_timestamp = decoder->timestamp;
1289 decoder->cycle_cnt = 0;
1290 decoder->have_calc_cyc_to_tsc = false;
1291 intel_pt_calc_cyc_to_tsc(decoder, false);
1292 }
1293
1294 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1295 }
1296
1297 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1298 {
1299 intel_pt_log("ERROR: Buffer overflow\n");
1300 intel_pt_clear_tx_flags(decoder);
1301 decoder->have_tma = false;
1302 decoder->cbr = 0;
1303 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1304 decoder->overflow = true;
1305 return -EOVERFLOW;
1306 }
1307
1308 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1309 {
1310 uint32_t ctc = decoder->packet.payload;
1311 uint32_t fc = decoder->packet.count;
1312 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1313
1314 if (!decoder->tsc_ctc_ratio_d)
1315 return;
1316
1317 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1318 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1319 if (decoder->tsc_ctc_mult) {
1320 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1321 } else {
1322 decoder->ctc_timestamp -= multdiv(ctc_rem,
1323 decoder->tsc_ctc_ratio_n,
1324 decoder->tsc_ctc_ratio_d);
1325 }
1326 decoder->ctc_delta = 0;
1327 decoder->have_tma = true;
1328 decoder->fixup_last_mtc = true;
1329 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1330 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1331 }
1332
1333 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1334 {
1335 uint64_t timestamp;
1336 uint32_t mtc, mtc_delta;
1337
1338 if (!decoder->have_tma)
1339 return;
1340
1341 mtc = decoder->packet.payload;
1342
1343 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1344 decoder->fixup_last_mtc = false;
1345 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1346 &decoder->last_mtc);
1347 }
1348
1349 if (mtc > decoder->last_mtc)
1350 mtc_delta = mtc - decoder->last_mtc;
1351 else
1352 mtc_delta = mtc + 256 - decoder->last_mtc;
1353
1354 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1355
1356 if (decoder->tsc_ctc_mult) {
1357 timestamp = decoder->ctc_timestamp +
1358 decoder->ctc_delta * decoder->tsc_ctc_mult;
1359 } else {
1360 timestamp = decoder->ctc_timestamp +
1361 multdiv(decoder->ctc_delta,
1362 decoder->tsc_ctc_ratio_n,
1363 decoder->tsc_ctc_ratio_d);
1364 }
1365
1366 if (timestamp < decoder->timestamp)
1367 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1368 timestamp, decoder->timestamp);
1369 else
1370 decoder->timestamp = timestamp;
1371
1372 decoder->timestamp_insn_cnt = 0;
1373 decoder->last_mtc = mtc;
1374
1375 if (decoder->last_packet_type == INTEL_PT_CYC) {
1376 decoder->cyc_ref_timestamp = decoder->timestamp;
1377 decoder->cycle_cnt = 0;
1378 decoder->have_calc_cyc_to_tsc = false;
1379 intel_pt_calc_cyc_to_tsc(decoder, true);
1380 }
1381 }
1382
1383 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1384 {
1385 unsigned int cbr = decoder->packet.payload;
1386
1387 if (decoder->cbr == cbr)
1388 return;
1389
1390 decoder->cbr = cbr;
1391 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1392 }
1393
1394 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1395 {
1396 uint64_t timestamp = decoder->cyc_ref_timestamp;
1397
1398 decoder->have_cyc = true;
1399
1400 decoder->cycle_cnt += decoder->packet.payload;
1401
1402 if (!decoder->cyc_ref_timestamp)
1403 return;
1404
1405 if (decoder->have_calc_cyc_to_tsc)
1406 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1407 else if (decoder->cbr)
1408 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1409 else
1410 return;
1411
1412 if (timestamp < decoder->timestamp)
1413 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1414 timestamp, decoder->timestamp);
1415 else
1416 decoder->timestamp = timestamp;
1417
1418 decoder->timestamp_insn_cnt = 0;
1419 }
1420
1421 /* Walk PSB+ packets when already in sync. */
1422 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1423 {
1424 int err;
1425
1426 while (1) {
1427 err = intel_pt_get_next_packet(decoder);
1428 if (err)
1429 return err;
1430
1431 switch (decoder->packet.type) {
1432 case INTEL_PT_PSBEND:
1433 return 0;
1434
1435 case INTEL_PT_TIP_PGD:
1436 case INTEL_PT_TIP_PGE:
1437 case INTEL_PT_TIP:
1438 case INTEL_PT_TNT:
1439 case INTEL_PT_TRACESTOP:
1440 case INTEL_PT_BAD:
1441 case INTEL_PT_PSB:
1442 decoder->have_tma = false;
1443 intel_pt_log("ERROR: Unexpected packet\n");
1444 return -EAGAIN;
1445
1446 case INTEL_PT_OVF:
1447 return intel_pt_overflow(decoder);
1448
1449 case INTEL_PT_TSC:
1450 intel_pt_calc_tsc_timestamp(decoder);
1451 break;
1452
1453 case INTEL_PT_TMA:
1454 intel_pt_calc_tma(decoder);
1455 break;
1456
1457 case INTEL_PT_CBR:
1458 intel_pt_calc_cbr(decoder);
1459 break;
1460
1461 case INTEL_PT_MODE_EXEC:
1462 decoder->exec_mode = decoder->packet.payload;
1463 break;
1464
1465 case INTEL_PT_PIP:
1466 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1467 break;
1468
1469 case INTEL_PT_FUP:
1470 decoder->pge = true;
1471 intel_pt_set_last_ip(decoder);
1472 break;
1473
1474 case INTEL_PT_MODE_TSX:
1475 intel_pt_update_in_tx(decoder);
1476 break;
1477
1478 case INTEL_PT_MTC:
1479 intel_pt_calc_mtc_timestamp(decoder);
1480 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1481 decoder->state.type |= INTEL_PT_INSTRUCTION;
1482 break;
1483
1484 case INTEL_PT_CYC:
1485 case INTEL_PT_VMCS:
1486 case INTEL_PT_MNT:
1487 case INTEL_PT_PAD:
1488 default:
1489 break;
1490 }
1491 }
1492 }
1493
1494 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1495 {
1496 int err;
1497
1498 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1499 decoder->tx_flags = 0;
1500 decoder->state.flags &= ~INTEL_PT_IN_TX;
1501 decoder->state.flags |= INTEL_PT_ABORT_TX;
1502 } else {
1503 decoder->state.flags |= INTEL_PT_ASYNC;
1504 }
1505
1506 while (1) {
1507 err = intel_pt_get_next_packet(decoder);
1508 if (err)
1509 return err;
1510
1511 switch (decoder->packet.type) {
1512 case INTEL_PT_TNT:
1513 case INTEL_PT_FUP:
1514 case INTEL_PT_TRACESTOP:
1515 case INTEL_PT_PSB:
1516 case INTEL_PT_TSC:
1517 case INTEL_PT_TMA:
1518 case INTEL_PT_CBR:
1519 case INTEL_PT_MODE_TSX:
1520 case INTEL_PT_BAD:
1521 case INTEL_PT_PSBEND:
1522 intel_pt_log("ERROR: Missing TIP after FUP\n");
1523 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1524 return -ENOENT;
1525
1526 case INTEL_PT_OVF:
1527 return intel_pt_overflow(decoder);
1528
1529 case INTEL_PT_TIP_PGD:
1530 decoder->state.from_ip = decoder->ip;
1531 decoder->state.to_ip = 0;
1532 if (decoder->packet.count != 0) {
1533 intel_pt_set_ip(decoder);
1534 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1535 decoder->ip);
1536 }
1537 decoder->pge = false;
1538 decoder->continuous_period = false;
1539 return 0;
1540
1541 case INTEL_PT_TIP_PGE:
1542 decoder->pge = true;
1543 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1544 decoder->ip);
1545 decoder->state.from_ip = 0;
1546 if (decoder->packet.count == 0) {
1547 decoder->state.to_ip = 0;
1548 } else {
1549 intel_pt_set_ip(decoder);
1550 decoder->state.to_ip = decoder->ip;
1551 }
1552 return 0;
1553
1554 case INTEL_PT_TIP:
1555 decoder->state.from_ip = decoder->ip;
1556 if (decoder->packet.count == 0) {
1557 decoder->state.to_ip = 0;
1558 } else {
1559 intel_pt_set_ip(decoder);
1560 decoder->state.to_ip = decoder->ip;
1561 }
1562 return 0;
1563
1564 case INTEL_PT_PIP:
1565 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1566 break;
1567
1568 case INTEL_PT_MTC:
1569 intel_pt_calc_mtc_timestamp(decoder);
1570 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1571 decoder->state.type |= INTEL_PT_INSTRUCTION;
1572 break;
1573
1574 case INTEL_PT_CYC:
1575 intel_pt_calc_cyc_timestamp(decoder);
1576 break;
1577
1578 case INTEL_PT_MODE_EXEC:
1579 decoder->exec_mode = decoder->packet.payload;
1580 break;
1581
1582 case INTEL_PT_VMCS:
1583 case INTEL_PT_MNT:
1584 case INTEL_PT_PAD:
1585 break;
1586
1587 default:
1588 return intel_pt_bug(decoder);
1589 }
1590 }
1591 }
1592
1593 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1594 {
1595 bool no_tip = false;
1596 int err;
1597
1598 while (1) {
1599 err = intel_pt_get_next_packet(decoder);
1600 if (err)
1601 return err;
1602 next:
1603 switch (decoder->packet.type) {
1604 case INTEL_PT_TNT:
1605 if (!decoder->packet.count)
1606 break;
1607 decoder->tnt = decoder->packet;
1608 decoder->pkt_state = INTEL_PT_STATE_TNT;
1609 err = intel_pt_walk_tnt(decoder);
1610 if (err == -EAGAIN)
1611 break;
1612 return err;
1613
1614 case INTEL_PT_TIP_PGD:
1615 if (decoder->packet.count != 0)
1616 intel_pt_set_last_ip(decoder);
1617 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1618 return intel_pt_walk_tip(decoder);
1619
1620 case INTEL_PT_TIP_PGE: {
1621 decoder->pge = true;
1622 if (decoder->packet.count == 0) {
1623 intel_pt_log_at("Skipping zero TIP.PGE",
1624 decoder->pos);
1625 break;
1626 }
1627 intel_pt_set_ip(decoder);
1628 decoder->state.from_ip = 0;
1629 decoder->state.to_ip = decoder->ip;
1630 return 0;
1631 }
1632
1633 case INTEL_PT_OVF:
1634 return intel_pt_overflow(decoder);
1635
1636 case INTEL_PT_TIP:
1637 if (decoder->packet.count != 0)
1638 intel_pt_set_last_ip(decoder);
1639 decoder->pkt_state = INTEL_PT_STATE_TIP;
1640 return intel_pt_walk_tip(decoder);
1641
1642 case INTEL_PT_FUP:
1643 if (decoder->packet.count == 0) {
1644 intel_pt_log_at("Skipping zero FUP",
1645 decoder->pos);
1646 no_tip = false;
1647 break;
1648 }
1649 intel_pt_set_last_ip(decoder);
1650 err = intel_pt_walk_fup(decoder);
1651 if (err != -EAGAIN) {
1652 if (err)
1653 return err;
1654 if (no_tip)
1655 decoder->pkt_state =
1656 INTEL_PT_STATE_FUP_NO_TIP;
1657 else
1658 decoder->pkt_state = INTEL_PT_STATE_FUP;
1659 return 0;
1660 }
1661 if (no_tip) {
1662 no_tip = false;
1663 break;
1664 }
1665 return intel_pt_walk_fup_tip(decoder);
1666
1667 case INTEL_PT_TRACESTOP:
1668 decoder->pge = false;
1669 decoder->continuous_period = false;
1670 intel_pt_clear_tx_flags(decoder);
1671 decoder->have_tma = false;
1672 break;
1673
1674 case INTEL_PT_PSB:
1675 intel_pt_clear_stack(&decoder->stack);
1676 err = intel_pt_walk_psbend(decoder);
1677 if (err == -EAGAIN)
1678 goto next;
1679 if (err)
1680 return err;
1681 break;
1682
1683 case INTEL_PT_PIP:
1684 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1685 break;
1686
1687 case INTEL_PT_MTC:
1688 intel_pt_calc_mtc_timestamp(decoder);
1689 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1690 break;
1691 /*
1692 * Ensure that there has been an instruction since the
1693 * last MTC.
1694 */
1695 if (!decoder->mtc_insn)
1696 break;
1697 decoder->mtc_insn = false;
1698 /* Ensure that there is a timestamp */
1699 if (!decoder->timestamp)
1700 break;
1701 decoder->state.type = INTEL_PT_INSTRUCTION;
1702 decoder->state.from_ip = decoder->ip;
1703 decoder->state.to_ip = 0;
1704 decoder->mtc_insn = false;
1705 return 0;
1706
1707 case INTEL_PT_TSC:
1708 intel_pt_calc_tsc_timestamp(decoder);
1709 break;
1710
1711 case INTEL_PT_TMA:
1712 intel_pt_calc_tma(decoder);
1713 break;
1714
1715 case INTEL_PT_CYC:
1716 intel_pt_calc_cyc_timestamp(decoder);
1717 break;
1718
1719 case INTEL_PT_CBR:
1720 intel_pt_calc_cbr(decoder);
1721 break;
1722
1723 case INTEL_PT_MODE_EXEC:
1724 decoder->exec_mode = decoder->packet.payload;
1725 break;
1726
1727 case INTEL_PT_MODE_TSX:
1728 /* MODE_TSX need not be followed by FUP */
1729 if (!decoder->pge) {
1730 intel_pt_update_in_tx(decoder);
1731 break;
1732 }
1733 err = intel_pt_mode_tsx(decoder, &no_tip);
1734 if (err)
1735 return err;
1736 goto next;
1737
1738 case INTEL_PT_BAD: /* Does not happen */
1739 return intel_pt_bug(decoder);
1740
1741 case INTEL_PT_PSBEND:
1742 case INTEL_PT_VMCS:
1743 case INTEL_PT_MNT:
1744 case INTEL_PT_PAD:
1745 break;
1746
1747 default:
1748 return intel_pt_bug(decoder);
1749 }
1750 }
1751 }
1752
1753 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1754 {
1755 return decoder->last_ip || decoder->packet.count == 0 ||
1756 decoder->packet.count == 3 || decoder->packet.count == 6;
1757 }
1758
1759 /* Walk PSB+ packets to get in sync. */
1760 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1761 {
1762 int err;
1763
1764 while (1) {
1765 err = intel_pt_get_next_packet(decoder);
1766 if (err)
1767 return err;
1768
1769 switch (decoder->packet.type) {
1770 case INTEL_PT_TIP_PGD:
1771 decoder->continuous_period = false;
1772 __fallthrough;
1773 case INTEL_PT_TIP_PGE:
1774 case INTEL_PT_TIP:
1775 intel_pt_log("ERROR: Unexpected packet\n");
1776 return -ENOENT;
1777
1778 case INTEL_PT_FUP:
1779 decoder->pge = true;
1780 if (intel_pt_have_ip(decoder)) {
1781 uint64_t current_ip = decoder->ip;
1782
1783 intel_pt_set_ip(decoder);
1784 if (current_ip)
1785 intel_pt_log_to("Setting IP",
1786 decoder->ip);
1787 }
1788 break;
1789
1790 case INTEL_PT_MTC:
1791 intel_pt_calc_mtc_timestamp(decoder);
1792 break;
1793
1794 case INTEL_PT_TSC:
1795 intel_pt_calc_tsc_timestamp(decoder);
1796 break;
1797
1798 case INTEL_PT_TMA:
1799 intel_pt_calc_tma(decoder);
1800 break;
1801
1802 case INTEL_PT_CYC:
1803 intel_pt_calc_cyc_timestamp(decoder);
1804 break;
1805
1806 case INTEL_PT_CBR:
1807 intel_pt_calc_cbr(decoder);
1808 break;
1809
1810 case INTEL_PT_PIP:
1811 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1812 break;
1813
1814 case INTEL_PT_MODE_EXEC:
1815 decoder->exec_mode = decoder->packet.payload;
1816 break;
1817
1818 case INTEL_PT_MODE_TSX:
1819 intel_pt_update_in_tx(decoder);
1820 break;
1821
1822 case INTEL_PT_TRACESTOP:
1823 decoder->pge = false;
1824 decoder->continuous_period = false;
1825 intel_pt_clear_tx_flags(decoder);
1826 __fallthrough;
1827
1828 case INTEL_PT_TNT:
1829 decoder->have_tma = false;
1830 intel_pt_log("ERROR: Unexpected packet\n");
1831 if (decoder->ip)
1832 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1833 else
1834 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1835 return -ENOENT;
1836
1837 case INTEL_PT_BAD: /* Does not happen */
1838 return intel_pt_bug(decoder);
1839
1840 case INTEL_PT_OVF:
1841 return intel_pt_overflow(decoder);
1842
1843 case INTEL_PT_PSBEND:
1844 return 0;
1845
1846 case INTEL_PT_PSB:
1847 case INTEL_PT_VMCS:
1848 case INTEL_PT_MNT:
1849 case INTEL_PT_PAD:
1850 default:
1851 break;
1852 }
1853 }
1854 }
1855
1856 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1857 {
1858 int err;
1859
1860 while (1) {
1861 err = intel_pt_get_next_packet(decoder);
1862 if (err)
1863 return err;
1864
1865 switch (decoder->packet.type) {
1866 case INTEL_PT_TIP_PGD:
1867 decoder->continuous_period = false;
1868 __fallthrough;
1869 case INTEL_PT_TIP_PGE:
1870 case INTEL_PT_TIP:
1871 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1872 if (intel_pt_have_ip(decoder))
1873 intel_pt_set_ip(decoder);
1874 if (decoder->ip)
1875 return 0;
1876 break;
1877
1878 case INTEL_PT_FUP:
1879 if (decoder->overflow) {
1880 if (intel_pt_have_ip(decoder))
1881 intel_pt_set_ip(decoder);
1882 if (decoder->ip)
1883 return 0;
1884 }
1885 if (decoder->packet.count)
1886 intel_pt_set_last_ip(decoder);
1887 break;
1888
1889 case INTEL_PT_MTC:
1890 intel_pt_calc_mtc_timestamp(decoder);
1891 break;
1892
1893 case INTEL_PT_TSC:
1894 intel_pt_calc_tsc_timestamp(decoder);
1895 break;
1896
1897 case INTEL_PT_TMA:
1898 intel_pt_calc_tma(decoder);
1899 break;
1900
1901 case INTEL_PT_CYC:
1902 intel_pt_calc_cyc_timestamp(decoder);
1903 break;
1904
1905 case INTEL_PT_CBR:
1906 intel_pt_calc_cbr(decoder);
1907 break;
1908
1909 case INTEL_PT_PIP:
1910 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1911 break;
1912
1913 case INTEL_PT_MODE_EXEC:
1914 decoder->exec_mode = decoder->packet.payload;
1915 break;
1916
1917 case INTEL_PT_MODE_TSX:
1918 intel_pt_update_in_tx(decoder);
1919 break;
1920
1921 case INTEL_PT_OVF:
1922 return intel_pt_overflow(decoder);
1923
1924 case INTEL_PT_BAD: /* Does not happen */
1925 return intel_pt_bug(decoder);
1926
1927 case INTEL_PT_TRACESTOP:
1928 decoder->pge = false;
1929 decoder->continuous_period = false;
1930 intel_pt_clear_tx_flags(decoder);
1931 decoder->have_tma = false;
1932 break;
1933
1934 case INTEL_PT_PSB:
1935 intel_pt_clear_stack(&decoder->stack);
1936 err = intel_pt_walk_psb(decoder);
1937 if (err)
1938 return err;
1939 if (decoder->ip) {
1940 /* Do not have a sample */
1941 decoder->state.type = 0;
1942 return 0;
1943 }
1944 break;
1945
1946 case INTEL_PT_TNT:
1947 case INTEL_PT_PSBEND:
1948 case INTEL_PT_VMCS:
1949 case INTEL_PT_MNT:
1950 case INTEL_PT_PAD:
1951 default:
1952 break;
1953 }
1954 }
1955 }
1956
1957 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1958 {
1959 int err;
1960
1961 intel_pt_log("Scanning for full IP\n");
1962 err = intel_pt_walk_to_ip(decoder);
1963 if (err)
1964 return err;
1965
1966 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1967 decoder->overflow = false;
1968
1969 decoder->state.from_ip = 0;
1970 decoder->state.to_ip = decoder->ip;
1971 intel_pt_log_to("Setting IP", decoder->ip);
1972
1973 return 0;
1974 }
1975
1976 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1977 {
1978 const unsigned char *end = decoder->buf + decoder->len;
1979 size_t i;
1980
1981 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1982 if (i > decoder->len)
1983 continue;
1984 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1985 return i;
1986 }
1987 return 0;
1988 }
1989
1990 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1991 {
1992 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1993 const char *psb = INTEL_PT_PSB_STR;
1994
1995 if (rest_psb > decoder->len ||
1996 memcmp(decoder->buf, psb + part_psb, rest_psb))
1997 return 0;
1998
1999 return rest_psb;
2000 }
2001
2002 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2003 int part_psb)
2004 {
2005 int rest_psb, ret;
2006
2007 decoder->pos += decoder->len;
2008 decoder->len = 0;
2009
2010 ret = intel_pt_get_next_data(decoder);
2011 if (ret)
2012 return ret;
2013
2014 rest_psb = intel_pt_rest_psb(decoder, part_psb);
2015 if (!rest_psb)
2016 return 0;
2017
2018 decoder->pos -= part_psb;
2019 decoder->next_buf = decoder->buf + rest_psb;
2020 decoder->next_len = decoder->len - rest_psb;
2021 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2022 decoder->buf = decoder->temp_buf;
2023 decoder->len = INTEL_PT_PSB_LEN;
2024
2025 return 0;
2026 }
2027
2028 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2029 {
2030 unsigned char *next;
2031 int ret;
2032
2033 intel_pt_log("Scanning for PSB\n");
2034 while (1) {
2035 if (!decoder->len) {
2036 ret = intel_pt_get_next_data(decoder);
2037 if (ret)
2038 return ret;
2039 }
2040
2041 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2042 INTEL_PT_PSB_LEN);
2043 if (!next) {
2044 int part_psb;
2045
2046 part_psb = intel_pt_part_psb(decoder);
2047 if (part_psb) {
2048 ret = intel_pt_get_split_psb(decoder, part_psb);
2049 if (ret)
2050 return ret;
2051 } else {
2052 decoder->pos += decoder->len;
2053 decoder->len = 0;
2054 }
2055 continue;
2056 }
2057
2058 decoder->pkt_step = next - decoder->buf;
2059 return intel_pt_get_next_packet(decoder);
2060 }
2061 }
2062
2063 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2064 {
2065 int err;
2066
2067 decoder->pge = false;
2068 decoder->continuous_period = false;
2069 decoder->last_ip = 0;
2070 decoder->ip = 0;
2071 intel_pt_clear_stack(&decoder->stack);
2072
2073 err = intel_pt_scan_for_psb(decoder);
2074 if (err)
2075 return err;
2076
2077 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2078
2079 err = intel_pt_walk_psb(decoder);
2080 if (err)
2081 return err;
2082
2083 if (decoder->ip) {
2084 decoder->state.type = 0; /* Do not have a sample */
2085 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2086 } else {
2087 return intel_pt_sync_ip(decoder);
2088 }
2089
2090 return 0;
2091 }
2092
2093 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2094 {
2095 uint64_t est = decoder->sample_insn_cnt << 1;
2096
2097 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2098 goto out;
2099
2100 est *= decoder->max_non_turbo_ratio;
2101 est /= decoder->cbr;
2102 out:
2103 return decoder->sample_timestamp + est;
2104 }
2105
2106 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2107 {
2108 int err;
2109
2110 do {
2111 decoder->state.type = INTEL_PT_BRANCH;
2112 decoder->state.flags = 0;
2113
2114 switch (decoder->pkt_state) {
2115 case INTEL_PT_STATE_NO_PSB:
2116 err = intel_pt_sync(decoder);
2117 break;
2118 case INTEL_PT_STATE_NO_IP:
2119 decoder->last_ip = 0;
2120 decoder->ip = 0;
2121 /* Fall through */
2122 case INTEL_PT_STATE_ERR_RESYNC:
2123 err = intel_pt_sync_ip(decoder);
2124 break;
2125 case INTEL_PT_STATE_IN_SYNC:
2126 err = intel_pt_walk_trace(decoder);
2127 break;
2128 case INTEL_PT_STATE_TNT:
2129 err = intel_pt_walk_tnt(decoder);
2130 if (err == -EAGAIN)
2131 err = intel_pt_walk_trace(decoder);
2132 break;
2133 case INTEL_PT_STATE_TIP:
2134 case INTEL_PT_STATE_TIP_PGD:
2135 err = intel_pt_walk_tip(decoder);
2136 break;
2137 case INTEL_PT_STATE_FUP:
2138 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2139 err = intel_pt_walk_fup(decoder);
2140 if (err == -EAGAIN)
2141 err = intel_pt_walk_fup_tip(decoder);
2142 else if (!err)
2143 decoder->pkt_state = INTEL_PT_STATE_FUP;
2144 break;
2145 case INTEL_PT_STATE_FUP_NO_TIP:
2146 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2147 err = intel_pt_walk_fup(decoder);
2148 if (err == -EAGAIN)
2149 err = intel_pt_walk_trace(decoder);
2150 break;
2151 default:
2152 err = intel_pt_bug(decoder);
2153 break;
2154 }
2155 } while (err == -ENOLINK);
2156
2157 if (err) {
2158 decoder->state.err = intel_pt_ext_err(err);
2159 decoder->state.from_ip = decoder->ip;
2160 decoder->sample_timestamp = decoder->timestamp;
2161 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2162 } else {
2163 decoder->state.err = 0;
2164 if (intel_pt_sample_time(decoder->pkt_state)) {
2165 decoder->sample_timestamp = decoder->timestamp;
2166 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2167 }
2168 }
2169
2170 decoder->state.timestamp = decoder->sample_timestamp;
2171 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2172 decoder->state.cr3 = decoder->cr3;
2173 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2174
2175 return &decoder->state;
2176 }
2177
2178 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2179 {
2180 if (len < INTEL_PT_PSB_LEN)
2181 return false;
2182 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2183 INTEL_PT_PSB_LEN);
2184 }
2185
2186 /**
2187 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2188 * @buf: pointer to buffer pointer
2189 * @len: size of buffer
2190 *
2191 * Updates the buffer pointer to point to the start of the next PSB packet if
2192 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2193 * @len is adjusted accordingly.
2194 *
2195 * Return: %true if a PSB packet is found, %false otherwise.
2196 */
2197 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2198 {
2199 unsigned char *next;
2200
2201 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2202 if (next) {
2203 *len -= next - *buf;
2204 *buf = next;
2205 return true;
2206 }
2207 return false;
2208 }
2209
2210 /**
2211 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2212 * packet.
2213 * @buf: pointer to buffer pointer
2214 * @len: size of buffer
2215 *
2216 * Updates the buffer pointer to point to the start of the following PSB packet
2217 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2218 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2219 *
2220 * Return: %true if a PSB packet is found, %false otherwise.
2221 */
2222 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2223 {
2224 unsigned char *next;
2225
2226 if (!*len)
2227 return false;
2228
2229 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2230 if (next) {
2231 *len -= next - *buf;
2232 *buf = next;
2233 return true;
2234 }
2235 return false;
2236 }
2237
2238 /**
2239 * intel_pt_last_psb - find the last PSB packet in a buffer.
2240 * @buf: buffer
2241 * @len: size of buffer
2242 *
2243 * This function finds the last PSB in a buffer.
2244 *
2245 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2246 */
2247 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2248 {
2249 const char *n = INTEL_PT_PSB_STR;
2250 unsigned char *p;
2251 size_t k;
2252
2253 if (len < INTEL_PT_PSB_LEN)
2254 return NULL;
2255
2256 k = len - INTEL_PT_PSB_LEN + 1;
2257 while (1) {
2258 p = memrchr(buf, n[0], k);
2259 if (!p)
2260 return NULL;
2261 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2262 return p;
2263 k = p - buf;
2264 if (!k)
2265 return NULL;
2266 }
2267 }
2268
2269 /**
2270 * intel_pt_next_tsc - find and return next TSC.
2271 * @buf: buffer
2272 * @len: size of buffer
2273 * @tsc: TSC value returned
2274 *
2275 * Find a TSC packet in @buf and return the TSC value. This function assumes
2276 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2277 * PSBEND packet is found.
2278 *
2279 * Return: %true if TSC is found, false otherwise.
2280 */
2281 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2282 {
2283 struct intel_pt_pkt packet;
2284 int ret;
2285
2286 while (len) {
2287 ret = intel_pt_get_packet(buf, len, &packet);
2288 if (ret <= 0)
2289 return false;
2290 if (packet.type == INTEL_PT_TSC) {
2291 *tsc = packet.payload;
2292 return true;
2293 }
2294 if (packet.type == INTEL_PT_PSBEND)
2295 return false;
2296 buf += ret;
2297 len -= ret;
2298 }
2299 return false;
2300 }
2301
2302 /**
2303 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2304 * @tsc1: first TSC to compare
2305 * @tsc2: second TSC to compare
2306 *
2307 * This function compares 7-byte TSC values allowing for the possibility that
2308 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2309 * around so for that purpose this function assumes the absolute difference is
2310 * less than half the maximum difference.
2311 *
2312 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2313 * after @tsc2.
2314 */
2315 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2316 {
2317 const uint64_t halfway = (1ULL << 55);
2318
2319 if (tsc1 == tsc2)
2320 return 0;
2321
2322 if (tsc1 < tsc2) {
2323 if (tsc2 - tsc1 < halfway)
2324 return -1;
2325 else
2326 return 1;
2327 } else {
2328 if (tsc1 - tsc2 < halfway)
2329 return 1;
2330 else
2331 return -1;
2332 }
2333 }
2334
2335 /**
2336 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2337 * using TSC.
2338 * @buf_a: first buffer
2339 * @len_a: size of first buffer
2340 * @buf_b: second buffer
2341 * @len_b: size of second buffer
2342 *
2343 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2344 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2345 * walk forward in @buf_b until a later TSC is found. A precondition is that
2346 * @buf_a and @buf_b are positioned at a PSB.
2347 *
2348 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2349 * @buf_b + @len_b if there is no non-overlapped data.
2350 */
2351 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2352 size_t len_a,
2353 unsigned char *buf_b,
2354 size_t len_b)
2355 {
2356 uint64_t tsc_a, tsc_b;
2357 unsigned char *p;
2358 size_t len;
2359
2360 p = intel_pt_last_psb(buf_a, len_a);
2361 if (!p)
2362 return buf_b; /* No PSB in buf_a => no overlap */
2363
2364 len = len_a - (p - buf_a);
2365 if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2366 /* The last PSB+ in buf_a is incomplete, so go back one more */
2367 len_a -= len;
2368 p = intel_pt_last_psb(buf_a, len_a);
2369 if (!p)
2370 return buf_b; /* No full PSB+ => assume no overlap */
2371 len = len_a - (p - buf_a);
2372 if (!intel_pt_next_tsc(p, len, &tsc_a))
2373 return buf_b; /* No TSC in buf_a => assume no overlap */
2374 }
2375
2376 while (1) {
2377 /* Ignore PSB+ with no TSC */
2378 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2379 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2380 return buf_b; /* tsc_a < tsc_b => no overlap */
2381
2382 if (!intel_pt_step_psb(&buf_b, &len_b))
2383 return buf_b + len_b; /* No PSB in buf_b => no data */
2384 }
2385 }
2386
2387 /**
2388 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2389 * @buf_a: first buffer
2390 * @len_a: size of first buffer
2391 * @buf_b: second buffer
2392 * @len_b: size of second buffer
2393 * @have_tsc: can use TSC packets to detect overlap
2394 *
2395 * When trace samples or snapshots are recorded there is the possibility that
2396 * the data overlaps. Note that, for the purposes of decoding, data is only
2397 * useful if it begins with a PSB packet.
2398 *
2399 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2400 * @buf_b + @len_b if there is no non-overlapped data.
2401 */
2402 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2403 unsigned char *buf_b, size_t len_b,
2404 bool have_tsc)
2405 {
2406 unsigned char *found;
2407
2408 /* Buffer 'b' must start at PSB so throw away everything before that */
2409 if (!intel_pt_next_psb(&buf_b, &len_b))
2410 return buf_b + len_b; /* No PSB */
2411
2412 if (!intel_pt_next_psb(&buf_a, &len_a))
2413 return buf_b; /* No overlap */
2414
2415 if (have_tsc) {
2416 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2417 if (found)
2418 return found;
2419 }
2420
2421 /*
2422 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2423 * we can ignore the first part of buffer 'a'.
2424 */
2425 while (len_b < len_a) {
2426 if (!intel_pt_step_psb(&buf_a, &len_a))
2427 return buf_b; /* No overlap */
2428 }
2429
2430 /* Now len_b >= len_a */
2431 if (len_b > len_a) {
2432 /* The leftover buffer 'b' must start at a PSB */
2433 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2434 if (!intel_pt_step_psb(&buf_a, &len_a))
2435 return buf_b; /* No overlap */
2436 }
2437 }
2438
2439 while (1) {
2440 /* Potential overlap so check the bytes */
2441 found = memmem(buf_a, len_a, buf_b, len_a);
2442 if (found)
2443 return buf_b + len_a;
2444
2445 /* Try again at next PSB in buffer 'a' */
2446 if (!intel_pt_step_psb(&buf_a, &len_a))
2447 return buf_b; /* No overlap */
2448
2449 /* The leftover buffer 'b' must start at a PSB */
2450 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2451 if (!intel_pt_step_psb(&buf_a, &len_a))
2452 return buf_b; /* No overlap */
2453 }
2454 }
2455 }
ubuntu-bionic-kernel mirror