]>
Commit | Line | Data |
---|---|---|
f4aa0819 AH |
1 | /* |
2 | * intel_pt_decoder.c: Intel Processor Trace support | |
3 | * Copyright (c) 2013-2014, Intel Corporation. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | */ | |
15 | ||
16 | #ifndef _GNU_SOURCE | |
17 | #define _GNU_SOURCE | |
18 | #endif | |
19 | #include <stdlib.h> | |
20 | #include <stdbool.h> | |
21 | #include <string.h> | |
22 | #include <errno.h> | |
23 | #include <stdint.h> | |
24 | #include <inttypes.h> | |
7ea6856d | 25 | #include <linux/compiler.h> |
f4aa0819 AH |
26 | |
27 | #include "../cache.h" | |
28 | #include "../util.h" | |
29 | ||
30 | #include "intel-pt-insn-decoder.h" | |
31 | #include "intel-pt-pkt-decoder.h" | |
32 | #include "intel-pt-decoder.h" | |
33 | #include "intel-pt-log.h" | |
34 | ||
35 | #define INTEL_PT_BLK_SIZE 1024 | |
36 | ||
37 | #define BIT63 (((uint64_t)1 << 63)) | |
38 | ||
39 | #define INTEL_PT_RETURN 1 | |
40 | ||
41 | /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ | |
42 | #define INTEL_PT_MAX_LOOPS 10000 | |
43 | ||
44 | struct intel_pt_blk { | |
45 | struct intel_pt_blk *prev; | |
46 | uint64_t ip[INTEL_PT_BLK_SIZE]; | |
47 | }; | |
48 | ||
49 | struct intel_pt_stack { | |
50 | struct intel_pt_blk *blk; | |
51 | struct intel_pt_blk *spare; | |
52 | int pos; | |
53 | }; | |
54 | ||
55 | enum intel_pt_pkt_state { | |
56 | INTEL_PT_STATE_NO_PSB, | |
57 | INTEL_PT_STATE_NO_IP, | |
58 | INTEL_PT_STATE_ERR_RESYNC, | |
59 | INTEL_PT_STATE_IN_SYNC, | |
60 | INTEL_PT_STATE_TNT, | |
61 | INTEL_PT_STATE_TIP, | |
62 | INTEL_PT_STATE_TIP_PGD, | |
63 | INTEL_PT_STATE_FUP, | |
64 | INTEL_PT_STATE_FUP_NO_TIP, | |
65 | }; | |
66 | ||
3f04d98e AH |
67 | static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) |
68 | { | |
69 | switch (pkt_state) { | |
70 | case INTEL_PT_STATE_NO_PSB: | |
71 | case INTEL_PT_STATE_NO_IP: | |
72 | case INTEL_PT_STATE_ERR_RESYNC: | |
73 | case INTEL_PT_STATE_IN_SYNC: | |
74 | case INTEL_PT_STATE_TNT: | |
75 | return true; | |
76 | case INTEL_PT_STATE_TIP: | |
77 | case INTEL_PT_STATE_TIP_PGD: | |
78 | case INTEL_PT_STATE_FUP: | |
79 | case INTEL_PT_STATE_FUP_NO_TIP: | |
80 | return false; | |
81 | default: | |
82 | return true; | |
83 | }; | |
84 | } | |
85 | ||
f4aa0819 AH |
86 | #ifdef INTEL_PT_STRICT |
87 | #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB | |
88 | #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB | |
89 | #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB | |
90 | #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB | |
91 | #else | |
92 | #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) | |
93 | #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP | |
94 | #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC | |
95 | #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC | |
96 | #endif | |
97 | ||
98 | struct intel_pt_decoder { | |
99 | int (*get_trace)(struct intel_pt_buffer *buffer, void *data); | |
100 | int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, | |
101 | uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, | |
102 | uint64_t max_insn_cnt, void *data); | |
9f1d122b | 103 | bool (*pgd_ip)(uint64_t ip, void *data); |
f4aa0819 AH |
104 | void *data; |
105 | struct intel_pt_state state; | |
106 | const unsigned char *buf; | |
107 | size_t len; | |
108 | bool return_compression; | |
83959817 | 109 | bool branch_enable; |
79b58424 | 110 | bool mtc_insn; |
f4aa0819 | 111 | bool pge; |
79b58424 | 112 | bool have_tma; |
cc336186 | 113 | bool have_cyc; |
3bccbe20 | 114 | bool fixup_last_mtc; |
ee14ac0e | 115 | bool have_last_ip; |
f4aa0819 AH |
116 | uint64_t pos; |
117 | uint64_t last_ip; | |
118 | uint64_t ip; | |
119 | uint64_t cr3; | |
120 | uint64_t timestamp; | |
121 | uint64_t tsc_timestamp; | |
122 | uint64_t ref_timestamp; | |
3f04d98e | 123 | uint64_t sample_timestamp; |
f4aa0819 | 124 | uint64_t ret_addr; |
79b58424 AH |
125 | uint64_t ctc_timestamp; |
126 | uint64_t ctc_delta; | |
cc336186 AH |
127 | uint64_t cycle_cnt; |
128 | uint64_t cyc_ref_timestamp; | |
79b58424 AH |
129 | uint32_t last_mtc; |
130 | uint32_t tsc_ctc_ratio_n; | |
131 | uint32_t tsc_ctc_ratio_d; | |
132 | uint32_t tsc_ctc_mult; | |
133 | uint32_t tsc_slip; | |
134 | uint32_t ctc_rem_mask; | |
135 | int mtc_shift; | |
f4aa0819 AH |
136 | struct intel_pt_stack stack; |
137 | enum intel_pt_pkt_state pkt_state; | |
138 | struct intel_pt_pkt packet; | |
139 | struct intel_pt_pkt tnt; | |
140 | int pkt_step; | |
141 | int pkt_len; | |
cc336186 | 142 | int last_packet_type; |
f4aa0819 | 143 | unsigned int cbr; |
0a7c700d | 144 | unsigned int cbr_seen; |
f4aa0819 | 145 | unsigned int max_non_turbo_ratio; |
cc336186 AH |
146 | double max_non_turbo_ratio_fp; |
147 | double cbr_cyc_to_tsc; | |
148 | double calc_cyc_to_tsc; | |
149 | bool have_calc_cyc_to_tsc; | |
f4aa0819 AH |
150 | int exec_mode; |
151 | unsigned int insn_bytes; | |
f4aa0819 AH |
152 | uint64_t period; |
153 | enum intel_pt_period_type period_type; | |
2a21d036 | 154 | uint64_t tot_insn_cnt; |
f4aa0819 AH |
155 | uint64_t period_insn_cnt; |
156 | uint64_t period_mask; | |
157 | uint64_t period_ticks; | |
158 | uint64_t last_masked_timestamp; | |
159 | bool continuous_period; | |
160 | bool overflow; | |
161 | bool set_fup_tx_flags; | |
a472e65f AH |
162 | bool set_fup_ptw; |
163 | bool set_fup_mwait; | |
164 | bool set_fup_pwre; | |
165 | bool set_fup_exstop; | |
f4aa0819 AH |
166 | unsigned int fup_tx_flags; |
167 | unsigned int tx_flags; | |
a472e65f AH |
168 | uint64_t fup_ptw_payload; |
169 | uint64_t fup_mwait_payload; | |
170 | uint64_t fup_pwre_payload; | |
0a7c700d | 171 | uint64_t cbr_payload; |
f4aa0819 | 172 | uint64_t timestamp_insn_cnt; |
3f04d98e | 173 | uint64_t sample_insn_cnt; |
f4aa0819 AH |
174 | uint64_t stuck_ip; |
175 | int no_progress; | |
176 | int stuck_ip_prd; | |
177 | int stuck_ip_cnt; | |
178 | const unsigned char *next_buf; | |
179 | size_t next_len; | |
180 | unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; | |
181 | }; | |
182 | ||
183 | static uint64_t intel_pt_lower_power_of_2(uint64_t x) | |
184 | { | |
185 | int i; | |
186 | ||
187 | for (i = 0; x != 1; i++) | |
188 | x >>= 1; | |
189 | ||
190 | return x << i; | |
191 | } | |
192 | ||
193 | static void intel_pt_setup_period(struct intel_pt_decoder *decoder) | |
194 | { | |
195 | if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { | |
196 | uint64_t period; | |
197 | ||
198 | period = intel_pt_lower_power_of_2(decoder->period); | |
199 | decoder->period_mask = ~(period - 1); | |
200 | decoder->period_ticks = period; | |
201 | } | |
202 | } | |
203 | ||
79b58424 AH |
204 | static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) |
205 | { | |
206 | if (!d) | |
207 | return 0; | |
208 | return (t / d) * n + ((t % d) * n) / d; | |
209 | } | |
210 | ||
f4aa0819 AH |
211 | struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) |
212 | { | |
213 | struct intel_pt_decoder *decoder; | |
214 | ||
215 | if (!params->get_trace || !params->walk_insn) | |
216 | return NULL; | |
217 | ||
218 | decoder = zalloc(sizeof(struct intel_pt_decoder)); | |
219 | if (!decoder) | |
220 | return NULL; | |
221 | ||
222 | decoder->get_trace = params->get_trace; | |
223 | decoder->walk_insn = params->walk_insn; | |
9f1d122b | 224 | decoder->pgd_ip = params->pgd_ip; |
f4aa0819 AH |
225 | decoder->data = params->data; |
226 | decoder->return_compression = params->return_compression; | |
83959817 | 227 | decoder->branch_enable = params->branch_enable; |
f4aa0819 | 228 | |
f4aa0819 AH |
229 | decoder->period = params->period; |
230 | decoder->period_type = params->period_type; | |
231 | ||
cc336186 AH |
232 | decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; |
233 | decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; | |
f4aa0819 AH |
234 | |
235 | intel_pt_setup_period(decoder); | |
236 | ||
79b58424 AH |
237 | decoder->mtc_shift = params->mtc_period; |
238 | decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; | |
239 | ||
240 | decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; | |
241 | decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; | |
242 | ||
243 | if (!decoder->tsc_ctc_ratio_n) | |
244 | decoder->tsc_ctc_ratio_d = 0; | |
245 | ||
246 | if (decoder->tsc_ctc_ratio_d) { | |
247 | if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) | |
248 | decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / | |
249 | decoder->tsc_ctc_ratio_d; | |
250 | ||
251 | /* | |
252 | * Allow for timestamps appearing to backwards because a TSC | |
253 | * packet has slipped past a MTC packet, so allow 2 MTC ticks | |
254 | * or ... | |
255 | */ | |
256 | decoder->tsc_slip = multdiv(2 << decoder->mtc_shift, | |
257 | decoder->tsc_ctc_ratio_n, | |
258 | decoder->tsc_ctc_ratio_d); | |
259 | } | |
260 | /* ... or 0x100 paranoia */ | |
261 | if (decoder->tsc_slip < 0x100) | |
262 | decoder->tsc_slip = 0x100; | |
263 | ||
264 | intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); | |
265 | intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); | |
266 | intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); | |
267 | intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); | |
268 | intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); | |
269 | ||
f4aa0819 AH |
270 | return decoder; |
271 | } | |
272 | ||
273 | static void intel_pt_pop_blk(struct intel_pt_stack *stack) | |
274 | { | |
275 | struct intel_pt_blk *blk = stack->blk; | |
276 | ||
277 | stack->blk = blk->prev; | |
278 | if (!stack->spare) | |
279 | stack->spare = blk; | |
280 | else | |
281 | free(blk); | |
282 | } | |
283 | ||
284 | static uint64_t intel_pt_pop(struct intel_pt_stack *stack) | |
285 | { | |
286 | if (!stack->pos) { | |
287 | if (!stack->blk) | |
288 | return 0; | |
289 | intel_pt_pop_blk(stack); | |
290 | if (!stack->blk) | |
291 | return 0; | |
292 | stack->pos = INTEL_PT_BLK_SIZE; | |
293 | } | |
294 | return stack->blk->ip[--stack->pos]; | |
295 | } | |
296 | ||
297 | static int intel_pt_alloc_blk(struct intel_pt_stack *stack) | |
298 | { | |
299 | struct intel_pt_blk *blk; | |
300 | ||
301 | if (stack->spare) { | |
302 | blk = stack->spare; | |
303 | stack->spare = NULL; | |
304 | } else { | |
305 | blk = malloc(sizeof(struct intel_pt_blk)); | |
306 | if (!blk) | |
307 | return -ENOMEM; | |
308 | } | |
309 | ||
310 | blk->prev = stack->blk; | |
311 | stack->blk = blk; | |
312 | stack->pos = 0; | |
313 | return 0; | |
314 | } | |
315 | ||
316 | static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) | |
317 | { | |
318 | int err; | |
319 | ||
320 | if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { | |
321 | err = intel_pt_alloc_blk(stack); | |
322 | if (err) | |
323 | return err; | |
324 | } | |
325 | ||
326 | stack->blk->ip[stack->pos++] = ip; | |
327 | return 0; | |
328 | } | |
329 | ||
330 | static void intel_pt_clear_stack(struct intel_pt_stack *stack) | |
331 | { | |
332 | while (stack->blk) | |
333 | intel_pt_pop_blk(stack); | |
334 | stack->pos = 0; | |
335 | } | |
336 | ||
337 | static void intel_pt_free_stack(struct intel_pt_stack *stack) | |
338 | { | |
339 | intel_pt_clear_stack(stack); | |
340 | zfree(&stack->blk); | |
341 | zfree(&stack->spare); | |
342 | } | |
343 | ||
344 | void intel_pt_decoder_free(struct intel_pt_decoder *decoder) | |
345 | { | |
346 | intel_pt_free_stack(&decoder->stack); | |
347 | free(decoder); | |
348 | } | |
349 | ||
350 | static int intel_pt_ext_err(int code) | |
351 | { | |
352 | switch (code) { | |
353 | case -ENOMEM: | |
354 | return INTEL_PT_ERR_NOMEM; | |
355 | case -ENOSYS: | |
356 | return INTEL_PT_ERR_INTERN; | |
357 | case -EBADMSG: | |
358 | return INTEL_PT_ERR_BADPKT; | |
359 | case -ENODATA: | |
360 | return INTEL_PT_ERR_NODATA; | |
361 | case -EILSEQ: | |
362 | return INTEL_PT_ERR_NOINSN; | |
363 | case -ENOENT: | |
364 | return INTEL_PT_ERR_MISMAT; | |
365 | case -EOVERFLOW: | |
366 | return INTEL_PT_ERR_OVR; | |
367 | case -ENOSPC: | |
368 | return INTEL_PT_ERR_LOST; | |
369 | case -ELOOP: | |
370 | return INTEL_PT_ERR_NELOOP; | |
371 | default: | |
372 | return INTEL_PT_ERR_UNK; | |
373 | } | |
374 | } | |
375 | ||
376 | static const char *intel_pt_err_msgs[] = { | |
377 | [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", | |
378 | [INTEL_PT_ERR_INTERN] = "Internal error", | |
379 | [INTEL_PT_ERR_BADPKT] = "Bad packet", | |
380 | [INTEL_PT_ERR_NODATA] = "No more data", | |
381 | [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", | |
382 | [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", | |
383 | [INTEL_PT_ERR_OVR] = "Overflow packet", | |
384 | [INTEL_PT_ERR_LOST] = "Lost trace data", | |
385 | [INTEL_PT_ERR_UNK] = "Unknown error!", | |
386 | [INTEL_PT_ERR_NELOOP] = "Never-ending loop", | |
387 | }; | |
388 | ||
389 | int intel_pt__strerror(int code, char *buf, size_t buflen) | |
390 | { | |
c0664893 | 391 | if (code < 1 || code >= INTEL_PT_ERR_MAX) |
f4aa0819 AH |
392 | code = INTEL_PT_ERR_UNK; |
393 | strlcpy(buf, intel_pt_err_msgs[code], buflen); | |
394 | return 0; | |
395 | } | |
396 | ||
e1717e04 | 397 | static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, |
f4aa0819 AH |
398 | uint64_t last_ip) |
399 | { | |
400 | uint64_t ip; | |
401 | ||
402 | switch (packet->count) { | |
e1717e04 | 403 | case 1: |
f4aa0819 AH |
404 | ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | |
405 | packet->payload; | |
406 | break; | |
e1717e04 | 407 | case 2: |
f4aa0819 AH |
408 | ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | |
409 | packet->payload; | |
410 | break; | |
e1717e04 AH |
411 | case 3: |
412 | ip = packet->payload; | |
413 | /* Sign-extend 6-byte ip */ | |
414 | if (ip & (uint64_t)0x800000000000ULL) | |
415 | ip |= (uint64_t)0xffff000000000000ULL; | |
416 | break; | |
417 | case 4: | |
418 | ip = (last_ip & (uint64_t)0xffff000000000000ULL) | | |
419 | packet->payload; | |
420 | break; | |
f4aa0819 AH |
421 | case 6: |
422 | ip = packet->payload; | |
423 | break; | |
424 | default: | |
425 | return 0; | |
426 | } | |
427 | ||
f4aa0819 AH |
428 | return ip; |
429 | } | |
430 | ||
431 | static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) | |
432 | { | |
e1717e04 | 433 | decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); |
ee14ac0e | 434 | decoder->have_last_ip = true; |
f4aa0819 AH |
435 | } |
436 | ||
437 | static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) | |
438 | { | |
439 | intel_pt_set_last_ip(decoder); | |
440 | decoder->ip = decoder->last_ip; | |
441 | } | |
442 | ||
443 | static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) | |
444 | { | |
445 | intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, | |
446 | decoder->buf); | |
447 | } | |
448 | ||
449 | static int intel_pt_bug(struct intel_pt_decoder *decoder) | |
450 | { | |
451 | intel_pt_log("ERROR: Internal error\n"); | |
452 | decoder->pkt_state = INTEL_PT_STATE_NO_PSB; | |
453 | return -ENOSYS; | |
454 | } | |
455 | ||
456 | static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) | |
457 | { | |
458 | decoder->tx_flags = 0; | |
459 | } | |
460 | ||
461 | static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) | |
462 | { | |
463 | decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; | |
464 | } | |
465 | ||
466 | static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) | |
467 | { | |
468 | intel_pt_clear_tx_flags(decoder); | |
79b58424 | 469 | decoder->have_tma = false; |
f4aa0819 AH |
470 | decoder->pkt_len = 1; |
471 | decoder->pkt_step = 1; | |
472 | intel_pt_decoder_log_packet(decoder); | |
473 | if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { | |
474 | intel_pt_log("ERROR: Bad packet\n"); | |
475 | decoder->pkt_state = INTEL_PT_STATE_ERR1; | |
476 | } | |
477 | return -EBADMSG; | |
478 | } | |
479 | ||
480 | static int intel_pt_get_data(struct intel_pt_decoder *decoder) | |
481 | { | |
482 | struct intel_pt_buffer buffer = { .buf = 0, }; | |
483 | int ret; | |
484 | ||
485 | decoder->pkt_step = 0; | |
486 | ||
487 | intel_pt_log("Getting more data\n"); | |
488 | ret = decoder->get_trace(&buffer, decoder->data); | |
489 | if (ret) | |
490 | return ret; | |
491 | decoder->buf = buffer.buf; | |
492 | decoder->len = buffer.len; | |
493 | if (!decoder->len) { | |
494 | intel_pt_log("No more data\n"); | |
495 | return -ENODATA; | |
496 | } | |
497 | if (!buffer.consecutive) { | |
498 | decoder->ip = 0; | |
499 | decoder->pkt_state = INTEL_PT_STATE_NO_PSB; | |
500 | decoder->ref_timestamp = buffer.ref_timestamp; | |
501 | decoder->timestamp = 0; | |
79b58424 | 502 | decoder->have_tma = false; |
f4aa0819 AH |
503 | decoder->state.trace_nr = buffer.trace_nr; |
504 | intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", | |
505 | decoder->ref_timestamp); | |
506 | return -ENOLINK; | |
507 | } | |
508 | ||
509 | return 0; | |
510 | } | |
511 | ||
512 | static int intel_pt_get_next_data(struct intel_pt_decoder *decoder) | |
513 | { | |
514 | if (!decoder->next_buf) | |
515 | return intel_pt_get_data(decoder); | |
516 | ||
517 | decoder->buf = decoder->next_buf; | |
518 | decoder->len = decoder->next_len; | |
519 | decoder->next_buf = 0; | |
520 | decoder->next_len = 0; | |
521 | return 0; | |
522 | } | |
523 | ||
524 | static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) | |
525 | { | |
526 | unsigned char *buf = decoder->temp_buf; | |
527 | size_t old_len, len, n; | |
528 | int ret; | |
529 | ||
530 | old_len = decoder->len; | |
531 | len = decoder->len; | |
532 | memcpy(buf, decoder->buf, len); | |
533 | ||
534 | ret = intel_pt_get_data(decoder); | |
535 | if (ret) { | |
536 | decoder->pos += old_len; | |
537 | return ret < 0 ? ret : -EINVAL; | |
538 | } | |
539 | ||
540 | n = INTEL_PT_PKT_MAX_SZ - len; | |
541 | if (n > decoder->len) | |
542 | n = decoder->len; | |
543 | memcpy(buf + len, decoder->buf, n); | |
544 | len += n; | |
545 | ||
546 | ret = intel_pt_get_packet(buf, len, &decoder->packet); | |
547 | if (ret < (int)old_len) { | |
548 | decoder->next_buf = decoder->buf; | |
549 | decoder->next_len = decoder->len; | |
550 | decoder->buf = buf; | |
551 | decoder->len = old_len; | |
552 | return intel_pt_bad_packet(decoder); | |
553 | } | |
554 | ||
555 | decoder->next_buf = decoder->buf + (ret - old_len); | |
556 | decoder->next_len = decoder->len - (ret - old_len); | |
557 | ||
558 | decoder->buf = buf; | |
559 | decoder->len = ret; | |
560 | ||
561 | return ret; | |
562 | } | |
563 | ||
cc336186 AH |
564 | struct intel_pt_pkt_info { |
565 | struct intel_pt_decoder *decoder; | |
566 | struct intel_pt_pkt packet; | |
567 | uint64_t pos; | |
568 | int pkt_len; | |
569 | int last_packet_type; | |
570 | void *data; | |
571 | }; | |
572 | ||
573 | typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); | |
574 | ||
575 | /* Lookahead packets in current buffer */ | |
576 | static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, | |
577 | intel_pt_pkt_cb_t cb, void *data) | |
578 | { | |
579 | struct intel_pt_pkt_info pkt_info; | |
580 | const unsigned char *buf = decoder->buf; | |
581 | size_t len = decoder->len; | |
582 | int ret; | |
583 | ||
584 | pkt_info.decoder = decoder; | |
585 | pkt_info.pos = decoder->pos; | |
586 | pkt_info.pkt_len = decoder->pkt_step; | |
587 | pkt_info.last_packet_type = decoder->last_packet_type; | |
588 | pkt_info.data = data; | |
589 | ||
590 | while (1) { | |
591 | do { | |
592 | pkt_info.pos += pkt_info.pkt_len; | |
593 | buf += pkt_info.pkt_len; | |
594 | len -= pkt_info.pkt_len; | |
595 | ||
596 | if (!len) | |
597 | return INTEL_PT_NEED_MORE_BYTES; | |
598 | ||
599 | ret = intel_pt_get_packet(buf, len, &pkt_info.packet); | |
600 | if (!ret) | |
601 | return INTEL_PT_NEED_MORE_BYTES; | |
602 | if (ret < 0) | |
603 | return ret; | |
604 | ||
605 | pkt_info.pkt_len = ret; | |
606 | } while (pkt_info.packet.type == INTEL_PT_PAD); | |
607 | ||
608 | ret = cb(&pkt_info); | |
609 | if (ret) | |
610 | return 0; | |
611 | ||
612 | pkt_info.last_packet_type = pkt_info.packet.type; | |
613 | } | |
614 | } | |
615 | ||
616 | struct intel_pt_calc_cyc_to_tsc_info { | |
617 | uint64_t cycle_cnt; | |
618 | unsigned int cbr; | |
619 | uint32_t last_mtc; | |
620 | uint64_t ctc_timestamp; | |
621 | uint64_t ctc_delta; | |
622 | uint64_t tsc_timestamp; | |
623 | uint64_t timestamp; | |
624 | bool have_tma; | |
3bccbe20 | 625 | bool fixup_last_mtc; |
cc336186 AH |
626 | bool from_mtc; |
627 | double cbr_cyc_to_tsc; | |
628 | }; | |
629 | ||
3bccbe20 AH |
630 | /* |
631 | * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower | |
632 | * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC | |
633 | * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA | |
634 | * packet by copying the missing bits from the current MTC assuming the least | |
635 | * difference between the two, and that the current MTC comes after last_mtc. | |
636 | */ | |
637 | static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, | |
638 | uint32_t *last_mtc) | |
639 | { | |
640 | uint32_t first_missing_bit = 1U << (16 - mtc_shift); | |
641 | uint32_t mask = ~(first_missing_bit - 1); | |
642 | ||
643 | *last_mtc |= mtc & mask; | |
644 | if (*last_mtc >= mtc) { | |
645 | *last_mtc -= first_missing_bit; | |
646 | *last_mtc &= 0xff; | |
647 | } | |
648 | } | |
649 | ||
cc336186 AH |
650 | static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) |
651 | { | |
652 | struct intel_pt_decoder *decoder = pkt_info->decoder; | |
653 | struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; | |
654 | uint64_t timestamp; | |
655 | double cyc_to_tsc; | |
656 | unsigned int cbr; | |
657 | uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; | |
658 | ||
659 | switch (pkt_info->packet.type) { | |
660 | case INTEL_PT_TNT: | |
661 | case INTEL_PT_TIP_PGE: | |
662 | case INTEL_PT_TIP: | |
663 | case INTEL_PT_FUP: | |
664 | case INTEL_PT_PSB: | |
665 | case INTEL_PT_PIP: | |
666 | case INTEL_PT_MODE_EXEC: | |
667 | case INTEL_PT_MODE_TSX: | |
668 | case INTEL_PT_PSBEND: | |
669 | case INTEL_PT_PAD: | |
670 | case INTEL_PT_VMCS: | |
671 | case INTEL_PT_MNT: | |
a472e65f AH |
672 | case INTEL_PT_PTWRITE: |
673 | case INTEL_PT_PTWRITE_IP: | |
cc336186 AH |
674 | return 0; |
675 | ||
676 | case INTEL_PT_MTC: | |
677 | if (!data->have_tma) | |
678 | return 0; | |
679 | ||
680 | mtc = pkt_info->packet.payload; | |
3bccbe20 AH |
681 | if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { |
682 | data->fixup_last_mtc = false; | |
683 | intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, | |
684 | &data->last_mtc); | |
685 | } | |
cc336186 AH |
686 | if (mtc > data->last_mtc) |
687 | mtc_delta = mtc - data->last_mtc; | |
688 | else | |
689 | mtc_delta = mtc + 256 - data->last_mtc; | |
690 | data->ctc_delta += mtc_delta << decoder->mtc_shift; | |
691 | data->last_mtc = mtc; | |
692 | ||
693 | if (decoder->tsc_ctc_mult) { | |
694 | timestamp = data->ctc_timestamp + | |
695 | data->ctc_delta * decoder->tsc_ctc_mult; | |
696 | } else { | |
697 | timestamp = data->ctc_timestamp + | |
698 | multdiv(data->ctc_delta, | |
699 | decoder->tsc_ctc_ratio_n, | |
700 | decoder->tsc_ctc_ratio_d); | |
701 | } | |
702 | ||
703 | if (timestamp < data->timestamp) | |
704 | return 1; | |
705 | ||
706 | if (pkt_info->last_packet_type != INTEL_PT_CYC) { | |
707 | data->timestamp = timestamp; | |
708 | return 0; | |
709 | } | |
710 | ||
711 | break; | |
712 | ||
713 | case INTEL_PT_TSC: | |
38b65b08 AH |
714 | /* |
715 | * For now, do not support using TSC packets - refer | |
716 | * intel_pt_calc_cyc_to_tsc(). | |
717 | */ | |
718 | if (data->from_mtc) | |
719 | return 1; | |
cc336186 AH |
720 | timestamp = pkt_info->packet.payload | |
721 | (data->timestamp & (0xffULL << 56)); | |
722 | if (data->from_mtc && timestamp < data->timestamp && | |
723 | data->timestamp - timestamp < decoder->tsc_slip) | |
724 | return 1; | |
9992c2d5 | 725 | if (timestamp < data->timestamp) |
cc336186 AH |
726 | timestamp += (1ULL << 56); |
727 | if (pkt_info->last_packet_type != INTEL_PT_CYC) { | |
728 | if (data->from_mtc) | |
729 | return 1; | |
730 | data->tsc_timestamp = timestamp; | |
731 | data->timestamp = timestamp; | |
732 | return 0; | |
733 | } | |
734 | break; | |
735 | ||
736 | case INTEL_PT_TMA: | |
737 | if (data->from_mtc) | |
738 | return 1; | |
739 | ||
740 | if (!decoder->tsc_ctc_ratio_d) | |
741 | return 0; | |
742 | ||
743 | ctc = pkt_info->packet.payload; | |
744 | fc = pkt_info->packet.count; | |
745 | ctc_rem = ctc & decoder->ctc_rem_mask; | |
746 | ||
747 | data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; | |
748 | ||
749 | data->ctc_timestamp = data->tsc_timestamp - fc; | |
750 | if (decoder->tsc_ctc_mult) { | |
751 | data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; | |
752 | } else { | |
753 | data->ctc_timestamp -= | |
754 | multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, | |
755 | decoder->tsc_ctc_ratio_d); | |
756 | } | |
757 | ||
758 | data->ctc_delta = 0; | |
759 | data->have_tma = true; | |
3bccbe20 | 760 | data->fixup_last_mtc = true; |
cc336186 AH |
761 | |
762 | return 0; | |
763 | ||
764 | case INTEL_PT_CYC: | |
765 | data->cycle_cnt += pkt_info->packet.payload; | |
766 | return 0; | |
767 | ||
768 | case INTEL_PT_CBR: | |
769 | cbr = pkt_info->packet.payload; | |
770 | if (data->cbr && data->cbr != cbr) | |
771 | return 1; | |
772 | data->cbr = cbr; | |
773 | data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; | |
774 | return 0; | |
775 | ||
776 | case INTEL_PT_TIP_PGD: | |
777 | case INTEL_PT_TRACESTOP: | |
a472e65f AH |
778 | case INTEL_PT_EXSTOP: |
779 | case INTEL_PT_EXSTOP_IP: | |
780 | case INTEL_PT_MWAIT: | |
781 | case INTEL_PT_PWRE: | |
782 | case INTEL_PT_PWRX: | |
cc336186 AH |
783 | case INTEL_PT_OVF: |
784 | case INTEL_PT_BAD: /* Does not happen */ | |
785 | default: | |
786 | return 1; | |
787 | } | |
788 | ||
789 | if (!data->cbr && decoder->cbr) { | |
790 | data->cbr = decoder->cbr; | |
791 | data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; | |
792 | } | |
793 | ||
794 | if (!data->cycle_cnt) | |
795 | return 1; | |
796 | ||
797 | cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; | |
798 | ||
799 | if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && | |
800 | cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { | |
801 | intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", | |
802 | cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); | |
803 | return 1; | |
804 | } | |
805 | ||
806 | decoder->calc_cyc_to_tsc = cyc_to_tsc; | |
807 | decoder->have_calc_cyc_to_tsc = true; | |
808 | ||
809 | if (data->cbr) { | |
810 | intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", | |
811 | cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); | |
812 | } else { | |
813 | intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", | |
814 | cyc_to_tsc, pkt_info->pos); | |
815 | } | |
816 | ||
817 | return 1; | |
818 | } | |
819 | ||
820 | static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, | |
821 | bool from_mtc) | |
822 | { | |
823 | struct intel_pt_calc_cyc_to_tsc_info data = { | |
824 | .cycle_cnt = 0, | |
825 | .cbr = 0, | |
826 | .last_mtc = decoder->last_mtc, | |
827 | .ctc_timestamp = decoder->ctc_timestamp, | |
828 | .ctc_delta = decoder->ctc_delta, | |
829 | .tsc_timestamp = decoder->tsc_timestamp, | |
830 | .timestamp = decoder->timestamp, | |
831 | .have_tma = decoder->have_tma, | |
3bccbe20 | 832 | .fixup_last_mtc = decoder->fixup_last_mtc, |
cc336186 AH |
833 | .from_mtc = from_mtc, |
834 | .cbr_cyc_to_tsc = 0, | |
835 | }; | |
836 | ||
38b65b08 AH |
837 | /* |
838 | * For now, do not support using TSC packets for at least the reasons: | |
839 | * 1) timing might have stopped | |
840 | * 2) TSC packets within PSB+ can slip against CYC packets | |
841 | */ | |
842 | if (!from_mtc) | |
843 | return; | |
844 | ||
cc336186 AH |
845 | intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); |
846 | } | |
847 | ||
f4aa0819 AH |
848 | static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) |
849 | { | |
850 | int ret; | |
851 | ||
cc336186 AH |
852 | decoder->last_packet_type = decoder->packet.type; |
853 | ||
f4aa0819 AH |
854 | do { |
855 | decoder->pos += decoder->pkt_step; | |
856 | decoder->buf += decoder->pkt_step; | |
857 | decoder->len -= decoder->pkt_step; | |
858 | ||
859 | if (!decoder->len) { | |
860 | ret = intel_pt_get_next_data(decoder); | |
861 | if (ret) | |
862 | return ret; | |
863 | } | |
864 | ||
865 | ret = intel_pt_get_packet(decoder->buf, decoder->len, | |
866 | &decoder->packet); | |
867 | if (ret == INTEL_PT_NEED_MORE_BYTES && | |
868 | decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { | |
869 | ret = intel_pt_get_split_packet(decoder); | |
870 | if (ret < 0) | |
871 | return ret; | |
872 | } | |
873 | if (ret <= 0) | |
874 | return intel_pt_bad_packet(decoder); | |
875 | ||
876 | decoder->pkt_len = ret; | |
877 | decoder->pkt_step = ret; | |
878 | intel_pt_decoder_log_packet(decoder); | |
879 | } while (decoder->packet.type == INTEL_PT_PAD); | |
880 | ||
881 | return 0; | |
882 | } | |
883 | ||
884 | static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) | |
885 | { | |
886 | uint64_t timestamp, masked_timestamp; | |
887 | ||
888 | timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; | |
889 | masked_timestamp = timestamp & decoder->period_mask; | |
890 | if (decoder->continuous_period) { | |
891 | if (masked_timestamp != decoder->last_masked_timestamp) | |
892 | return 1; | |
893 | } else { | |
894 | timestamp += 1; | |
895 | masked_timestamp = timestamp & decoder->period_mask; | |
896 | if (masked_timestamp != decoder->last_masked_timestamp) { | |
897 | decoder->last_masked_timestamp = masked_timestamp; | |
898 | decoder->continuous_period = true; | |
899 | } | |
900 | } | |
901 | return decoder->period_ticks - (timestamp - masked_timestamp); | |
902 | } | |
903 | ||
904 | static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) | |
905 | { | |
906 | switch (decoder->period_type) { | |
907 | case INTEL_PT_PERIOD_INSTRUCTIONS: | |
908 | return decoder->period - decoder->period_insn_cnt; | |
909 | case INTEL_PT_PERIOD_TICKS: | |
910 | return intel_pt_next_period(decoder); | |
911 | case INTEL_PT_PERIOD_NONE: | |
79b58424 | 912 | case INTEL_PT_PERIOD_MTC: |
f4aa0819 AH |
913 | default: |
914 | return 0; | |
915 | } | |
916 | } | |
917 | ||
918 | static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) | |
919 | { | |
920 | uint64_t timestamp, masked_timestamp; | |
921 | ||
922 | switch (decoder->period_type) { | |
923 | case INTEL_PT_PERIOD_INSTRUCTIONS: | |
924 | decoder->period_insn_cnt = 0; | |
925 | break; | |
926 | case INTEL_PT_PERIOD_TICKS: | |
927 | timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; | |
928 | masked_timestamp = timestamp & decoder->period_mask; | |
929 | decoder->last_masked_timestamp = masked_timestamp; | |
930 | break; | |
931 | case INTEL_PT_PERIOD_NONE: | |
79b58424 | 932 | case INTEL_PT_PERIOD_MTC: |
f4aa0819 AH |
933 | default: |
934 | break; | |
935 | } | |
936 | ||
937 | decoder->state.type |= INTEL_PT_INSTRUCTION; | |
938 | } | |
939 | ||
940 | static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, | |
941 | struct intel_pt_insn *intel_pt_insn, uint64_t ip) | |
942 | { | |
943 | uint64_t max_insn_cnt, insn_cnt = 0; | |
944 | int err; | |
945 | ||
79b58424 AH |
946 | if (!decoder->mtc_insn) |
947 | decoder->mtc_insn = true; | |
948 | ||
f4aa0819 AH |
949 | max_insn_cnt = intel_pt_next_sample(decoder); |
950 | ||
951 | err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, | |
952 | max_insn_cnt, decoder->data); | |
953 | ||
2a21d036 | 954 | decoder->tot_insn_cnt += insn_cnt; |
f4aa0819 | 955 | decoder->timestamp_insn_cnt += insn_cnt; |
3f04d98e | 956 | decoder->sample_insn_cnt += insn_cnt; |
f4aa0819 AH |
957 | decoder->period_insn_cnt += insn_cnt; |
958 | ||
959 | if (err) { | |
960 | decoder->no_progress = 0; | |
961 | decoder->pkt_state = INTEL_PT_STATE_ERR2; | |
962 | intel_pt_log_at("ERROR: Failed to get instruction", | |
963 | decoder->ip); | |
964 | if (err == -ENOENT) | |
965 | return -ENOLINK; | |
966 | return -EILSEQ; | |
967 | } | |
968 | ||
969 | if (ip && decoder->ip == ip) { | |
970 | err = -EAGAIN; | |
971 | goto out; | |
972 | } | |
973 | ||
974 | if (max_insn_cnt && insn_cnt >= max_insn_cnt) | |
975 | intel_pt_sample_insn(decoder); | |
976 | ||
977 | if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { | |
978 | decoder->state.type = INTEL_PT_INSTRUCTION; | |
979 | decoder->state.from_ip = decoder->ip; | |
980 | decoder->state.to_ip = 0; | |
981 | decoder->ip += intel_pt_insn->length; | |
982 | err = INTEL_PT_RETURN; | |
983 | goto out; | |
984 | } | |
985 | ||
986 | if (intel_pt_insn->op == INTEL_PT_OP_CALL) { | |
987 | /* Zero-length calls are excluded */ | |
988 | if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || | |
989 | intel_pt_insn->rel) { | |
990 | err = intel_pt_push(&decoder->stack, decoder->ip + | |
991 | intel_pt_insn->length); | |
992 | if (err) | |
993 | goto out; | |
994 | } | |
995 | } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { | |
996 | decoder->ret_addr = intel_pt_pop(&decoder->stack); | |
997 | } | |
998 | ||
999 | if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { | |
1000 | int cnt = decoder->no_progress++; | |
1001 | ||
1002 | decoder->state.from_ip = decoder->ip; | |
1003 | decoder->ip += intel_pt_insn->length + | |
1004 | intel_pt_insn->rel; | |
1005 | decoder->state.to_ip = decoder->ip; | |
1006 | err = INTEL_PT_RETURN; | |
1007 | ||
1008 | /* | |
1009 | * Check for being stuck in a loop. This can happen if a | |
1010 | * decoder error results in the decoder erroneously setting the | |
1011 | * ip to an address that is itself in an infinite loop that | |
1012 | * consumes no packets. When that happens, there must be an | |
1013 | * unconditional branch. | |
1014 | */ | |
1015 | if (cnt) { | |
1016 | if (cnt == 1) { | |
1017 | decoder->stuck_ip = decoder->state.to_ip; | |
1018 | decoder->stuck_ip_prd = 1; | |
1019 | decoder->stuck_ip_cnt = 1; | |
1020 | } else if (cnt > INTEL_PT_MAX_LOOPS || | |
1021 | decoder->state.to_ip == decoder->stuck_ip) { | |
1022 | intel_pt_log_at("ERROR: Never-ending loop", | |
1023 | decoder->state.to_ip); | |
1024 | decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; | |
1025 | err = -ELOOP; | |
1026 | goto out; | |
1027 | } else if (!--decoder->stuck_ip_cnt) { | |
1028 | decoder->stuck_ip_prd += 1; | |
1029 | decoder->stuck_ip_cnt = decoder->stuck_ip_prd; | |
1030 | decoder->stuck_ip = decoder->state.to_ip; | |
1031 | } | |
1032 | } | |
1033 | goto out_no_progress; | |
1034 | } | |
1035 | out: | |
1036 | decoder->no_progress = 0; | |
1037 | out_no_progress: | |
1038 | decoder->state.insn_op = intel_pt_insn->op; | |
1039 | decoder->state.insn_len = intel_pt_insn->length; | |
faaa8768 AK |
1040 | memcpy(decoder->state.insn, intel_pt_insn->buf, |
1041 | INTEL_PT_INSN_BUF_SZ); | |
f4aa0819 AH |
1042 | |
1043 | if (decoder->tx_flags & INTEL_PT_IN_TX) | |
1044 | decoder->state.flags |= INTEL_PT_IN_TX; | |
1045 | ||
1046 | return err; | |
1047 | } | |
1048 | ||
a472e65f AH |
1049 | static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) |
1050 | { | |
1051 | bool ret = false; | |
1052 | ||
1053 | if (decoder->set_fup_tx_flags) { | |
1054 | decoder->set_fup_tx_flags = false; | |
1055 | decoder->tx_flags = decoder->fup_tx_flags; | |
1056 | decoder->state.type = INTEL_PT_TRANSACTION; | |
1057 | decoder->state.from_ip = decoder->ip; | |
1058 | decoder->state.to_ip = 0; | |
1059 | decoder->state.flags = decoder->fup_tx_flags; | |
1060 | return true; | |
1061 | } | |
1062 | if (decoder->set_fup_ptw) { | |
1063 | decoder->set_fup_ptw = false; | |
1064 | decoder->state.type = INTEL_PT_PTW; | |
1065 | decoder->state.flags |= INTEL_PT_FUP_IP; | |
1066 | decoder->state.from_ip = decoder->ip; | |
1067 | decoder->state.to_ip = 0; | |
1068 | decoder->state.ptw_payload = decoder->fup_ptw_payload; | |
1069 | return true; | |
1070 | } | |
1071 | if (decoder->set_fup_mwait) { | |
1072 | decoder->set_fup_mwait = false; | |
1073 | decoder->state.type = INTEL_PT_MWAIT_OP; | |
1074 | decoder->state.from_ip = decoder->ip; | |
1075 | decoder->state.to_ip = 0; | |
1076 | decoder->state.mwait_payload = decoder->fup_mwait_payload; | |
1077 | ret = true; | |
1078 | } | |
1079 | if (decoder->set_fup_pwre) { | |
1080 | decoder->set_fup_pwre = false; | |
1081 | decoder->state.type |= INTEL_PT_PWR_ENTRY; | |
1082 | decoder->state.type &= ~INTEL_PT_BRANCH; | |
1083 | decoder->state.from_ip = decoder->ip; | |
1084 | decoder->state.to_ip = 0; | |
1085 | decoder->state.pwre_payload = decoder->fup_pwre_payload; | |
1086 | ret = true; | |
1087 | } | |
1088 | if (decoder->set_fup_exstop) { | |
1089 | decoder->set_fup_exstop = false; | |
1090 | decoder->state.type |= INTEL_PT_EX_STOP; | |
1091 | decoder->state.type &= ~INTEL_PT_BRANCH; | |
1092 | decoder->state.flags |= INTEL_PT_FUP_IP; | |
1093 | decoder->state.from_ip = decoder->ip; | |
1094 | decoder->state.to_ip = 0; | |
1095 | ret = true; | |
1096 | } | |
1097 | return ret; | |
1098 | } | |
1099 | ||
f4aa0819 AH |
1100 | static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) |
1101 | { | |
1102 | struct intel_pt_insn intel_pt_insn; | |
1103 | uint64_t ip; | |
1104 | int err; | |
1105 | ||
1106 | ip = decoder->last_ip; | |
1107 | ||
1108 | while (1) { | |
1109 | err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); | |
1110 | if (err == INTEL_PT_RETURN) | |
1111 | return 0; | |
1112 | if (err == -EAGAIN) { | |
a472e65f | 1113 | if (intel_pt_fup_event(decoder)) |
f4aa0819 | 1114 | return 0; |
f4aa0819 AH |
1115 | return err; |
1116 | } | |
1117 | decoder->set_fup_tx_flags = false; | |
1118 | if (err) | |
1119 | return err; | |
1120 | ||
1121 | if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { | |
1122 | intel_pt_log_at("ERROR: Unexpected indirect branch", | |
1123 | decoder->ip); | |
1124 | decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; | |
1125 | return -ENOENT; | |
1126 | } | |
1127 | ||
1128 | if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { | |
1129 | intel_pt_log_at("ERROR: Unexpected conditional branch", | |
1130 | decoder->ip); | |
1131 | decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; | |
1132 | return -ENOENT; | |
1133 | } | |
1134 | ||
1135 | intel_pt_bug(decoder); | |
1136 | } | |
1137 | } | |
1138 | ||
1139 | static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) | |
1140 | { | |
1141 | struct intel_pt_insn intel_pt_insn; | |
1142 | int err; | |
1143 | ||
1144 | err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); | |
9f1d122b AH |
1145 | if (err == INTEL_PT_RETURN && |
1146 | decoder->pgd_ip && | |
1147 | decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && | |
1148 | (decoder->state.type & INTEL_PT_BRANCH) && | |
1149 | decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { | |
1150 | /* Unconditional branch leaving filter region */ | |
1151 | decoder->no_progress = 0; | |
1152 | decoder->pge = false; | |
1153 | decoder->continuous_period = false; | |
1154 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
1155 | decoder->state.to_ip = 0; | |
1156 | return 0; | |
1157 | } | |
f4aa0819 AH |
1158 | if (err == INTEL_PT_RETURN) |
1159 | return 0; | |
1160 | if (err) | |
1161 | return err; | |
1162 | ||
1163 | if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { | |
1164 | if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { | |
1165 | decoder->pge = false; | |
1166 | decoder->continuous_period = false; | |
1167 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
1168 | decoder->state.from_ip = decoder->ip; | |
1169 | decoder->state.to_ip = 0; | |
1170 | if (decoder->packet.count != 0) | |
1171 | decoder->ip = decoder->last_ip; | |
1172 | } else { | |
1173 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
1174 | decoder->state.from_ip = decoder->ip; | |
1175 | if (decoder->packet.count == 0) { | |
1176 | decoder->state.to_ip = 0; | |
1177 | } else { | |
1178 | decoder->state.to_ip = decoder->last_ip; | |
1179 | decoder->ip = decoder->last_ip; | |
1180 | } | |
1181 | } | |
1182 | return 0; | |
1183 | } | |
1184 | ||
1185 | if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { | |
9f1d122b AH |
1186 | uint64_t to_ip = decoder->ip + intel_pt_insn.length + |
1187 | intel_pt_insn.rel; | |
1188 | ||
1189 | if (decoder->pgd_ip && | |
1190 | decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && | |
1191 | decoder->pgd_ip(to_ip, decoder->data)) { | |
1192 | /* Conditional branch leaving filter region */ | |
1193 | decoder->pge = false; | |
1194 | decoder->continuous_period = false; | |
1195 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
1196 | decoder->ip = to_ip; | |
1197 | decoder->state.from_ip = decoder->ip; | |
1198 | decoder->state.to_ip = 0; | |
1199 | return 0; | |
1200 | } | |
f4aa0819 AH |
1201 | intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", |
1202 | decoder->ip); | |
1203 | decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; | |
1204 | return -ENOENT; | |
1205 | } | |
1206 | ||
1207 | return intel_pt_bug(decoder); | |
1208 | } | |
1209 | ||
1210 | static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) | |
1211 | { | |
1212 | struct intel_pt_insn intel_pt_insn; | |
1213 | int err; | |
1214 | ||
1215 | while (1) { | |
1216 | err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); | |
1217 | if (err == INTEL_PT_RETURN) | |
1218 | return 0; | |
1219 | if (err) | |
1220 | return err; | |
1221 | ||
1222 | if (intel_pt_insn.op == INTEL_PT_OP_RET) { | |
1223 | if (!decoder->return_compression) { | |
1224 | intel_pt_log_at("ERROR: RET when expecting conditional branch", | |
1225 | decoder->ip); | |
1226 | decoder->pkt_state = INTEL_PT_STATE_ERR3; | |
1227 | return -ENOENT; | |
1228 | } | |
1229 | if (!decoder->ret_addr) { | |
1230 | intel_pt_log_at("ERROR: Bad RET compression (stack empty)", | |
1231 | decoder->ip); | |
1232 | decoder->pkt_state = INTEL_PT_STATE_ERR3; | |
1233 | return -ENOENT; | |
1234 | } | |
1235 | if (!(decoder->tnt.payload & BIT63)) { | |
1236 | intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", | |
1237 | decoder->ip); | |
1238 | decoder->pkt_state = INTEL_PT_STATE_ERR3; | |
1239 | return -ENOENT; | |
1240 | } | |
1241 | decoder->tnt.count -= 1; | |
1242 | if (!decoder->tnt.count) | |
1243 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
1244 | decoder->tnt.payload <<= 1; | |
1245 | decoder->state.from_ip = decoder->ip; | |
1246 | decoder->ip = decoder->ret_addr; | |
1247 | decoder->state.to_ip = decoder->ip; | |
1248 | return 0; | |
1249 | } | |
1250 | ||
1251 | if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { | |
1252 | /* Handle deferred TIPs */ | |
1253 | err = intel_pt_get_next_packet(decoder); | |
1254 | if (err) | |
1255 | return err; | |
1256 | if (decoder->packet.type != INTEL_PT_TIP || | |
1257 | decoder->packet.count == 0) { | |
1258 | intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", | |
1259 | decoder->ip); | |
1260 | decoder->pkt_state = INTEL_PT_STATE_ERR3; | |
1261 | decoder->pkt_step = 0; | |
1262 | return -ENOENT; | |
1263 | } | |
1264 | intel_pt_set_last_ip(decoder); | |
1265 | decoder->state.from_ip = decoder->ip; | |
1266 | decoder->state.to_ip = decoder->last_ip; | |
1267 | decoder->ip = decoder->last_ip; | |
1268 | return 0; | |
1269 | } | |
1270 | ||
1271 | if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { | |
1272 | decoder->tnt.count -= 1; | |
1273 | if (!decoder->tnt.count) | |
1274 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
1275 | if (decoder->tnt.payload & BIT63) { | |
1276 | decoder->tnt.payload <<= 1; | |
1277 | decoder->state.from_ip = decoder->ip; | |
1278 | decoder->ip += intel_pt_insn.length + | |
1279 | intel_pt_insn.rel; | |
1280 | decoder->state.to_ip = decoder->ip; | |
1281 | return 0; | |
1282 | } | |
1283 | /* Instruction sample for a non-taken branch */ | |
1284 | if (decoder->state.type & INTEL_PT_INSTRUCTION) { | |
1285 | decoder->tnt.payload <<= 1; | |
1286 | decoder->state.type = INTEL_PT_INSTRUCTION; | |
1287 | decoder->state.from_ip = decoder->ip; | |
1288 | decoder->state.to_ip = 0; | |
1289 | decoder->ip += intel_pt_insn.length; | |
1290 | return 0; | |
1291 | } | |
1292 | decoder->ip += intel_pt_insn.length; | |
1293 | if (!decoder->tnt.count) | |
1294 | return -EAGAIN; | |
1295 | decoder->tnt.payload <<= 1; | |
1296 | continue; | |
1297 | } | |
1298 | ||
1299 | return intel_pt_bug(decoder); | |
1300 | } | |
1301 | } | |
1302 | ||
1303 | static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) | |
1304 | { | |
1305 | unsigned int fup_tx_flags; | |
1306 | int err; | |
1307 | ||
1308 | fup_tx_flags = decoder->packet.payload & | |
1309 | (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); | |
1310 | err = intel_pt_get_next_packet(decoder); | |
1311 | if (err) | |
1312 | return err; | |
1313 | if (decoder->packet.type == INTEL_PT_FUP) { | |
1314 | decoder->fup_tx_flags = fup_tx_flags; | |
1315 | decoder->set_fup_tx_flags = true; | |
1316 | if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) | |
1317 | *no_tip = true; | |
1318 | } else { | |
1319 | intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", | |
1320 | decoder->pos); | |
1321 | intel_pt_update_in_tx(decoder); | |
1322 | } | |
1323 | return 0; | |
1324 | } | |
1325 | ||
1326 | static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) | |
1327 | { | |
1328 | uint64_t timestamp; | |
1329 | ||
79b58424 AH |
1330 | decoder->have_tma = false; |
1331 | ||
f4aa0819 AH |
1332 | if (decoder->ref_timestamp) { |
1333 | timestamp = decoder->packet.payload | | |
1334 | (decoder->ref_timestamp & (0xffULL << 56)); | |
1335 | if (timestamp < decoder->ref_timestamp) { | |
1336 | if (decoder->ref_timestamp - timestamp > (1ULL << 55)) | |
1337 | timestamp += (1ULL << 56); | |
1338 | } else { | |
1339 | if (timestamp - decoder->ref_timestamp > (1ULL << 55)) | |
1340 | timestamp -= (1ULL << 56); | |
1341 | } | |
1342 | decoder->tsc_timestamp = timestamp; | |
1343 | decoder->timestamp = timestamp; | |
1344 | decoder->ref_timestamp = 0; | |
1345 | decoder->timestamp_insn_cnt = 0; | |
1346 | } else if (decoder->timestamp) { | |
1347 | timestamp = decoder->packet.payload | | |
1348 | (decoder->timestamp & (0xffULL << 56)); | |
79b58424 | 1349 | decoder->tsc_timestamp = timestamp; |
f4aa0819 | 1350 | if (timestamp < decoder->timestamp && |
79b58424 AH |
1351 | decoder->timestamp - timestamp < decoder->tsc_slip) { |
1352 | intel_pt_log_to("Suppressing backwards timestamp", | |
f4aa0819 AH |
1353 | timestamp); |
1354 | timestamp = decoder->timestamp; | |
1355 | } | |
9992c2d5 | 1356 | if (timestamp < decoder->timestamp) { |
f4aa0819 AH |
1357 | intel_pt_log_to("Wraparound timestamp", timestamp); |
1358 | timestamp += (1ULL << 56); | |
79b58424 | 1359 | decoder->tsc_timestamp = timestamp; |
f4aa0819 | 1360 | } |
f4aa0819 AH |
1361 | decoder->timestamp = timestamp; |
1362 | decoder->timestamp_insn_cnt = 0; | |
1363 | } | |
1364 | ||
cc336186 AH |
1365 | if (decoder->last_packet_type == INTEL_PT_CYC) { |
1366 | decoder->cyc_ref_timestamp = decoder->timestamp; | |
1367 | decoder->cycle_cnt = 0; | |
1368 | decoder->have_calc_cyc_to_tsc = false; | |
1369 | intel_pt_calc_cyc_to_tsc(decoder, false); | |
1370 | } | |
1371 | ||
f4aa0819 AH |
1372 | intel_pt_log_to("Setting timestamp", decoder->timestamp); |
1373 | } | |
1374 | ||
1375 | static int intel_pt_overflow(struct intel_pt_decoder *decoder) | |
1376 | { | |
1377 | intel_pt_log("ERROR: Buffer overflow\n"); | |
1378 | intel_pt_clear_tx_flags(decoder); | |
79b58424 | 1379 | decoder->have_tma = false; |
cc336186 | 1380 | decoder->cbr = 0; |
f4aa0819 AH |
1381 | decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; |
1382 | decoder->overflow = true; | |
1383 | return -EOVERFLOW; | |
1384 | } | |
1385 | ||
79b58424 AH |
1386 | static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) |
1387 | { | |
1388 | uint32_t ctc = decoder->packet.payload; | |
1389 | uint32_t fc = decoder->packet.count; | |
1390 | uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; | |
1391 | ||
1392 | if (!decoder->tsc_ctc_ratio_d) | |
1393 | return; | |
1394 | ||
1395 | decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; | |
1396 | decoder->ctc_timestamp = decoder->tsc_timestamp - fc; | |
1397 | if (decoder->tsc_ctc_mult) { | |
1398 | decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; | |
1399 | } else { | |
1400 | decoder->ctc_timestamp -= multdiv(ctc_rem, | |
1401 | decoder->tsc_ctc_ratio_n, | |
1402 | decoder->tsc_ctc_ratio_d); | |
1403 | } | |
1404 | decoder->ctc_delta = 0; | |
1405 | decoder->have_tma = true; | |
3bccbe20 | 1406 | decoder->fixup_last_mtc = true; |
79b58424 AH |
1407 | intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", |
1408 | decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); | |
1409 | } | |
1410 | ||
1411 | static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) | |
1412 | { | |
1413 | uint64_t timestamp; | |
1414 | uint32_t mtc, mtc_delta; | |
1415 | ||
1416 | if (!decoder->have_tma) | |
1417 | return; | |
1418 | ||
1419 | mtc = decoder->packet.payload; | |
1420 | ||
3bccbe20 AH |
1421 | if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { |
1422 | decoder->fixup_last_mtc = false; | |
1423 | intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, | |
1424 | &decoder->last_mtc); | |
1425 | } | |
1426 | ||
79b58424 AH |
1427 | if (mtc > decoder->last_mtc) |
1428 | mtc_delta = mtc - decoder->last_mtc; | |
1429 | else | |
1430 | mtc_delta = mtc + 256 - decoder->last_mtc; | |
1431 | ||
1432 | decoder->ctc_delta += mtc_delta << decoder->mtc_shift; | |
1433 | ||
1434 | if (decoder->tsc_ctc_mult) { | |
1435 | timestamp = decoder->ctc_timestamp + | |
1436 | decoder->ctc_delta * decoder->tsc_ctc_mult; | |
1437 | } else { | |
1438 | timestamp = decoder->ctc_timestamp + | |
1439 | multdiv(decoder->ctc_delta, | |
1440 | decoder->tsc_ctc_ratio_n, | |
1441 | decoder->tsc_ctc_ratio_d); | |
1442 | } | |
1443 | ||
1444 | if (timestamp < decoder->timestamp) | |
1445 | intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", | |
1446 | timestamp, decoder->timestamp); | |
1447 | else | |
1448 | decoder->timestamp = timestamp; | |
1449 | ||
1450 | decoder->timestamp_insn_cnt = 0; | |
1451 | decoder->last_mtc = mtc; | |
cc336186 AH |
1452 | |
1453 | if (decoder->last_packet_type == INTEL_PT_CYC) { | |
1454 | decoder->cyc_ref_timestamp = decoder->timestamp; | |
1455 | decoder->cycle_cnt = 0; | |
1456 | decoder->have_calc_cyc_to_tsc = false; | |
1457 | intel_pt_calc_cyc_to_tsc(decoder, true); | |
1458 | } | |
1459 | } | |
1460 | ||
1461 | static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) | |
1462 | { | |
26fb2fb1 | 1463 | unsigned int cbr = decoder->packet.payload & 0xff; |
cc336186 | 1464 | |
0a7c700d AH |
1465 | decoder->cbr_payload = decoder->packet.payload; |
1466 | ||
cc336186 AH |
1467 | if (decoder->cbr == cbr) |
1468 | return; | |
1469 | ||
1470 | decoder->cbr = cbr; | |
1471 | decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; | |
1472 | } | |
1473 | ||
1474 | static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) | |
1475 | { | |
1476 | uint64_t timestamp = decoder->cyc_ref_timestamp; | |
1477 | ||
1478 | decoder->have_cyc = true; | |
1479 | ||
1480 | decoder->cycle_cnt += decoder->packet.payload; | |
1481 | ||
1482 | if (!decoder->cyc_ref_timestamp) | |
1483 | return; | |
1484 | ||
1485 | if (decoder->have_calc_cyc_to_tsc) | |
1486 | timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; | |
1487 | else if (decoder->cbr) | |
1488 | timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; | |
1489 | else | |
1490 | return; | |
1491 | ||
1492 | if (timestamp < decoder->timestamp) | |
1493 | intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", | |
1494 | timestamp, decoder->timestamp); | |
1495 | else | |
1496 | decoder->timestamp = timestamp; | |
51ee6481 AH |
1497 | |
1498 | decoder->timestamp_insn_cnt = 0; | |
79b58424 AH |
1499 | } |
1500 | ||
f4aa0819 AH |
1501 | /* Walk PSB+ packets when already in sync. */ |
1502 | static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) | |
1503 | { | |
1504 | int err; | |
1505 | ||
1506 | while (1) { | |
1507 | err = intel_pt_get_next_packet(decoder); | |
1508 | if (err) | |
1509 | return err; | |
1510 | ||
1511 | switch (decoder->packet.type) { | |
1512 | case INTEL_PT_PSBEND: | |
1513 | return 0; | |
1514 | ||
1515 | case INTEL_PT_TIP_PGD: | |
1516 | case INTEL_PT_TIP_PGE: | |
1517 | case INTEL_PT_TIP: | |
1518 | case INTEL_PT_TNT: | |
3d498078 | 1519 | case INTEL_PT_TRACESTOP: |
f4aa0819 AH |
1520 | case INTEL_PT_BAD: |
1521 | case INTEL_PT_PSB: | |
a472e65f AH |
1522 | case INTEL_PT_PTWRITE: |
1523 | case INTEL_PT_PTWRITE_IP: | |
1524 | case INTEL_PT_EXSTOP: | |
1525 | case INTEL_PT_EXSTOP_IP: | |
1526 | case INTEL_PT_MWAIT: | |
1527 | case INTEL_PT_PWRE: | |
1528 | case INTEL_PT_PWRX: | |
79b58424 | 1529 | decoder->have_tma = false; |
f4aa0819 AH |
1530 | intel_pt_log("ERROR: Unexpected packet\n"); |
1531 | return -EAGAIN; | |
1532 | ||
1533 | case INTEL_PT_OVF: | |
1534 | return intel_pt_overflow(decoder); | |
1535 | ||
1536 | case INTEL_PT_TSC: | |
1537 | intel_pt_calc_tsc_timestamp(decoder); | |
1538 | break; | |
1539 | ||
3d498078 | 1540 | case INTEL_PT_TMA: |
79b58424 | 1541 | intel_pt_calc_tma(decoder); |
3d498078 AH |
1542 | break; |
1543 | ||
f4aa0819 | 1544 | case INTEL_PT_CBR: |
cc336186 | 1545 | intel_pt_calc_cbr(decoder); |
f4aa0819 AH |
1546 | break; |
1547 | ||
1548 | case INTEL_PT_MODE_EXEC: | |
1549 | decoder->exec_mode = decoder->packet.payload; | |
1550 | break; | |
1551 | ||
1552 | case INTEL_PT_PIP: | |
3d498078 | 1553 | decoder->cr3 = decoder->packet.payload & (BIT63 - 1); |
f4aa0819 AH |
1554 | break; |
1555 | ||
1556 | case INTEL_PT_FUP: | |
1557 | decoder->pge = true; | |
f952eace AH |
1558 | if (decoder->packet.count) |
1559 | intel_pt_set_last_ip(decoder); | |
f4aa0819 AH |
1560 | break; |
1561 | ||
1562 | case INTEL_PT_MODE_TSX: | |
1563 | intel_pt_update_in_tx(decoder); | |
1564 | break; | |
1565 | ||
3d498078 | 1566 | case INTEL_PT_MTC: |
79b58424 AH |
1567 | intel_pt_calc_mtc_timestamp(decoder); |
1568 | if (decoder->period_type == INTEL_PT_PERIOD_MTC) | |
1569 | decoder->state.type |= INTEL_PT_INSTRUCTION; | |
3d498078 AH |
1570 | break; |
1571 | ||
1572 | case INTEL_PT_CYC: | |
1573 | case INTEL_PT_VMCS: | |
1574 | case INTEL_PT_MNT: | |
f4aa0819 AH |
1575 | case INTEL_PT_PAD: |
1576 | default: | |
1577 | break; | |
1578 | } | |
1579 | } | |
1580 | } | |
1581 | ||
1582 | static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) | |
1583 | { | |
1584 | int err; | |
1585 | ||
1586 | if (decoder->tx_flags & INTEL_PT_ABORT_TX) { | |
1587 | decoder->tx_flags = 0; | |
1588 | decoder->state.flags &= ~INTEL_PT_IN_TX; | |
1589 | decoder->state.flags |= INTEL_PT_ABORT_TX; | |
1590 | } else { | |
1591 | decoder->state.flags |= INTEL_PT_ASYNC; | |
1592 | } | |
1593 | ||
1594 | while (1) { | |
1595 | err = intel_pt_get_next_packet(decoder); | |
1596 | if (err) | |
1597 | return err; | |
1598 | ||
1599 | switch (decoder->packet.type) { | |
1600 | case INTEL_PT_TNT: | |
1601 | case INTEL_PT_FUP: | |
3d498078 | 1602 | case INTEL_PT_TRACESTOP: |
f4aa0819 AH |
1603 | case INTEL_PT_PSB: |
1604 | case INTEL_PT_TSC: | |
3d498078 | 1605 | case INTEL_PT_TMA: |
f4aa0819 AH |
1606 | case INTEL_PT_CBR: |
1607 | case INTEL_PT_MODE_TSX: | |
1608 | case INTEL_PT_BAD: | |
1609 | case INTEL_PT_PSBEND: | |
a472e65f AH |
1610 | case INTEL_PT_PTWRITE: |
1611 | case INTEL_PT_PTWRITE_IP: | |
1612 | case INTEL_PT_EXSTOP: | |
1613 | case INTEL_PT_EXSTOP_IP: | |
1614 | case INTEL_PT_MWAIT: | |
1615 | case INTEL_PT_PWRE: | |
1616 | case INTEL_PT_PWRX: | |
f4aa0819 AH |
1617 | intel_pt_log("ERROR: Missing TIP after FUP\n"); |
1618 | decoder->pkt_state = INTEL_PT_STATE_ERR3; | |
1619 | return -ENOENT; | |
1620 | ||
1621 | case INTEL_PT_OVF: | |
1622 | return intel_pt_overflow(decoder); | |
1623 | ||
1624 | case INTEL_PT_TIP_PGD: | |
1625 | decoder->state.from_ip = decoder->ip; | |
1626 | decoder->state.to_ip = 0; | |
1627 | if (decoder->packet.count != 0) { | |
1628 | intel_pt_set_ip(decoder); | |
1629 | intel_pt_log("Omitting PGD ip " x64_fmt "\n", | |
1630 | decoder->ip); | |
1631 | } | |
1632 | decoder->pge = false; | |
1633 | decoder->continuous_period = false; | |
1634 | return 0; | |
1635 | ||
1636 | case INTEL_PT_TIP_PGE: | |
1637 | decoder->pge = true; | |
1638 | intel_pt_log("Omitting PGE ip " x64_fmt "\n", | |
1639 | decoder->ip); | |
1640 | decoder->state.from_ip = 0; | |
1641 | if (decoder->packet.count == 0) { | |
1642 | decoder->state.to_ip = 0; | |
1643 | } else { | |
1644 | intel_pt_set_ip(decoder); | |
1645 | decoder->state.to_ip = decoder->ip; | |
1646 | } | |
1647 | return 0; | |
1648 | ||
1649 | case INTEL_PT_TIP: | |
1650 | decoder->state.from_ip = decoder->ip; | |
1651 | if (decoder->packet.count == 0) { | |
1652 | decoder->state.to_ip = 0; | |
1653 | } else { | |
1654 | intel_pt_set_ip(decoder); | |
1655 | decoder->state.to_ip = decoder->ip; | |
1656 | } | |
1657 | return 0; | |
1658 | ||
1659 | case INTEL_PT_PIP: | |
3d498078 AH |
1660 | decoder->cr3 = decoder->packet.payload & (BIT63 - 1); |
1661 | break; | |
1662 | ||
1663 | case INTEL_PT_MTC: | |
79b58424 AH |
1664 | intel_pt_calc_mtc_timestamp(decoder); |
1665 | if (decoder->period_type == INTEL_PT_PERIOD_MTC) | |
1666 | decoder->state.type |= INTEL_PT_INSTRUCTION; | |
3d498078 AH |
1667 | break; |
1668 | ||
1669 | case INTEL_PT_CYC: | |
cc336186 | 1670 | intel_pt_calc_cyc_timestamp(decoder); |
f4aa0819 AH |
1671 | break; |
1672 | ||
1673 | case INTEL_PT_MODE_EXEC: | |
1674 | decoder->exec_mode = decoder->packet.payload; | |
1675 | break; | |
1676 | ||
3d498078 AH |
1677 | case INTEL_PT_VMCS: |
1678 | case INTEL_PT_MNT: | |
f4aa0819 AH |
1679 | case INTEL_PT_PAD: |
1680 | break; | |
1681 | ||
1682 | default: | |
1683 | return intel_pt_bug(decoder); | |
1684 | } | |
1685 | } | |
1686 | } | |
1687 | ||
1688 | static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) | |
1689 | { | |
1690 | bool no_tip = false; | |
1691 | int err; | |
1692 | ||
1693 | while (1) { | |
1694 | err = intel_pt_get_next_packet(decoder); | |
1695 | if (err) | |
1696 | return err; | |
1697 | next: | |
1698 | switch (decoder->packet.type) { | |
1699 | case INTEL_PT_TNT: | |
1700 | if (!decoder->packet.count) | |
1701 | break; | |
1702 | decoder->tnt = decoder->packet; | |
1703 | decoder->pkt_state = INTEL_PT_STATE_TNT; | |
1704 | err = intel_pt_walk_tnt(decoder); | |
1705 | if (err == -EAGAIN) | |
1706 | break; | |
1707 | return err; | |
1708 | ||
1709 | case INTEL_PT_TIP_PGD: | |
1710 | if (decoder->packet.count != 0) | |
1711 | intel_pt_set_last_ip(decoder); | |
1712 | decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; | |
1713 | return intel_pt_walk_tip(decoder); | |
1714 | ||
1715 | case INTEL_PT_TIP_PGE: { | |
1716 | decoder->pge = true; | |
1717 | if (decoder->packet.count == 0) { | |
1718 | intel_pt_log_at("Skipping zero TIP.PGE", | |
1719 | decoder->pos); | |
1720 | break; | |
1721 | } | |
1722 | intel_pt_set_ip(decoder); | |
1723 | decoder->state.from_ip = 0; | |
1724 | decoder->state.to_ip = decoder->ip; | |
1725 | return 0; | |
1726 | } | |
1727 | ||
1728 | case INTEL_PT_OVF: | |
1729 | return intel_pt_overflow(decoder); | |
1730 | ||
1731 | case INTEL_PT_TIP: | |
1732 | if (decoder->packet.count != 0) | |
1733 | intel_pt_set_last_ip(decoder); | |
1734 | decoder->pkt_state = INTEL_PT_STATE_TIP; | |
1735 | return intel_pt_walk_tip(decoder); | |
1736 | ||
1737 | case INTEL_PT_FUP: | |
1738 | if (decoder->packet.count == 0) { | |
1739 | intel_pt_log_at("Skipping zero FUP", | |
1740 | decoder->pos); | |
1741 | no_tip = false; | |
1742 | break; | |
1743 | } | |
1744 | intel_pt_set_last_ip(decoder); | |
83959817 AH |
1745 | if (!decoder->branch_enable) { |
1746 | decoder->ip = decoder->last_ip; | |
a472e65f AH |
1747 | if (intel_pt_fup_event(decoder)) |
1748 | return 0; | |
1749 | no_tip = false; | |
83959817 AH |
1750 | break; |
1751 | } | |
a472e65f AH |
1752 | if (decoder->set_fup_mwait) |
1753 | no_tip = true; | |
f4aa0819 AH |
1754 | err = intel_pt_walk_fup(decoder); |
1755 | if (err != -EAGAIN) { | |
1756 | if (err) | |
1757 | return err; | |
1758 | if (no_tip) | |
1759 | decoder->pkt_state = | |
1760 | INTEL_PT_STATE_FUP_NO_TIP; | |
1761 | else | |
1762 | decoder->pkt_state = INTEL_PT_STATE_FUP; | |
1763 | return 0; | |
1764 | } | |
1765 | if (no_tip) { | |
1766 | no_tip = false; | |
1767 | break; | |
1768 | } | |
1769 | return intel_pt_walk_fup_tip(decoder); | |
1770 | ||
3d498078 | 1771 | case INTEL_PT_TRACESTOP: |
7eacca3e AH |
1772 | decoder->pge = false; |
1773 | decoder->continuous_period = false; | |
1774 | intel_pt_clear_tx_flags(decoder); | |
1775 | decoder->have_tma = false; | |
3d498078 AH |
1776 | break; |
1777 | ||
f4aa0819 | 1778 | case INTEL_PT_PSB: |
ee14ac0e AH |
1779 | decoder->last_ip = 0; |
1780 | decoder->have_last_ip = true; | |
f4aa0819 AH |
1781 | intel_pt_clear_stack(&decoder->stack); |
1782 | err = intel_pt_walk_psbend(decoder); | |
1783 | if (err == -EAGAIN) | |
1784 | goto next; | |
1785 | if (err) | |
1786 | return err; | |
1787 | break; | |
1788 | ||
1789 | case INTEL_PT_PIP: | |
3d498078 AH |
1790 | decoder->cr3 = decoder->packet.payload & (BIT63 - 1); |
1791 | break; | |
1792 | ||
1793 | case INTEL_PT_MTC: | |
79b58424 AH |
1794 | intel_pt_calc_mtc_timestamp(decoder); |
1795 | if (decoder->period_type != INTEL_PT_PERIOD_MTC) | |
1796 | break; | |
1797 | /* | |
1798 | * Ensure that there has been an instruction since the | |
1799 | * last MTC. | |
1800 | */ | |
1801 | if (!decoder->mtc_insn) | |
1802 | break; | |
1803 | decoder->mtc_insn = false; | |
1804 | /* Ensure that there is a timestamp */ | |
1805 | if (!decoder->timestamp) | |
1806 | break; | |
1807 | decoder->state.type = INTEL_PT_INSTRUCTION; | |
1808 | decoder->state.from_ip = decoder->ip; | |
1809 | decoder->state.to_ip = 0; | |
1810 | decoder->mtc_insn = false; | |
1811 | return 0; | |
f4aa0819 AH |
1812 | |
1813 | case INTEL_PT_TSC: | |
1814 | intel_pt_calc_tsc_timestamp(decoder); | |
1815 | break; | |
1816 | ||
3d498078 | 1817 | case INTEL_PT_TMA: |
79b58424 | 1818 | intel_pt_calc_tma(decoder); |
3d498078 AH |
1819 | break; |
1820 | ||
1821 | case INTEL_PT_CYC: | |
cc336186 | 1822 | intel_pt_calc_cyc_timestamp(decoder); |
3d498078 AH |
1823 | break; |
1824 | ||
f4aa0819 | 1825 | case INTEL_PT_CBR: |
cc336186 | 1826 | intel_pt_calc_cbr(decoder); |
0a7c700d AH |
1827 | if (!decoder->branch_enable && |
1828 | decoder->cbr != decoder->cbr_seen) { | |
1829 | decoder->cbr_seen = decoder->cbr; | |
1830 | decoder->state.type = INTEL_PT_CBR_CHG; | |
1831 | decoder->state.from_ip = decoder->ip; | |
1832 | decoder->state.to_ip = 0; | |
1833 | decoder->state.cbr_payload = | |
1834 | decoder->packet.payload; | |
1835 | return 0; | |
1836 | } | |
f4aa0819 AH |
1837 | break; |
1838 | ||
1839 | case INTEL_PT_MODE_EXEC: | |
1840 | decoder->exec_mode = decoder->packet.payload; | |
1841 | break; | |
1842 | ||
1843 | case INTEL_PT_MODE_TSX: | |
1844 | /* MODE_TSX need not be followed by FUP */ | |
1845 | if (!decoder->pge) { | |
1846 | intel_pt_update_in_tx(decoder); | |
1847 | break; | |
1848 | } | |
1849 | err = intel_pt_mode_tsx(decoder, &no_tip); | |
1850 | if (err) | |
1851 | return err; | |
1852 | goto next; | |
1853 | ||
1854 | case INTEL_PT_BAD: /* Does not happen */ | |
1855 | return intel_pt_bug(decoder); | |
1856 | ||
1857 | case INTEL_PT_PSBEND: | |
3d498078 AH |
1858 | case INTEL_PT_VMCS: |
1859 | case INTEL_PT_MNT: | |
f4aa0819 AH |
1860 | case INTEL_PT_PAD: |
1861 | break; | |
1862 | ||
a472e65f AH |
1863 | case INTEL_PT_PTWRITE_IP: |
1864 | decoder->fup_ptw_payload = decoder->packet.payload; | |
1865 | err = intel_pt_get_next_packet(decoder); | |
1866 | if (err) | |
1867 | return err; | |
1868 | if (decoder->packet.type == INTEL_PT_FUP) { | |
1869 | decoder->set_fup_ptw = true; | |
1870 | no_tip = true; | |
1871 | } else { | |
1872 | intel_pt_log_at("ERROR: Missing FUP after PTWRITE", | |
1873 | decoder->pos); | |
1874 | } | |
1875 | goto next; | |
1876 | ||
1877 | case INTEL_PT_PTWRITE: | |
1878 | decoder->state.type = INTEL_PT_PTW; | |
1879 | decoder->state.from_ip = decoder->ip; | |
1880 | decoder->state.to_ip = 0; | |
1881 | decoder->state.ptw_payload = decoder->packet.payload; | |
1882 | return 0; | |
1883 | ||
1884 | case INTEL_PT_MWAIT: | |
1885 | decoder->fup_mwait_payload = decoder->packet.payload; | |
1886 | decoder->set_fup_mwait = true; | |
1887 | break; | |
1888 | ||
1889 | case INTEL_PT_PWRE: | |
1890 | if (decoder->set_fup_mwait) { | |
1891 | decoder->fup_pwre_payload = | |
1892 | decoder->packet.payload; | |
1893 | decoder->set_fup_pwre = true; | |
1894 | break; | |
1895 | } | |
1896 | decoder->state.type = INTEL_PT_PWR_ENTRY; | |
1897 | decoder->state.from_ip = decoder->ip; | |
1898 | decoder->state.to_ip = 0; | |
1899 | decoder->state.pwrx_payload = decoder->packet.payload; | |
1900 | return 0; | |
1901 | ||
1902 | case INTEL_PT_EXSTOP_IP: | |
1903 | err = intel_pt_get_next_packet(decoder); | |
1904 | if (err) | |
1905 | return err; | |
1906 | if (decoder->packet.type == INTEL_PT_FUP) { | |
1907 | decoder->set_fup_exstop = true; | |
1908 | no_tip = true; | |
1909 | } else { | |
1910 | intel_pt_log_at("ERROR: Missing FUP after EXSTOP", | |
1911 | decoder->pos); | |
1912 | } | |
1913 | goto next; | |
1914 | ||
1915 | case INTEL_PT_EXSTOP: | |
1916 | decoder->state.type = INTEL_PT_EX_STOP; | |
1917 | decoder->state.from_ip = decoder->ip; | |
1918 | decoder->state.to_ip = 0; | |
1919 | return 0; | |
1920 | ||
1921 | case INTEL_PT_PWRX: | |
1922 | decoder->state.type = INTEL_PT_PWR_EXIT; | |
1923 | decoder->state.from_ip = decoder->ip; | |
1924 | decoder->state.to_ip = 0; | |
1925 | decoder->state.pwrx_payload = decoder->packet.payload; | |
1926 | return 0; | |
1927 | ||
f4aa0819 AH |
1928 | default: |
1929 | return intel_pt_bug(decoder); | |
1930 | } | |
1931 | } | |
1932 | } | |
1933 | ||
e1717e04 AH |
1934 | static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) |
1935 | { | |
f952eace AH |
1936 | return decoder->packet.count && |
1937 | (decoder->have_last_ip || decoder->packet.count == 3 || | |
1938 | decoder->packet.count == 6); | |
e1717e04 AH |
1939 | } |
1940 | ||
f4aa0819 AH |
1941 | /* Walk PSB+ packets to get in sync. */ |
1942 | static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) | |
1943 | { | |
1944 | int err; | |
1945 | ||
1946 | while (1) { | |
1947 | err = intel_pt_get_next_packet(decoder); | |
1948 | if (err) | |
1949 | return err; | |
1950 | ||
1951 | switch (decoder->packet.type) { | |
1952 | case INTEL_PT_TIP_PGD: | |
1953 | decoder->continuous_period = false; | |
7ea6856d | 1954 | __fallthrough; |
f4aa0819 AH |
1955 | case INTEL_PT_TIP_PGE: |
1956 | case INTEL_PT_TIP: | |
a472e65f AH |
1957 | case INTEL_PT_PTWRITE: |
1958 | case INTEL_PT_PTWRITE_IP: | |
1959 | case INTEL_PT_EXSTOP: | |
1960 | case INTEL_PT_EXSTOP_IP: | |
1961 | case INTEL_PT_MWAIT: | |
1962 | case INTEL_PT_PWRE: | |
1963 | case INTEL_PT_PWRX: | |
f4aa0819 AH |
1964 | intel_pt_log("ERROR: Unexpected packet\n"); |
1965 | return -ENOENT; | |
1966 | ||
1967 | case INTEL_PT_FUP: | |
1968 | decoder->pge = true; | |
e1717e04 | 1969 | if (intel_pt_have_ip(decoder)) { |
f4aa0819 AH |
1970 | uint64_t current_ip = decoder->ip; |
1971 | ||
1972 | intel_pt_set_ip(decoder); | |
1973 | if (current_ip) | |
1974 | intel_pt_log_to("Setting IP", | |
1975 | decoder->ip); | |
1976 | } | |
1977 | break; | |
1978 | ||
3d498078 | 1979 | case INTEL_PT_MTC: |
79b58424 | 1980 | intel_pt_calc_mtc_timestamp(decoder); |
3d498078 AH |
1981 | break; |
1982 | ||
f4aa0819 AH |
1983 | case INTEL_PT_TSC: |
1984 | intel_pt_calc_tsc_timestamp(decoder); | |
1985 | break; | |
1986 | ||
3d498078 | 1987 | case INTEL_PT_TMA: |
79b58424 | 1988 | intel_pt_calc_tma(decoder); |
3d498078 AH |
1989 | break; |
1990 | ||
1991 | case INTEL_PT_CYC: | |
cc336186 | 1992 | intel_pt_calc_cyc_timestamp(decoder); |
3d498078 AH |
1993 | break; |
1994 | ||
f4aa0819 | 1995 | case INTEL_PT_CBR: |
cc336186 | 1996 | intel_pt_calc_cbr(decoder); |
f4aa0819 AH |
1997 | break; |
1998 | ||
1999 | case INTEL_PT_PIP: | |
3d498078 | 2000 | decoder->cr3 = decoder->packet.payload & (BIT63 - 1); |
f4aa0819 AH |
2001 | break; |
2002 | ||
2003 | case INTEL_PT_MODE_EXEC: | |
2004 | decoder->exec_mode = decoder->packet.payload; | |
2005 | break; | |
2006 | ||
2007 | case INTEL_PT_MODE_TSX: | |
2008 | intel_pt_update_in_tx(decoder); | |
2009 | break; | |
2010 | ||
3d498078 | 2011 | case INTEL_PT_TRACESTOP: |
7eacca3e AH |
2012 | decoder->pge = false; |
2013 | decoder->continuous_period = false; | |
2014 | intel_pt_clear_tx_flags(decoder); | |
7ea6856d ACM |
2015 | __fallthrough; |
2016 | ||
f4aa0819 | 2017 | case INTEL_PT_TNT: |
79b58424 | 2018 | decoder->have_tma = false; |
f4aa0819 AH |
2019 | intel_pt_log("ERROR: Unexpected packet\n"); |
2020 | if (decoder->ip) | |
2021 | decoder->pkt_state = INTEL_PT_STATE_ERR4; | |
2022 | else | |
2023 | decoder->pkt_state = INTEL_PT_STATE_ERR3; | |
2024 | return -ENOENT; | |
2025 | ||
2026 | case INTEL_PT_BAD: /* Does not happen */ | |
2027 | return intel_pt_bug(decoder); | |
2028 | ||
2029 | case INTEL_PT_OVF: | |
2030 | return intel_pt_overflow(decoder); | |
2031 | ||
2032 | case INTEL_PT_PSBEND: | |
2033 | return 0; | |
2034 | ||
2035 | case INTEL_PT_PSB: | |
3d498078 AH |
2036 | case INTEL_PT_VMCS: |
2037 | case INTEL_PT_MNT: | |
f4aa0819 AH |
2038 | case INTEL_PT_PAD: |
2039 | default: | |
2040 | break; | |
2041 | } | |
2042 | } | |
2043 | } | |
2044 | ||
2045 | static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) | |
2046 | { | |
2047 | int err; | |
2048 | ||
2049 | while (1) { | |
2050 | err = intel_pt_get_next_packet(decoder); | |
2051 | if (err) | |
2052 | return err; | |
2053 | ||
2054 | switch (decoder->packet.type) { | |
2055 | case INTEL_PT_TIP_PGD: | |
2056 | decoder->continuous_period = false; | |
7ea6856d | 2057 | __fallthrough; |
f4aa0819 AH |
2058 | case INTEL_PT_TIP_PGE: |
2059 | case INTEL_PT_TIP: | |
2060 | decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD; | |
e1717e04 | 2061 | if (intel_pt_have_ip(decoder)) |
f4aa0819 AH |
2062 | intel_pt_set_ip(decoder); |
2063 | if (decoder->ip) | |
2064 | return 0; | |
2065 | break; | |
2066 | ||
2067 | case INTEL_PT_FUP: | |
622b7a47 AH |
2068 | if (intel_pt_have_ip(decoder)) |
2069 | intel_pt_set_ip(decoder); | |
2070 | if (decoder->ip) | |
2071 | return 0; | |
f4aa0819 AH |
2072 | break; |
2073 | ||
3d498078 | 2074 | case INTEL_PT_MTC: |
79b58424 | 2075 | intel_pt_calc_mtc_timestamp(decoder); |
3d498078 AH |
2076 | break; |
2077 | ||
f4aa0819 AH |
2078 | case INTEL_PT_TSC: |
2079 | intel_pt_calc_tsc_timestamp(decoder); | |
2080 | break; | |
2081 | ||
3d498078 | 2082 | case INTEL_PT_TMA: |
79b58424 | 2083 | intel_pt_calc_tma(decoder); |
3d498078 AH |
2084 | break; |
2085 | ||
2086 | case INTEL_PT_CYC: | |
cc336186 | 2087 | intel_pt_calc_cyc_timestamp(decoder); |
3d498078 AH |
2088 | break; |
2089 | ||
f4aa0819 | 2090 | case INTEL_PT_CBR: |
cc336186 | 2091 | intel_pt_calc_cbr(decoder); |
f4aa0819 AH |
2092 | break; |
2093 | ||
2094 | case INTEL_PT_PIP: | |
3d498078 | 2095 | decoder->cr3 = decoder->packet.payload & (BIT63 - 1); |
f4aa0819 AH |
2096 | break; |
2097 | ||
2098 | case INTEL_PT_MODE_EXEC: | |
2099 | decoder->exec_mode = decoder->packet.payload; | |
2100 | break; | |
2101 | ||
2102 | case INTEL_PT_MODE_TSX: | |
2103 | intel_pt_update_in_tx(decoder); | |
2104 | break; | |
2105 | ||
2106 | case INTEL_PT_OVF: | |
2107 | return intel_pt_overflow(decoder); | |
2108 | ||
2109 | case INTEL_PT_BAD: /* Does not happen */ | |
2110 | return intel_pt_bug(decoder); | |
2111 | ||
3d498078 | 2112 | case INTEL_PT_TRACESTOP: |
7eacca3e AH |
2113 | decoder->pge = false; |
2114 | decoder->continuous_period = false; | |
2115 | intel_pt_clear_tx_flags(decoder); | |
2116 | decoder->have_tma = false; | |
3d498078 AH |
2117 | break; |
2118 | ||
f4aa0819 | 2119 | case INTEL_PT_PSB: |
ee14ac0e AH |
2120 | decoder->last_ip = 0; |
2121 | decoder->have_last_ip = true; | |
12b70806 | 2122 | intel_pt_clear_stack(&decoder->stack); |
f4aa0819 AH |
2123 | err = intel_pt_walk_psb(decoder); |
2124 | if (err) | |
2125 | return err; | |
2126 | if (decoder->ip) { | |
2127 | /* Do not have a sample */ | |
2128 | decoder->state.type = 0; | |
2129 | return 0; | |
2130 | } | |
2131 | break; | |
2132 | ||
2133 | case INTEL_PT_TNT: | |
2134 | case INTEL_PT_PSBEND: | |
3d498078 AH |
2135 | case INTEL_PT_VMCS: |
2136 | case INTEL_PT_MNT: | |
f4aa0819 | 2137 | case INTEL_PT_PAD: |
a472e65f AH |
2138 | case INTEL_PT_PTWRITE: |
2139 | case INTEL_PT_PTWRITE_IP: | |
2140 | case INTEL_PT_EXSTOP: | |
2141 | case INTEL_PT_EXSTOP_IP: | |
2142 | case INTEL_PT_MWAIT: | |
2143 | case INTEL_PT_PWRE: | |
2144 | case INTEL_PT_PWRX: | |
f4aa0819 AH |
2145 | default: |
2146 | break; | |
2147 | } | |
2148 | } | |
2149 | } | |
2150 | ||
2151 | static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) | |
2152 | { | |
2153 | int err; | |
2154 | ||
6a558f12 | 2155 | decoder->set_fup_tx_flags = false; |
a472e65f AH |
2156 | decoder->set_fup_ptw = false; |
2157 | decoder->set_fup_mwait = false; | |
2158 | decoder->set_fup_pwre = false; | |
2159 | decoder->set_fup_exstop = false; | |
6a558f12 | 2160 | |
83959817 AH |
2161 | if (!decoder->branch_enable) { |
2162 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
2163 | decoder->overflow = false; | |
2164 | decoder->state.type = 0; /* Do not have a sample */ | |
2165 | return 0; | |
2166 | } | |
2167 | ||
f4aa0819 AH |
2168 | intel_pt_log("Scanning for full IP\n"); |
2169 | err = intel_pt_walk_to_ip(decoder); | |
2170 | if (err) | |
2171 | return err; | |
2172 | ||
2173 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
2174 | decoder->overflow = false; | |
2175 | ||
2176 | decoder->state.from_ip = 0; | |
2177 | decoder->state.to_ip = decoder->ip; | |
2178 | intel_pt_log_to("Setting IP", decoder->ip); | |
2179 | ||
2180 | return 0; | |
2181 | } | |
2182 | ||
2183 | static int intel_pt_part_psb(struct intel_pt_decoder *decoder) | |
2184 | { | |
2185 | const unsigned char *end = decoder->buf + decoder->len; | |
2186 | size_t i; | |
2187 | ||
2188 | for (i = INTEL_PT_PSB_LEN - 1; i; i--) { | |
2189 | if (i > decoder->len) | |
2190 | continue; | |
2191 | if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) | |
2192 | return i; | |
2193 | } | |
2194 | return 0; | |
2195 | } | |
2196 | ||
2197 | static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) | |
2198 | { | |
2199 | size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; | |
2200 | const char *psb = INTEL_PT_PSB_STR; | |
2201 | ||
2202 | if (rest_psb > decoder->len || | |
2203 | memcmp(decoder->buf, psb + part_psb, rest_psb)) | |
2204 | return 0; | |
2205 | ||
2206 | return rest_psb; | |
2207 | } | |
2208 | ||
2209 | static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, | |
2210 | int part_psb) | |
2211 | { | |
2212 | int rest_psb, ret; | |
2213 | ||
2214 | decoder->pos += decoder->len; | |
2215 | decoder->len = 0; | |
2216 | ||
2217 | ret = intel_pt_get_next_data(decoder); | |
2218 | if (ret) | |
2219 | return ret; | |
2220 | ||
2221 | rest_psb = intel_pt_rest_psb(decoder, part_psb); | |
2222 | if (!rest_psb) | |
2223 | return 0; | |
2224 | ||
2225 | decoder->pos -= part_psb; | |
2226 | decoder->next_buf = decoder->buf + rest_psb; | |
2227 | decoder->next_len = decoder->len - rest_psb; | |
2228 | memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); | |
2229 | decoder->buf = decoder->temp_buf; | |
2230 | decoder->len = INTEL_PT_PSB_LEN; | |
2231 | ||
2232 | return 0; | |
2233 | } | |
2234 | ||
2235 | static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) | |
2236 | { | |
2237 | unsigned char *next; | |
2238 | int ret; | |
2239 | ||
2240 | intel_pt_log("Scanning for PSB\n"); | |
2241 | while (1) { | |
2242 | if (!decoder->len) { | |
2243 | ret = intel_pt_get_next_data(decoder); | |
2244 | if (ret) | |
2245 | return ret; | |
2246 | } | |
2247 | ||
2248 | next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, | |
2249 | INTEL_PT_PSB_LEN); | |
2250 | if (!next) { | |
2251 | int part_psb; | |
2252 | ||
2253 | part_psb = intel_pt_part_psb(decoder); | |
2254 | if (part_psb) { | |
2255 | ret = intel_pt_get_split_psb(decoder, part_psb); | |
2256 | if (ret) | |
2257 | return ret; | |
2258 | } else { | |
2259 | decoder->pos += decoder->len; | |
2260 | decoder->len = 0; | |
2261 | } | |
2262 | continue; | |
2263 | } | |
2264 | ||
2265 | decoder->pkt_step = next - decoder->buf; | |
2266 | return intel_pt_get_next_packet(decoder); | |
2267 | } | |
2268 | } | |
2269 | ||
2270 | static int intel_pt_sync(struct intel_pt_decoder *decoder) | |
2271 | { | |
2272 | int err; | |
2273 | ||
2274 | decoder->pge = false; | |
2275 | decoder->continuous_period = false; | |
ee14ac0e | 2276 | decoder->have_last_ip = false; |
f4aa0819 AH |
2277 | decoder->last_ip = 0; |
2278 | decoder->ip = 0; | |
2279 | intel_pt_clear_stack(&decoder->stack); | |
2280 | ||
2281 | err = intel_pt_scan_for_psb(decoder); | |
2282 | if (err) | |
2283 | return err; | |
2284 | ||
ee14ac0e | 2285 | decoder->have_last_ip = true; |
f4aa0819 AH |
2286 | decoder->pkt_state = INTEL_PT_STATE_NO_IP; |
2287 | ||
2288 | err = intel_pt_walk_psb(decoder); | |
2289 | if (err) | |
2290 | return err; | |
2291 | ||
2292 | if (decoder->ip) { | |
2293 | decoder->state.type = 0; /* Do not have a sample */ | |
2294 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
2295 | } else { | |
2296 | return intel_pt_sync_ip(decoder); | |
2297 | } | |
2298 | ||
2299 | return 0; | |
2300 | } | |
2301 | ||
2302 | static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) | |
2303 | { | |
3f04d98e | 2304 | uint64_t est = decoder->sample_insn_cnt << 1; |
f4aa0819 AH |
2305 | |
2306 | if (!decoder->cbr || !decoder->max_non_turbo_ratio) | |
2307 | goto out; | |
2308 | ||
2309 | est *= decoder->max_non_turbo_ratio; | |
2310 | est /= decoder->cbr; | |
2311 | out: | |
3f04d98e | 2312 | return decoder->sample_timestamp + est; |
f4aa0819 AH |
2313 | } |
2314 | ||
2315 | const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) | |
2316 | { | |
2317 | int err; | |
2318 | ||
2319 | do { | |
2320 | decoder->state.type = INTEL_PT_BRANCH; | |
2321 | decoder->state.flags = 0; | |
2322 | ||
2323 | switch (decoder->pkt_state) { | |
2324 | case INTEL_PT_STATE_NO_PSB: | |
2325 | err = intel_pt_sync(decoder); | |
2326 | break; | |
2327 | case INTEL_PT_STATE_NO_IP: | |
ee14ac0e | 2328 | decoder->have_last_ip = false; |
f4aa0819 | 2329 | decoder->last_ip = 0; |
ad7167a8 | 2330 | decoder->ip = 0; |
04194207 | 2331 | __fallthrough; |
f4aa0819 AH |
2332 | case INTEL_PT_STATE_ERR_RESYNC: |
2333 | err = intel_pt_sync_ip(decoder); | |
2334 | break; | |
2335 | case INTEL_PT_STATE_IN_SYNC: | |
2336 | err = intel_pt_walk_trace(decoder); | |
2337 | break; | |
2338 | case INTEL_PT_STATE_TNT: | |
2339 | err = intel_pt_walk_tnt(decoder); | |
2340 | if (err == -EAGAIN) | |
2341 | err = intel_pt_walk_trace(decoder); | |
2342 | break; | |
2343 | case INTEL_PT_STATE_TIP: | |
2344 | case INTEL_PT_STATE_TIP_PGD: | |
2345 | err = intel_pt_walk_tip(decoder); | |
2346 | break; | |
2347 | case INTEL_PT_STATE_FUP: | |
2348 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
2349 | err = intel_pt_walk_fup(decoder); | |
2350 | if (err == -EAGAIN) | |
2351 | err = intel_pt_walk_fup_tip(decoder); | |
2352 | else if (!err) | |
2353 | decoder->pkt_state = INTEL_PT_STATE_FUP; | |
2354 | break; | |
2355 | case INTEL_PT_STATE_FUP_NO_TIP: | |
2356 | decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; | |
2357 | err = intel_pt_walk_fup(decoder); | |
2358 | if (err == -EAGAIN) | |
2359 | err = intel_pt_walk_trace(decoder); | |
2360 | break; | |
2361 | default: | |
2362 | err = intel_pt_bug(decoder); | |
2363 | break; | |
2364 | } | |
2365 | } while (err == -ENOLINK); | |
2366 | ||
22c06892 AH |
2367 | if (err) { |
2368 | decoder->state.err = intel_pt_ext_err(err); | |
2369 | decoder->state.from_ip = decoder->ip; | |
3f04d98e AH |
2370 | decoder->sample_timestamp = decoder->timestamp; |
2371 | decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; | |
22c06892 AH |
2372 | } else { |
2373 | decoder->state.err = 0; | |
0a7c700d AH |
2374 | if (decoder->cbr != decoder->cbr_seen && decoder->state.type) { |
2375 | decoder->cbr_seen = decoder->cbr; | |
2376 | decoder->state.type |= INTEL_PT_CBR_CHG; | |
2377 | decoder->state.cbr_payload = decoder->cbr_payload; | |
2378 | } | |
3f04d98e AH |
2379 | if (intel_pt_sample_time(decoder->pkt_state)) { |
2380 | decoder->sample_timestamp = decoder->timestamp; | |
2381 | decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; | |
2382 | } | |
22c06892 AH |
2383 | } |
2384 | ||
3f04d98e | 2385 | decoder->state.timestamp = decoder->sample_timestamp; |
f4aa0819 AH |
2386 | decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); |
2387 | decoder->state.cr3 = decoder->cr3; | |
2a21d036 | 2388 | decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; |
f4aa0819 | 2389 | |
f4aa0819 AH |
2390 | return &decoder->state; |
2391 | } | |
2392 | ||
2393 | static bool intel_pt_at_psb(unsigned char *buf, size_t len) | |
2394 | { | |
2395 | if (len < INTEL_PT_PSB_LEN) | |
2396 | return false; | |
2397 | return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR, | |
2398 | INTEL_PT_PSB_LEN); | |
2399 | } | |
2400 | ||
2401 | /** | |
2402 | * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. | |
2403 | * @buf: pointer to buffer pointer | |
2404 | * @len: size of buffer | |
2405 | * | |
2406 | * Updates the buffer pointer to point to the start of the next PSB packet if | |
2407 | * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, | |
2408 | * @len is adjusted accordingly. | |
2409 | * | |
2410 | * Return: %true if a PSB packet is found, %false otherwise. | |
2411 | */ | |
2412 | static bool intel_pt_next_psb(unsigned char **buf, size_t *len) | |
2413 | { | |
2414 | unsigned char *next; | |
2415 | ||
2416 | next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); | |
2417 | if (next) { | |
2418 | *len -= next - *buf; | |
2419 | *buf = next; | |
2420 | return true; | |
2421 | } | |
2422 | return false; | |
2423 | } | |
2424 | ||
2425 | /** | |
2426 | * intel_pt_step_psb - move buffer pointer to the start of the following PSB | |
2427 | * packet. | |
2428 | * @buf: pointer to buffer pointer | |
2429 | * @len: size of buffer | |
2430 | * | |
2431 | * Updates the buffer pointer to point to the start of the following PSB packet | |
2432 | * (skipping the PSB at @buf itself) if there is one, otherwise the buffer | |
2433 | * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. | |
2434 | * | |
2435 | * Return: %true if a PSB packet is found, %false otherwise. | |
2436 | */ | |
2437 | static bool intel_pt_step_psb(unsigned char **buf, size_t *len) | |
2438 | { | |
2439 | unsigned char *next; | |
2440 | ||
2441 | if (!*len) | |
2442 | return false; | |
2443 | ||
2444 | next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); | |
2445 | if (next) { | |
2446 | *len -= next - *buf; | |
2447 | *buf = next; | |
2448 | return true; | |
2449 | } | |
2450 | return false; | |
2451 | } | |
2452 | ||
2453 | /** | |
2454 | * intel_pt_last_psb - find the last PSB packet in a buffer. | |
2455 | * @buf: buffer | |
2456 | * @len: size of buffer | |
2457 | * | |
2458 | * This function finds the last PSB in a buffer. | |
2459 | * | |
2460 | * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. | |
2461 | */ | |
2462 | static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) | |
2463 | { | |
2464 | const char *n = INTEL_PT_PSB_STR; | |
2465 | unsigned char *p; | |
2466 | size_t k; | |
2467 | ||
2468 | if (len < INTEL_PT_PSB_LEN) | |
2469 | return NULL; | |
2470 | ||
2471 | k = len - INTEL_PT_PSB_LEN + 1; | |
2472 | while (1) { | |
2473 | p = memrchr(buf, n[0], k); | |
2474 | if (!p) | |
2475 | return NULL; | |
2476 | if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) | |
2477 | return p; | |
2478 | k = p - buf; | |
2479 | if (!k) | |
2480 | return NULL; | |
2481 | } | |
2482 | } | |
2483 | ||
2484 | /** | |
2485 | * intel_pt_next_tsc - find and return next TSC. | |
2486 | * @buf: buffer | |
2487 | * @len: size of buffer | |
2488 | * @tsc: TSC value returned | |
2489 | * | |
2490 | * Find a TSC packet in @buf and return the TSC value. This function assumes | |
2491 | * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a | |
2492 | * PSBEND packet is found. | |
2493 | * | |
2494 | * Return: %true if TSC is found, false otherwise. | |
2495 | */ | |
2496 | static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc) | |
2497 | { | |
2498 | struct intel_pt_pkt packet; | |
2499 | int ret; | |
2500 | ||
2501 | while (len) { | |
2502 | ret = intel_pt_get_packet(buf, len, &packet); | |
2503 | if (ret <= 0) | |
2504 | return false; | |
2505 | if (packet.type == INTEL_PT_TSC) { | |
2506 | *tsc = packet.payload; | |
2507 | return true; | |
2508 | } | |
2509 | if (packet.type == INTEL_PT_PSBEND) | |
2510 | return false; | |
2511 | buf += ret; | |
2512 | len -= ret; | |
2513 | } | |
2514 | return false; | |
2515 | } | |
2516 | ||
2517 | /** | |
2518 | * intel_pt_tsc_cmp - compare 7-byte TSCs. | |
2519 | * @tsc1: first TSC to compare | |
2520 | * @tsc2: second TSC to compare | |
2521 | * | |
2522 | * This function compares 7-byte TSC values allowing for the possibility that | |
2523 | * TSC wrapped around. Generally it is not possible to know if TSC has wrapped | |
2524 | * around so for that purpose this function assumes the absolute difference is | |
2525 | * less than half the maximum difference. | |
2526 | * | |
2527 | * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is | |
2528 | * after @tsc2. | |
2529 | */ | |
2530 | static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) | |
2531 | { | |
2532 | const uint64_t halfway = (1ULL << 55); | |
2533 | ||
2534 | if (tsc1 == tsc2) | |
2535 | return 0; | |
2536 | ||
2537 | if (tsc1 < tsc2) { | |
2538 | if (tsc2 - tsc1 < halfway) | |
2539 | return -1; | |
2540 | else | |
2541 | return 1; | |
2542 | } else { | |
2543 | if (tsc1 - tsc2 < halfway) | |
2544 | return 1; | |
2545 | else | |
2546 | return -1; | |
2547 | } | |
2548 | } | |
2549 | ||
2550 | /** | |
2551 | * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data | |
2552 | * using TSC. | |
2553 | * @buf_a: first buffer | |
2554 | * @len_a: size of first buffer | |
2555 | * @buf_b: second buffer | |
2556 | * @len_b: size of second buffer | |
2557 | * | |
2558 | * If the trace contains TSC we can look at the last TSC of @buf_a and the | |
2559 | * first TSC of @buf_b in order to determine if the buffers overlap, and then | |
2560 | * walk forward in @buf_b until a later TSC is found. A precondition is that | |
2561 | * @buf_a and @buf_b are positioned at a PSB. | |
2562 | * | |
2563 | * Return: A pointer into @buf_b from where non-overlapped data starts, or | |
2564 | * @buf_b + @len_b if there is no non-overlapped data. | |
2565 | */ | |
2566 | static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, | |
2567 | size_t len_a, | |
2568 | unsigned char *buf_b, | |
2569 | size_t len_b) | |
2570 | { | |
2571 | uint64_t tsc_a, tsc_b; | |
2572 | unsigned char *p; | |
2573 | size_t len; | |
2574 | ||
2575 | p = intel_pt_last_psb(buf_a, len_a); | |
2576 | if (!p) | |
2577 | return buf_b; /* No PSB in buf_a => no overlap */ | |
2578 | ||
2579 | len = len_a - (p - buf_a); | |
2580 | if (!intel_pt_next_tsc(p, len, &tsc_a)) { | |
2581 | /* The last PSB+ in buf_a is incomplete, so go back one more */ | |
2582 | len_a -= len; | |
2583 | p = intel_pt_last_psb(buf_a, len_a); | |
2584 | if (!p) | |
2585 | return buf_b; /* No full PSB+ => assume no overlap */ | |
2586 | len = len_a - (p - buf_a); | |
2587 | if (!intel_pt_next_tsc(p, len, &tsc_a)) | |
2588 | return buf_b; /* No TSC in buf_a => assume no overlap */ | |
2589 | } | |
2590 | ||
2591 | while (1) { | |
2592 | /* Ignore PSB+ with no TSC */ | |
2593 | if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) && | |
2594 | intel_pt_tsc_cmp(tsc_a, tsc_b) < 0) | |
2595 | return buf_b; /* tsc_a < tsc_b => no overlap */ | |
2596 | ||
2597 | if (!intel_pt_step_psb(&buf_b, &len_b)) | |
2598 | return buf_b + len_b; /* No PSB in buf_b => no data */ | |
2599 | } | |
2600 | } | |
2601 | ||
2602 | /** | |
2603 | * intel_pt_find_overlap - determine start of non-overlapped trace data. | |
2604 | * @buf_a: first buffer | |
2605 | * @len_a: size of first buffer | |
2606 | * @buf_b: second buffer | |
2607 | * @len_b: size of second buffer | |
2608 | * @have_tsc: can use TSC packets to detect overlap | |
2609 | * | |
2610 | * When trace samples or snapshots are recorded there is the possibility that | |
2611 | * the data overlaps. Note that, for the purposes of decoding, data is only | |
2612 | * useful if it begins with a PSB packet. | |
2613 | * | |
2614 | * Return: A pointer into @buf_b from where non-overlapped data starts, or | |
2615 | * @buf_b + @len_b if there is no non-overlapped data. | |
2616 | */ | |
2617 | unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, | |
2618 | unsigned char *buf_b, size_t len_b, | |
2619 | bool have_tsc) | |
2620 | { | |
2621 | unsigned char *found; | |
2622 | ||
2623 | /* Buffer 'b' must start at PSB so throw away everything before that */ | |
2624 | if (!intel_pt_next_psb(&buf_b, &len_b)) | |
2625 | return buf_b + len_b; /* No PSB */ | |
2626 | ||
2627 | if (!intel_pt_next_psb(&buf_a, &len_a)) | |
2628 | return buf_b; /* No overlap */ | |
2629 | ||
2630 | if (have_tsc) { | |
2631 | found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b); | |
2632 | if (found) | |
2633 | return found; | |
2634 | } | |
2635 | ||
2636 | /* | |
2637 | * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, | |
2638 | * we can ignore the first part of buffer 'a'. | |
2639 | */ | |
2640 | while (len_b < len_a) { | |
2641 | if (!intel_pt_step_psb(&buf_a, &len_a)) | |
2642 | return buf_b; /* No overlap */ | |
2643 | } | |
2644 | ||
2645 | /* Now len_b >= len_a */ | |
2646 | if (len_b > len_a) { | |
2647 | /* The leftover buffer 'b' must start at a PSB */ | |
2648 | while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { | |
2649 | if (!intel_pt_step_psb(&buf_a, &len_a)) | |
2650 | return buf_b; /* No overlap */ | |
2651 | } | |
2652 | } | |
2653 | ||
2654 | while (1) { | |
2655 | /* Potential overlap so check the bytes */ | |
2656 | found = memmem(buf_a, len_a, buf_b, len_a); | |
2657 | if (found) | |
2658 | return buf_b + len_a; | |
2659 | ||
2660 | /* Try again at next PSB in buffer 'a' */ | |
2661 | if (!intel_pt_step_psb(&buf_a, &len_a)) | |
2662 | return buf_b; /* No overlap */ | |
2663 | ||
2664 | /* The leftover buffer 'b' must start at a PSB */ | |
2665 | while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) { | |
2666 | if (!intel_pt_step_psb(&buf_a, &len_a)) | |
2667 | return buf_b; /* No overlap */ | |
2668 | } | |
2669 | } | |
2670 | } |