tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_pt_decoder.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2014, Intel Corporation.
   5  */
   6
   7 #ifndef _GNU_SOURCE
   8 #define _GNU_SOURCE
   9 #endif
  10 #include <stdlib.h>
  11 #include <stdbool.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <linux/compiler.h>
  17 #include <linux/string.h>
  18 #include <linux/zalloc.h>
  19
  20 #include "../auxtrace.h"
  21
  22 #include "intel-pt-insn-decoder.h"
  23 #include "intel-pt-pkt-decoder.h"
  24 #include "intel-pt-decoder.h"
  25 #include "intel-pt-log.h"
  26
  27 #define BITULL(x) (1ULL << (x))
  28
  29 /* IA32_RTIT_CTL MSR bits */
  30 #define INTEL_PT_CYC_ENABLE             BITULL(1)
  31 #define INTEL_PT_CYC_THRESHOLD          (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
  32 #define INTEL_PT_CYC_THRESHOLD_SHIFT    19
  33
  34 #define INTEL_PT_BLK_SIZE 1024
  35
  36 #define BIT63 (((uint64_t)1 << 63))
  37
  38 #define SEVEN_BYTES 0xffffffffffffffULL
  39
  40 #define NO_VMCS 0xffffffffffULL
  41
  42 #define INTEL_PT_RETURN 1
  43
  44 /*
  45  * Default maximum number of loops with no packets consumed i.e. stuck in a
  46  * loop.
  47  */
  48 #define INTEL_PT_MAX_LOOPS 100000
  49
  50 struct intel_pt_blk {
  51         struct intel_pt_blk *prev;
  52         uint64_t ip[INTEL_PT_BLK_SIZE];
  53 };
  54
  55 struct intel_pt_stack {
  56         struct intel_pt_blk *blk;
  57         struct intel_pt_blk *spare;
  58         int pos;
  59 };
  60
  61 enum intel_pt_p_once {
  62         INTEL_PT_PRT_ONCE_UNK_VMCS,
  63         INTEL_PT_PRT_ONCE_ERANGE,
  64 };
  65
  66 enum intel_pt_pkt_state {
  67         INTEL_PT_STATE_NO_PSB,
  68         INTEL_PT_STATE_NO_IP,
  69         INTEL_PT_STATE_ERR_RESYNC,
  70         INTEL_PT_STATE_IN_SYNC,
  71         INTEL_PT_STATE_TNT_CONT,
  72         INTEL_PT_STATE_TNT,
  73         INTEL_PT_STATE_TIP,
  74         INTEL_PT_STATE_TIP_PGD,
  75         INTEL_PT_STATE_FUP,
  76         INTEL_PT_STATE_FUP_NO_TIP,
  77         INTEL_PT_STATE_FUP_IN_PSB,
  78         INTEL_PT_STATE_RESAMPLE,
  79         INTEL_PT_STATE_VM_TIME_CORRELATION,
  80 };
  81
  82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  83 {
  84         switch (pkt_state) {
  85         case INTEL_PT_STATE_NO_PSB:
  86         case INTEL_PT_STATE_NO_IP:
  87         case INTEL_PT_STATE_ERR_RESYNC:
  88         case INTEL_PT_STATE_IN_SYNC:
  89         case INTEL_PT_STATE_TNT_CONT:
  90         case INTEL_PT_STATE_RESAMPLE:
  91         case INTEL_PT_STATE_VM_TIME_CORRELATION:
  92                 return true;
  93         case INTEL_PT_STATE_TNT:
  94         case INTEL_PT_STATE_TIP:
  95         case INTEL_PT_STATE_TIP_PGD:
  96         case INTEL_PT_STATE_FUP:
  97         case INTEL_PT_STATE_FUP_NO_TIP:
  98         case INTEL_PT_STATE_FUP_IN_PSB:
  99                 return false;
 100         default:
 101                 return true;
 102         };
 103 }
 104
 105 #ifdef INTEL_PT_STRICT
 106 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
 107 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
 108 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
 109 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
 110 #else
 111 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
 112 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
 113 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
 114 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
 115 #endif
 116
 117 struct intel_pt_decoder {
 118         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 119         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 120                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 121                          uint64_t max_insn_cnt, void *data);
 122         bool (*pgd_ip)(uint64_t ip, void *data);
 123         int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
 124         struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);
 125         void *data;
 126         struct intel_pt_state state;
 127         const unsigned char *buf;
 128         size_t len;
 129         bool return_compression;
 130         bool branch_enable;
 131         bool mtc_insn;
 132         bool pge;
 133         bool have_tma;
 134         bool have_cyc;
 135         bool fixup_last_mtc;
 136         bool have_last_ip;
 137         bool in_psb;
 138         bool hop;
 139         bool leap;
 140         bool emulated_ptwrite;
 141         bool vm_time_correlation;
 142         bool vm_tm_corr_dry_run;
 143         bool vm_tm_corr_reliable;
 144         bool vm_tm_corr_same_buf;
 145         bool vm_tm_corr_continuous;
 146         bool nr;
 147         bool next_nr;
 148         bool iflag;
 149         bool next_iflag;
 150         enum intel_pt_param_flags flags;
 151         uint64_t pos;
 152         uint64_t last_ip;
 153         uint64_t ip;
 154         uint64_t pip_payload;
 155         uint64_t timestamp;
 156         uint64_t tsc_timestamp;
 157         uint64_t ref_timestamp;
 158         uint64_t buf_timestamp;
 159         uint64_t sample_timestamp;
 160         uint64_t ret_addr;
 161         uint64_t ctc_timestamp;
 162         uint64_t ctc_delta;
 163         uint64_t cycle_cnt;
 164         uint64_t cyc_ref_timestamp;
 165         uint64_t first_timestamp;
 166         uint64_t last_reliable_timestamp;
 167         uint64_t vmcs;
 168         uint64_t print_once;
 169         uint64_t last_ctc;
 170         uint32_t last_mtc;
 171         uint32_t tsc_ctc_ratio_n;
 172         uint32_t tsc_ctc_ratio_d;
 173         uint32_t tsc_ctc_mult;
 174         uint32_t tsc_slip;
 175         uint32_t ctc_rem_mask;
 176         int mtc_shift;
 177         struct intel_pt_stack stack;
 178         enum intel_pt_pkt_state pkt_state;
 179         enum intel_pt_pkt_ctx pkt_ctx;
 180         enum intel_pt_pkt_ctx prev_pkt_ctx;
 181         enum intel_pt_blk_type blk_type;
 182         int blk_type_pos;
 183         struct intel_pt_pkt packet;
 184         struct intel_pt_pkt tnt;
 185         int pkt_step;
 186         int pkt_len;
 187         int last_packet_type;
 188         unsigned int cbr;
 189         unsigned int cbr_seen;
 190         unsigned int max_non_turbo_ratio;
 191         double max_non_turbo_ratio_fp;
 192         double cbr_cyc_to_tsc;
 193         double calc_cyc_to_tsc;
 194         bool have_calc_cyc_to_tsc;
 195         int exec_mode;
 196         unsigned int insn_bytes;
 197         uint64_t period;
 198         enum intel_pt_period_type period_type;
 199         uint64_t tot_insn_cnt;
 200         uint64_t period_insn_cnt;
 201         uint64_t period_mask;
 202         uint64_t period_ticks;
 203         uint64_t last_masked_timestamp;
 204         uint64_t tot_cyc_cnt;
 205         uint64_t sample_tot_cyc_cnt;
 206         uint64_t base_cyc_cnt;
 207         uint64_t cyc_cnt_timestamp;
 208         uint64_t ctl;
 209         uint64_t cyc_threshold;
 210         double tsc_to_cyc;
 211         bool continuous_period;
 212         bool overflow;
 213         bool set_fup_tx_flags;
 214         bool set_fup_ptw;
 215         bool set_fup_mwait;
 216         bool set_fup_pwre;
 217         bool set_fup_exstop;
 218         bool set_fup_bep;
 219         bool set_fup_cfe_ip;
 220         bool set_fup_cfe;
 221         bool set_fup_mode_exec;
 222         bool sample_cyc;
 223         unsigned int fup_tx_flags;
 224         unsigned int tx_flags;
 225         uint64_t fup_ptw_payload;
 226         uint64_t fup_mwait_payload;
 227         uint64_t fup_pwre_payload;
 228         uint64_t cbr_payload;
 229         uint64_t timestamp_insn_cnt;
 230         uint64_t sample_insn_cnt;
 231         uint64_t stuck_ip;
 232         struct intel_pt_pkt fup_cfe_pkt;
 233         int max_loops;
 234         int no_progress;
 235         int stuck_ip_prd;
 236         int stuck_ip_cnt;
 237         uint64_t psb_ip;
 238         const unsigned char *next_buf;
 239         size_t next_len;
 240         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 241         int evd_cnt;
 242         struct intel_pt_evd evd[INTEL_PT_MAX_EVDS];
 243 };
 244
 245 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 246 {
 247         int i;
 248
 249         for (i = 0; x != 1; i++)
 250                 x >>= 1;
 251
 252         return x << i;
 253 }
 254
 255 __printf(1, 2)
 256 static void p_log(const char *fmt, ...)
 257 {
 258         char buf[512];
 259         va_list args;
 260
 261         va_start(args, fmt);
 262         vsnprintf(buf, sizeof(buf), fmt, args);
 263         va_end(args);
 264
 265         fprintf(stderr, "%s\n", buf);
 266         intel_pt_log("%s\n", buf);
 267 }
 268
 269 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
 270                                 enum intel_pt_p_once id)
 271 {
 272         uint64_t bit = 1ULL << id;
 273
 274         if (decoder->print_once & bit)
 275                 return false;
 276         decoder->print_once |= bit;
 277         return true;
 278 }
 279
 280 static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
 281 {
 282         if (!(ctl & INTEL_PT_CYC_ENABLE))
 283                 return 0;
 284
 285         return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
 286 }
 287
 288 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 289 {
 290         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 291                 uint64_t period;
 292
 293                 period = intel_pt_lower_power_of_2(decoder->period);
 294                 decoder->period_mask  = ~(period - 1);
 295                 decoder->period_ticks = period;
 296         }
 297 }
 298
 299 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 300 {
 301         if (!d)
 302                 return 0;
 303         return (t / d) * n + ((t % d) * n) / d;
 304 }
 305
 306 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 307 {
 308         struct intel_pt_decoder *decoder;
 309
 310         if (!params->get_trace || !params->walk_insn)
 311                 return NULL;
 312
 313         decoder = zalloc(sizeof(struct intel_pt_decoder));
 314         if (!decoder)
 315                 return NULL;
 316
 317         decoder->get_trace          = params->get_trace;
 318         decoder->walk_insn          = params->walk_insn;
 319         decoder->pgd_ip             = params->pgd_ip;
 320         decoder->lookahead          = params->lookahead;
 321         decoder->findnew_vmcs_info  = params->findnew_vmcs_info;
 322         decoder->data               = params->data;
 323         decoder->return_compression = params->return_compression;
 324         decoder->branch_enable      = params->branch_enable;
 325         decoder->hop                = params->quick >= 1;
 326         decoder->leap               = params->quick >= 2;
 327         decoder->vm_time_correlation = params->vm_time_correlation;
 328         decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
 329         decoder->first_timestamp    = params->first_timestamp;
 330         decoder->last_reliable_timestamp = params->first_timestamp;
 331         decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
 332
 333         decoder->flags              = params->flags;
 334
 335         decoder->ctl                = params->ctl;
 336         decoder->period             = params->period;
 337         decoder->period_type        = params->period_type;
 338
 339         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 340         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 341
 342         decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
 343
 344         intel_pt_setup_period(decoder);
 345
 346         decoder->mtc_shift = params->mtc_period;
 347         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 348
 349         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 350         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 351
 352         if (!decoder->tsc_ctc_ratio_n)
 353                 decoder->tsc_ctc_ratio_d = 0;
 354
 355         if (decoder->tsc_ctc_ratio_d) {
 356                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 357                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 358                                                 decoder->tsc_ctc_ratio_d;
 359         }
 360
 361         /*
 362          * A TSC packet can slip past MTC packets so that the timestamp appears
 363          * to go backwards. One estimate is that can be up to about 40 CPU
 364          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 365          * slippage an order of magnitude more to be on the safe side.
 366          */
 367         decoder->tsc_slip = 0x10000;
 368
 369         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 370         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 371         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 372         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 373         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 374
 375         if (decoder->hop)
 376                 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
 377
 378         return decoder;
 379 }
 380
 381 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
 382                                   uint64_t first_timestamp)
 383 {
 384         decoder->first_timestamp = first_timestamp;
 385 }
 386
 387 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 388 {
 389         struct intel_pt_blk *blk = stack->blk;
 390
 391         stack->blk = blk->prev;
 392         if (!stack->spare)
 393                 stack->spare = blk;
 394         else
 395                 free(blk);
 396 }
 397
 398 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 399 {
 400         if (!stack->pos) {
 401                 if (!stack->blk)
 402                         return 0;
 403                 intel_pt_pop_blk(stack);
 404                 if (!stack->blk)
 405                         return 0;
 406                 stack->pos = INTEL_PT_BLK_SIZE;
 407         }
 408         return stack->blk->ip[--stack->pos];
 409 }
 410
 411 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 412 {
 413         struct intel_pt_blk *blk;
 414
 415         if (stack->spare) {
 416                 blk = stack->spare;
 417                 stack->spare = NULL;
 418         } else {
 419                 blk = malloc(sizeof(struct intel_pt_blk));
 420                 if (!blk)
 421                         return -ENOMEM;
 422         }
 423
 424         blk->prev = stack->blk;
 425         stack->blk = blk;
 426         stack->pos = 0;
 427         return 0;
 428 }
 429
 430 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 431 {
 432         int err;
 433
 434         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 435                 err = intel_pt_alloc_blk(stack);
 436                 if (err)
 437                         return err;
 438         }
 439
 440         stack->blk->ip[stack->pos++] = ip;
 441         return 0;
 442 }
 443
 444 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 445 {
 446         while (stack->blk)
 447                 intel_pt_pop_blk(stack);
 448         stack->pos = 0;
 449 }
 450
 451 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 452 {
 453         intel_pt_clear_stack(stack);
 454         zfree(&stack->blk);
 455         zfree(&stack->spare);
 456 }
 457
 458 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 459 {
 460         intel_pt_free_stack(&decoder->stack);
 461         free(decoder);
 462 }
 463
 464 static int intel_pt_ext_err(int code)
 465 {
 466         switch (code) {
 467         case -ENOMEM:
 468                 return INTEL_PT_ERR_NOMEM;
 469         case -ENOSYS:
 470                 return INTEL_PT_ERR_INTERN;
 471         case -EBADMSG:
 472                 return INTEL_PT_ERR_BADPKT;
 473         case -ENODATA:
 474                 return INTEL_PT_ERR_NODATA;
 475         case -EILSEQ:
 476                 return INTEL_PT_ERR_NOINSN;
 477         case -ENOENT:
 478                 return INTEL_PT_ERR_MISMAT;
 479         case -EOVERFLOW:
 480                 return INTEL_PT_ERR_OVR;
 481         case -ENOSPC:
 482                 return INTEL_PT_ERR_LOST;
 483         case -ELOOP:
 484                 return INTEL_PT_ERR_NELOOP;
 485         case -ECONNRESET:
 486                 return INTEL_PT_ERR_EPTW;
 487         default:
 488                 return INTEL_PT_ERR_UNK;
 489         }
 490 }
 491
 492 static const char *intel_pt_err_msgs[] = {
 493         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 494         [INTEL_PT_ERR_INTERN] = "Internal error",
 495         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 496         [INTEL_PT_ERR_NODATA] = "No more data",
 497         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 498         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 499         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 500         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 501         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 502         [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
 503         [INTEL_PT_ERR_EPTW]   = "Broken emulated ptwrite",
 504 };
 505
 506 int intel_pt__strerror(int code, char *buf, size_t buflen)
 507 {
 508         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 509                 code = INTEL_PT_ERR_UNK;
 510         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 511         return 0;
 512 }
 513
 514 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 515                                  uint64_t last_ip)
 516 {
 517         uint64_t ip;
 518
 519         switch (packet->count) {
 520         case 1:
 521                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 522                      packet->payload;
 523                 break;
 524         case 2:
 525                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 526                      packet->payload;
 527                 break;
 528         case 3:
 529                 ip = packet->payload;
 530                 /* Sign-extend 6-byte ip */
 531                 if (ip & (uint64_t)0x800000000000ULL)
 532                         ip |= (uint64_t)0xffff000000000000ULL;
 533                 break;
 534         case 4:
 535                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 536                      packet->payload;
 537                 break;
 538         case 6:
 539                 ip = packet->payload;
 540                 break;
 541         default:
 542                 return 0;
 543         }
 544
 545         return ip;
 546 }
 547
 548 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 549 {
 550         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 551         decoder->have_last_ip = true;
 552 }
 553
 554 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 555 {
 556         intel_pt_set_last_ip(decoder);
 557         decoder->ip = decoder->last_ip;
 558 }
 559
 560 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 561 {
 562         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 563                             decoder->buf);
 564 }
 565
 566 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 567 {
 568         intel_pt_log("ERROR: Internal error\n");
 569         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 570         return -ENOSYS;
 571 }
 572
 573 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 574 {
 575         decoder->tx_flags = 0;
 576 }
 577
 578 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 579 {
 580         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 581 }
 582
 583 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
 584 {
 585         decoder->pip_payload = decoder->packet.payload;
 586 }
 587
 588 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
 589 {
 590         decoder->next_nr = decoder->pip_payload & 1;
 591 }
 592
 593 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
 594 {
 595         decoder->nr = decoder->pip_payload & 1;
 596         decoder->next_nr = decoder->nr;
 597 }
 598
 599 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
 600 {
 601         intel_pt_update_pip(decoder);
 602         intel_pt_set_nr(decoder);
 603 }
 604
 605 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 606 {
 607         intel_pt_clear_tx_flags(decoder);
 608         decoder->have_tma = false;
 609         decoder->pkt_len = 1;
 610         decoder->pkt_step = 1;
 611         intel_pt_decoder_log_packet(decoder);
 612         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 613                 intel_pt_log("ERROR: Bad packet\n");
 614                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 615         }
 616         return -EBADMSG;
 617 }
 618
 619 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
 620 {
 621         decoder->sample_timestamp = decoder->timestamp;
 622         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
 623         decoder->state.cycles = decoder->tot_cyc_cnt;
 624 }
 625
 626 static void intel_pt_reposition(struct intel_pt_decoder *decoder)
 627 {
 628         decoder->ip = 0;
 629         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 630         decoder->timestamp = 0;
 631         decoder->have_tma = false;
 632 }
 633
 634 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
 635 {
 636         struct intel_pt_buffer buffer = { .buf = 0, };
 637         int ret;
 638
 639         decoder->pkt_step = 0;
 640
 641         intel_pt_log("Getting more data\n");
 642         ret = decoder->get_trace(&buffer, decoder->data);
 643         if (ret)
 644                 return ret;
 645         decoder->buf = buffer.buf;
 646         decoder->len = buffer.len;
 647         if (!decoder->len) {
 648                 intel_pt_log("No more data\n");
 649                 return -ENODATA;
 650         }
 651         decoder->buf_timestamp = buffer.ref_timestamp;
 652         if (!buffer.consecutive || reposition) {
 653                 intel_pt_reposition(decoder);
 654                 decoder->ref_timestamp = buffer.ref_timestamp;
 655                 decoder->state.trace_nr = buffer.trace_nr;
 656                 decoder->vm_tm_corr_same_buf = false;
 657                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 658                              decoder->ref_timestamp);
 659                 return -ENOLINK;
 660         }
 661
 662         return 0;
 663 }
 664
 665 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
 666                                   bool reposition)
 667 {
 668         if (!decoder->next_buf)
 669                 return intel_pt_get_data(decoder, reposition);
 670
 671         decoder->buf = decoder->next_buf;
 672         decoder->len = decoder->next_len;
 673         decoder->next_buf = 0;
 674         decoder->next_len = 0;
 675         return 0;
 676 }
 677
 678 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 679 {
 680         unsigned char *buf = decoder->temp_buf;
 681         size_t old_len, len, n;
 682         int ret;
 683
 684         old_len = decoder->len;
 685         len = decoder->len;
 686         memcpy(buf, decoder->buf, len);
 687
 688         ret = intel_pt_get_data(decoder, false);
 689         if (ret) {
 690                 decoder->pos += old_len;
 691                 return ret < 0 ? ret : -EINVAL;
 692         }
 693
 694         n = INTEL_PT_PKT_MAX_SZ - len;
 695         if (n > decoder->len)
 696                 n = decoder->len;
 697         memcpy(buf + len, decoder->buf, n);
 698         len += n;
 699
 700         decoder->prev_pkt_ctx = decoder->pkt_ctx;
 701         ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
 702         if (ret < (int)old_len) {
 703                 decoder->next_buf = decoder->buf;
 704                 decoder->next_len = decoder->len;
 705                 decoder->buf = buf;
 706                 decoder->len = old_len;
 707                 return intel_pt_bad_packet(decoder);
 708         }
 709
 710         decoder->next_buf = decoder->buf + (ret - old_len);
 711         decoder->next_len = decoder->len - (ret - old_len);
 712
 713         decoder->buf = buf;
 714         decoder->len = ret;
 715
 716         return ret;
 717 }
 718
 719 struct intel_pt_pkt_info {
 720         struct intel_pt_decoder   *decoder;
 721         struct intel_pt_pkt       packet;
 722         uint64_t                  pos;
 723         int                       pkt_len;
 724         int                       last_packet_type;
 725         void                      *data;
 726 };
 727
 728 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 729
 730 /* Lookahead packets in current buffer */
 731 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 732                                   intel_pt_pkt_cb_t cb, void *data)
 733 {
 734         struct intel_pt_pkt_info pkt_info;
 735         const unsigned char *buf = decoder->buf;
 736         enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
 737         size_t len = decoder->len;
 738         int ret;
 739
 740         pkt_info.decoder          = decoder;
 741         pkt_info.pos              = decoder->pos;
 742         pkt_info.pkt_len          = decoder->pkt_step;
 743         pkt_info.last_packet_type = decoder->last_packet_type;
 744         pkt_info.data             = data;
 745
 746         while (1) {
 747                 do {
 748                         pkt_info.pos += pkt_info.pkt_len;
 749                         buf          += pkt_info.pkt_len;
 750                         len          -= pkt_info.pkt_len;
 751
 752                         if (!len)
 753                                 return INTEL_PT_NEED_MORE_BYTES;
 754
 755                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
 756                                                   &pkt_ctx);
 757                         if (!ret)
 758                                 return INTEL_PT_NEED_MORE_BYTES;
 759                         if (ret < 0)
 760                                 return ret;
 761
 762                         pkt_info.pkt_len = ret;
 763                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 764
 765                 ret = cb(&pkt_info);
 766                 if (ret)
 767                         return 0;
 768
 769                 pkt_info.last_packet_type = pkt_info.packet.type;
 770         }
 771 }
 772
 773 struct intel_pt_calc_cyc_to_tsc_info {
 774         uint64_t        cycle_cnt;
 775         unsigned int    cbr;
 776         uint32_t        last_mtc;
 777         uint64_t        ctc_timestamp;
 778         uint64_t        ctc_delta;
 779         uint64_t        tsc_timestamp;
 780         uint64_t        timestamp;
 781         bool            have_tma;
 782         bool            fixup_last_mtc;
 783         bool            from_mtc;
 784         double          cbr_cyc_to_tsc;
 785 };
 786
 787 /*
 788  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 789  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 790  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 791  * packet by copying the missing bits from the current MTC assuming the least
 792  * difference between the two, and that the current MTC comes after last_mtc.
 793  */
 794 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 795                                     uint32_t *last_mtc)
 796 {
 797         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 798         uint32_t mask = ~(first_missing_bit - 1);
 799
 800         *last_mtc |= mtc & mask;
 801         if (*last_mtc >= mtc) {
 802                 *last_mtc -= first_missing_bit;
 803                 *last_mtc &= 0xff;
 804         }
 805 }
 806
 807 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 808 {
 809         struct intel_pt_decoder *decoder = pkt_info->decoder;
 810         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 811         uint64_t timestamp;
 812         double cyc_to_tsc;
 813         unsigned int cbr;
 814         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 815
 816         switch (pkt_info->packet.type) {
 817         case INTEL_PT_TNT:
 818         case INTEL_PT_TIP_PGE:
 819         case INTEL_PT_TIP:
 820         case INTEL_PT_FUP:
 821         case INTEL_PT_PSB:
 822         case INTEL_PT_PIP:
 823         case INTEL_PT_MODE_EXEC:
 824         case INTEL_PT_MODE_TSX:
 825         case INTEL_PT_PSBEND:
 826         case INTEL_PT_PAD:
 827         case INTEL_PT_VMCS:
 828         case INTEL_PT_MNT:
 829         case INTEL_PT_PTWRITE:
 830         case INTEL_PT_PTWRITE_IP:
 831         case INTEL_PT_BBP:
 832         case INTEL_PT_BIP:
 833         case INTEL_PT_BEP:
 834         case INTEL_PT_BEP_IP:
 835         case INTEL_PT_CFE:
 836         case INTEL_PT_CFE_IP:
 837         case INTEL_PT_EVD:
 838                 return 0;
 839
 840         case INTEL_PT_MTC:
 841                 if (!data->have_tma)
 842                         return 0;
 843
 844                 mtc = pkt_info->packet.payload;
 845                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 846                         data->fixup_last_mtc = false;
 847                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 848                                                 &data->last_mtc);
 849                 }
 850                 if (mtc > data->last_mtc)
 851                         mtc_delta = mtc - data->last_mtc;
 852                 else
 853                         mtc_delta = mtc + 256 - data->last_mtc;
 854                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 855                 data->last_mtc = mtc;
 856
 857                 if (decoder->tsc_ctc_mult) {
 858                         timestamp = data->ctc_timestamp +
 859                                 data->ctc_delta * decoder->tsc_ctc_mult;
 860                 } else {
 861                         timestamp = data->ctc_timestamp +
 862                                 multdiv(data->ctc_delta,
 863                                         decoder->tsc_ctc_ratio_n,
 864                                         decoder->tsc_ctc_ratio_d);
 865                 }
 866
 867                 if (timestamp < data->timestamp)
 868                         return 1;
 869
 870                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 871                         data->timestamp = timestamp;
 872                         return 0;
 873                 }
 874
 875                 break;
 876
 877         case INTEL_PT_TSC:
 878                 /*
 879                  * For now, do not support using TSC packets - refer
 880                  * intel_pt_calc_cyc_to_tsc().
 881                  */
 882                 if (data->from_mtc)
 883                         return 1;
 884                 timestamp = pkt_info->packet.payload |
 885                             (data->timestamp & (0xffULL << 56));
 886                 if (data->from_mtc && timestamp < data->timestamp &&
 887                     data->timestamp - timestamp < decoder->tsc_slip)
 888                         return 1;
 889                 if (timestamp < data->timestamp)
 890                         timestamp += (1ULL << 56);
 891                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 892                         if (data->from_mtc)
 893                                 return 1;
 894                         data->tsc_timestamp = timestamp;
 895                         data->timestamp = timestamp;
 896                         return 0;
 897                 }
 898                 break;
 899
 900         case INTEL_PT_TMA:
 901                 if (data->from_mtc)
 902                         return 1;
 903
 904                 if (!decoder->tsc_ctc_ratio_d)
 905                         return 0;
 906
 907                 ctc = pkt_info->packet.payload;
 908                 fc = pkt_info->packet.count;
 909                 ctc_rem = ctc & decoder->ctc_rem_mask;
 910
 911                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 912
 913                 data->ctc_timestamp = data->tsc_timestamp - fc;
 914                 if (decoder->tsc_ctc_mult) {
 915                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 916                 } else {
 917                         data->ctc_timestamp -=
 918                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 919                                         decoder->tsc_ctc_ratio_d);
 920                 }
 921
 922                 data->ctc_delta = 0;
 923                 data->have_tma = true;
 924                 data->fixup_last_mtc = true;
 925
 926                 return 0;
 927
 928         case INTEL_PT_CYC:
 929                 data->cycle_cnt += pkt_info->packet.payload;
 930                 return 0;
 931
 932         case INTEL_PT_CBR:
 933                 cbr = pkt_info->packet.payload;
 934                 if (data->cbr && data->cbr != cbr)
 935                         return 1;
 936                 data->cbr = cbr;
 937                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 938                 return 0;
 939
 940         case INTEL_PT_TIP_PGD:
 941         case INTEL_PT_TRACESTOP:
 942         case INTEL_PT_EXSTOP:
 943         case INTEL_PT_EXSTOP_IP:
 944         case INTEL_PT_MWAIT:
 945         case INTEL_PT_PWRE:
 946         case INTEL_PT_PWRX:
 947         case INTEL_PT_OVF:
 948         case INTEL_PT_BAD: /* Does not happen */
 949         default:
 950                 return 1;
 951         }
 952
 953         if (!data->cbr && decoder->cbr) {
 954                 data->cbr = decoder->cbr;
 955                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 956         }
 957
 958         if (!data->cycle_cnt)
 959                 return 1;
 960
 961         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 962
 963         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 964             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 965                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 966                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 967                 return 1;
 968         }
 969
 970         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 971         decoder->have_calc_cyc_to_tsc = true;
 972
 973         if (data->cbr) {
 974                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 975                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 976         } else {
 977                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 978                              cyc_to_tsc, pkt_info->pos);
 979         }
 980
 981         return 1;
 982 }
 983
 984 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 985                                      bool from_mtc)
 986 {
 987         struct intel_pt_calc_cyc_to_tsc_info data = {
 988                 .cycle_cnt      = 0,
 989                 .cbr            = 0,
 990                 .last_mtc       = decoder->last_mtc,
 991                 .ctc_timestamp  = decoder->ctc_timestamp,
 992                 .ctc_delta      = decoder->ctc_delta,
 993                 .tsc_timestamp  = decoder->tsc_timestamp,
 994                 .timestamp      = decoder->timestamp,
 995                 .have_tma       = decoder->have_tma,
 996                 .fixup_last_mtc = decoder->fixup_last_mtc,
 997                 .from_mtc       = from_mtc,
 998                 .cbr_cyc_to_tsc = 0,
 999         };
1000
1001         /*
1002          * For now, do not support using TSC packets for at least the reasons:
1003          * 1) timing might have stopped
1004          * 2) TSC packets within PSB+ can slip against CYC packets
1005          */
1006         if (!from_mtc)
1007                 return;
1008
1009         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
1010 }
1011
1012 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
1013 {
1014         int ret;
1015
1016         decoder->last_packet_type = decoder->packet.type;
1017
1018         do {
1019                 decoder->pos += decoder->pkt_step;
1020                 decoder->buf += decoder->pkt_step;
1021                 decoder->len -= decoder->pkt_step;
1022
1023                 if (!decoder->len) {
1024                         ret = intel_pt_get_next_data(decoder, false);
1025                         if (ret)
1026                                 return ret;
1027                 }
1028
1029                 decoder->prev_pkt_ctx = decoder->pkt_ctx;
1030                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
1031                                           &decoder->packet, &decoder->pkt_ctx);
1032                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1033                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1034                         ret = intel_pt_get_split_packet(decoder);
1035                         if (ret < 0)
1036                                 return ret;
1037                 }
1038                 if (ret <= 0)
1039                         return intel_pt_bad_packet(decoder);
1040
1041                 decoder->pkt_len = ret;
1042                 decoder->pkt_step = ret;
1043                 intel_pt_decoder_log_packet(decoder);
1044         } while (decoder->packet.type == INTEL_PT_PAD);
1045
1046         return 0;
1047 }
1048
1049 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1050 {
1051         uint64_t timestamp, masked_timestamp;
1052
1053         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1054         masked_timestamp = timestamp & decoder->period_mask;
1055         if (decoder->continuous_period) {
1056                 if (masked_timestamp > decoder->last_masked_timestamp)
1057                         return 1;
1058         } else {
1059                 timestamp += 1;
1060                 masked_timestamp = timestamp & decoder->period_mask;
1061                 if (masked_timestamp > decoder->last_masked_timestamp) {
1062                         decoder->last_masked_timestamp = masked_timestamp;
1063                         decoder->continuous_period = true;
1064                 }
1065         }
1066
1067         if (masked_timestamp < decoder->last_masked_timestamp)
1068                 return decoder->period_ticks;
1069
1070         return decoder->period_ticks - (timestamp - masked_timestamp);
1071 }
1072
1073 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1074 {
1075         switch (decoder->period_type) {
1076         case INTEL_PT_PERIOD_INSTRUCTIONS:
1077                 return decoder->period - decoder->period_insn_cnt;
1078         case INTEL_PT_PERIOD_TICKS:
1079                 return intel_pt_next_period(decoder);
1080         case INTEL_PT_PERIOD_NONE:
1081         case INTEL_PT_PERIOD_MTC:
1082         default:
1083                 return 0;
1084         }
1085 }
1086
1087 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1088 {
1089         uint64_t timestamp, masked_timestamp;
1090
1091         switch (decoder->period_type) {
1092         case INTEL_PT_PERIOD_INSTRUCTIONS:
1093                 decoder->period_insn_cnt = 0;
1094                 break;
1095         case INTEL_PT_PERIOD_TICKS:
1096                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1097                 masked_timestamp = timestamp & decoder->period_mask;
1098                 if (masked_timestamp > decoder->last_masked_timestamp)
1099                         decoder->last_masked_timestamp = masked_timestamp;
1100                 else
1101                         decoder->last_masked_timestamp += decoder->period_ticks;
1102                 break;
1103         case INTEL_PT_PERIOD_NONE:
1104         case INTEL_PT_PERIOD_MTC:
1105         default:
1106                 break;
1107         }
1108
1109         decoder->state.type |= INTEL_PT_INSTRUCTION;
1110 }
1111
1112 /*
1113  * Sample FUP instruction at the same time as reporting the FUP event, so the
1114  * instruction sample gets the same flags as the FUP event.
1115  */
1116 static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder)
1117 {
1118         struct intel_pt_insn intel_pt_insn;
1119         uint64_t max_insn_cnt, insn_cnt = 0;
1120         int err;
1121
1122         decoder->state.insn_op = INTEL_PT_OP_OTHER;
1123         decoder->state.insn_len = 0;
1124
1125         if (!decoder->branch_enable || !decoder->pge || decoder->hop ||
1126             decoder->ip != decoder->last_ip)
1127                 return;
1128
1129         if (!decoder->mtc_insn)
1130                 decoder->mtc_insn = true;
1131
1132         max_insn_cnt = intel_pt_next_sample(decoder);
1133         if (max_insn_cnt != 1)
1134                 return;
1135
1136         err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip,
1137                                  0, max_insn_cnt, decoder->data);
1138         /* Ignore error, it will be reported next walk anyway */
1139         if (err)
1140                 return;
1141
1142         if (intel_pt_insn.branch != INTEL_PT_BR_NO_BRANCH) {
1143                 intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip);
1144                 return;
1145         }
1146
1147         decoder->tot_insn_cnt += insn_cnt;
1148         decoder->timestamp_insn_cnt += insn_cnt;
1149         decoder->sample_insn_cnt += insn_cnt;
1150         decoder->period_insn_cnt += insn_cnt;
1151
1152         intel_pt_sample_insn(decoder);
1153
1154         decoder->state.type |= INTEL_PT_INSTRUCTION;
1155         decoder->ip += intel_pt_insn.length;
1156 }
1157
1158 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1159                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1160 {
1161         uint64_t max_insn_cnt, insn_cnt = 0;
1162         int err;
1163
1164         if (!decoder->mtc_insn)
1165                 decoder->mtc_insn = true;
1166
1167         max_insn_cnt = intel_pt_next_sample(decoder);
1168
1169         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1170                                  max_insn_cnt, decoder->data);
1171
1172         decoder->tot_insn_cnt += insn_cnt;
1173         decoder->timestamp_insn_cnt += insn_cnt;
1174         decoder->sample_insn_cnt += insn_cnt;
1175         decoder->period_insn_cnt += insn_cnt;
1176
1177         if (err) {
1178                 decoder->no_progress = 0;
1179                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
1180                 intel_pt_log_at("ERROR: Failed to get instruction",
1181                                 decoder->ip);
1182                 if (err == -ENOENT)
1183                         return -ENOLINK;
1184                 return -EILSEQ;
1185         }
1186
1187         if (ip && decoder->ip == ip) {
1188                 err = -EAGAIN;
1189                 goto out;
1190         }
1191
1192         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1193                 intel_pt_sample_insn(decoder);
1194
1195         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1196                 decoder->state.type = INTEL_PT_INSTRUCTION;
1197                 decoder->state.from_ip = decoder->ip;
1198                 decoder->state.to_ip = 0;
1199                 decoder->ip += intel_pt_insn->length;
1200                 err = INTEL_PT_RETURN;
1201                 goto out;
1202         }
1203
1204         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1205                 /* Zero-length calls are excluded */
1206                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1207                     intel_pt_insn->rel) {
1208                         err = intel_pt_push(&decoder->stack, decoder->ip +
1209                                             intel_pt_insn->length);
1210                         if (err)
1211                                 goto out;
1212                 }
1213         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1214                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1215         }
1216
1217         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1218                 int cnt = decoder->no_progress++;
1219
1220                 decoder->state.from_ip = decoder->ip;
1221                 decoder->ip += intel_pt_insn->length +
1222                                 intel_pt_insn->rel;
1223                 decoder->state.to_ip = decoder->ip;
1224                 err = INTEL_PT_RETURN;
1225
1226                 /*
1227                  * Check for being stuck in a loop.  This can happen if a
1228                  * decoder error results in the decoder erroneously setting the
1229                  * ip to an address that is itself in an infinite loop that
1230                  * consumes no packets.  When that happens, there must be an
1231                  * unconditional branch.
1232                  */
1233                 if (cnt) {
1234                         if (cnt == 1) {
1235                                 decoder->stuck_ip = decoder->state.to_ip;
1236                                 decoder->stuck_ip_prd = 1;
1237                                 decoder->stuck_ip_cnt = 1;
1238                         } else if (cnt > decoder->max_loops ||
1239                                    decoder->state.to_ip == decoder->stuck_ip) {
1240                                 intel_pt_log_at("ERROR: Never-ending loop",
1241                                                 decoder->state.to_ip);
1242                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1243                                 err = -ELOOP;
1244                                 goto out;
1245                         } else if (!--decoder->stuck_ip_cnt) {
1246                                 decoder->stuck_ip_prd += 1;
1247                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1248                                 decoder->stuck_ip = decoder->state.to_ip;
1249                         }
1250                 }
1251                 goto out_no_progress;
1252         }
1253 out:
1254         decoder->no_progress = 0;
1255 out_no_progress:
1256         decoder->state.insn_op = intel_pt_insn->op;
1257         decoder->state.insn_len = intel_pt_insn->length;
1258         memcpy(decoder->state.insn, intel_pt_insn->buf,
1259                INTEL_PT_INSN_BUF_SZ);
1260
1261         if (decoder->tx_flags & INTEL_PT_IN_TX)
1262                 decoder->state.flags |= INTEL_PT_IN_TX;
1263
1264         return err;
1265 }
1266
1267 static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder)
1268 {
1269         bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
1270
1271         decoder->exec_mode = decoder->packet.payload;
1272         decoder->iflag = iflag;
1273         decoder->next_iflag = iflag;
1274         decoder->state.from_iflag = iflag;
1275         decoder->state.to_iflag = iflag;
1276 }
1277
1278 static void intel_pt_mode_exec(struct intel_pt_decoder *decoder)
1279 {
1280         bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
1281
1282         decoder->exec_mode = decoder->packet.payload;
1283         decoder->next_iflag = iflag;
1284 }
1285
1286 static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder)
1287 {
1288         decoder->state.type |= INTEL_PT_IFLAG_CHG;
1289         decoder->state.from_iflag = decoder->iflag;
1290         decoder->state.to_iflag = decoder->next_iflag;
1291         decoder->iflag = decoder->next_iflag;
1292 }
1293
1294 static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder)
1295 {
1296         if (decoder->iflag != decoder->next_iflag)
1297                 intel_pt_sample_iflag(decoder);
1298 }
1299
1300 static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder)
1301 {
1302         decoder->set_fup_tx_flags = false;
1303         decoder->set_fup_ptw = false;
1304         decoder->set_fup_mwait = false;
1305         decoder->set_fup_pwre = false;
1306         decoder->set_fup_exstop = false;
1307         decoder->set_fup_bep = false;
1308         decoder->set_fup_cfe_ip = false;
1309         decoder->set_fup_cfe = false;
1310         decoder->evd_cnt = 0;
1311         decoder->set_fup_mode_exec = false;
1312         decoder->iflag = decoder->next_iflag;
1313 }
1314
1315 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip)
1316 {
1317         enum intel_pt_sample_type type = decoder->state.type;
1318         bool sample_fup_insn = false;
1319         bool ret = false;
1320
1321         decoder->state.type &= ~INTEL_PT_BRANCH;
1322
1323         if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) {
1324                 bool ip = decoder->set_fup_cfe_ip;
1325
1326                 decoder->set_fup_cfe_ip = false;
1327                 decoder->set_fup_cfe = false;
1328                 decoder->state.type |= INTEL_PT_EVT;
1329                 if (!ip && decoder->pge)
1330                         decoder->state.type |= INTEL_PT_BRANCH;
1331                 decoder->state.cfe_type = decoder->fup_cfe_pkt.count;
1332                 decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload;
1333                 decoder->state.evd_cnt = decoder->evd_cnt;
1334                 decoder->state.evd = decoder->evd;
1335                 decoder->evd_cnt = 0;
1336                 if (ip || decoder->pge)
1337                         decoder->state.flags |= INTEL_PT_FUP_IP;
1338                 ret = true;
1339         }
1340         if (decoder->set_fup_mode_exec) {
1341                 decoder->set_fup_mode_exec = false;
1342                 intel_pt_sample_iflag(decoder);
1343                 sample_fup_insn = no_tip;
1344                 ret = true;
1345         }
1346         if (decoder->set_fup_tx_flags) {
1347                 decoder->set_fup_tx_flags = false;
1348                 decoder->tx_flags = decoder->fup_tx_flags;
1349                 decoder->state.type |= INTEL_PT_TRANSACTION;
1350                 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1351                         decoder->state.type |= INTEL_PT_BRANCH;
1352                 decoder->state.flags = decoder->fup_tx_flags;
1353                 ret = true;
1354         }
1355         if (decoder->set_fup_ptw) {
1356                 decoder->set_fup_ptw = false;
1357                 decoder->state.type |= INTEL_PT_PTW;
1358                 decoder->state.flags |= INTEL_PT_FUP_IP;
1359                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1360                 ret = true;
1361         }
1362         if (decoder->set_fup_mwait) {
1363                 decoder->set_fup_mwait = false;
1364                 decoder->state.type |= INTEL_PT_MWAIT_OP;
1365                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1366                 ret = true;
1367         }
1368         if (decoder->set_fup_pwre) {
1369                 decoder->set_fup_pwre = false;
1370                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1371                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1372                 ret = true;
1373         }
1374         if (decoder->set_fup_exstop) {
1375                 decoder->set_fup_exstop = false;
1376                 decoder->state.type |= INTEL_PT_EX_STOP;
1377                 decoder->state.flags |= INTEL_PT_FUP_IP;
1378                 ret = true;
1379         }
1380         if (decoder->set_fup_bep) {
1381                 decoder->set_fup_bep = false;
1382                 decoder->state.type |= INTEL_PT_BLK_ITEMS;
1383                 ret = true;
1384         }
1385         if (decoder->overflow) {
1386                 decoder->overflow = false;
1387                 if (!ret && !decoder->pge) {
1388                         if (decoder->hop) {
1389                                 decoder->state.type = 0;
1390                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
1391                         }
1392                         decoder->pge = true;
1393                         decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;
1394                         decoder->state.from_ip = 0;
1395                         decoder->state.to_ip = decoder->ip;
1396                         return true;
1397                 }
1398         }
1399         if (ret) {
1400                 decoder->state.from_ip = decoder->ip;
1401                 decoder->state.to_ip = 0;
1402                 if (sample_fup_insn)
1403                         intel_pt_sample_fup_insn(decoder);
1404         } else {
1405                 decoder->state.type = type;
1406         }
1407         return ret;
1408 }
1409
1410 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1411                                           struct intel_pt_insn *intel_pt_insn,
1412                                           uint64_t ip, int err)
1413 {
1414         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1415                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1416                ip == decoder->ip + intel_pt_insn->length;
1417 }
1418
1419 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1420 {
1421         struct intel_pt_insn intel_pt_insn;
1422         uint64_t ip;
1423         int err;
1424
1425         ip = decoder->last_ip;
1426
1427         while (1) {
1428                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1429                 if (err == INTEL_PT_RETURN)
1430                         return 0;
1431                 if (err == -EAGAIN ||
1432                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1433                         bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1434
1435                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1436                         if (intel_pt_fup_event(decoder, no_tip) && no_tip)
1437                                 return 0;
1438                         return -EAGAIN;
1439                 }
1440                 decoder->set_fup_tx_flags = false;
1441                 if (err)
1442                         return err;
1443
1444                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1445                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1446                                         decoder->ip);
1447                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1448                         return -ENOENT;
1449                 }
1450
1451                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1452                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1453                                         decoder->ip);
1454                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1455                         return -ENOENT;
1456                 }
1457
1458                 intel_pt_bug(decoder);
1459         }
1460 }
1461
1462 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1463 {
1464         struct intel_pt_insn intel_pt_insn;
1465         int err;
1466
1467         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1468         if (err == INTEL_PT_RETURN &&
1469             decoder->pgd_ip &&
1470             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1471             (decoder->state.type & INTEL_PT_BRANCH) &&
1472             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1473                 /* Unconditional branch leaving filter region */
1474                 decoder->no_progress = 0;
1475                 decoder->pge = false;
1476                 decoder->continuous_period = false;
1477                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1478                 decoder->state.type |= INTEL_PT_TRACE_END;
1479                 intel_pt_update_nr(decoder);
1480                 return 0;
1481         }
1482         if (err == INTEL_PT_RETURN)
1483                 return 0;
1484         if (err)
1485                 return err;
1486
1487         intel_pt_update_nr(decoder);
1488         intel_pt_sample_iflag_chg(decoder);
1489
1490         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1491                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1492                         decoder->pge = false;
1493                         decoder->continuous_period = false;
1494                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1495                         decoder->state.from_ip = decoder->ip;
1496                         if (decoder->packet.count == 0) {
1497                                 decoder->state.to_ip = 0;
1498                         } else {
1499                                 decoder->state.to_ip = decoder->last_ip;
1500                                 decoder->ip = decoder->last_ip;
1501                         }
1502                         decoder->state.type |= INTEL_PT_TRACE_END;
1503                 } else {
1504                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1505                         decoder->state.from_ip = decoder->ip;
1506                         if (decoder->packet.count == 0) {
1507                                 decoder->state.to_ip = 0;
1508                         } else {
1509                                 decoder->state.to_ip = decoder->last_ip;
1510                                 decoder->ip = decoder->last_ip;
1511                         }
1512                 }
1513                 return 0;
1514         }
1515
1516         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1517                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1518                                  intel_pt_insn.rel;
1519
1520                 if (decoder->pgd_ip &&
1521                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1522                     decoder->pgd_ip(to_ip, decoder->data)) {
1523                         /* Conditional branch leaving filter region */
1524                         decoder->pge = false;
1525                         decoder->continuous_period = false;
1526                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1527                         decoder->ip = to_ip;
1528                         decoder->state.from_ip = decoder->ip;
1529                         decoder->state.to_ip = to_ip;
1530                         decoder->state.type |= INTEL_PT_TRACE_END;
1531                         return 0;
1532                 }
1533                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1534                                 decoder->ip);
1535                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1536                 return -ENOENT;
1537         }
1538
1539         return intel_pt_bug(decoder);
1540 }
1541
1542 struct eptw_data {
1543         int bit_countdown;
1544         uint64_t payload;
1545 };
1546
1547 static int intel_pt_eptw_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
1548 {
1549         struct eptw_data *data = pkt_info->data;
1550         int nr_bits;
1551
1552         switch (pkt_info->packet.type) {
1553         case INTEL_PT_PAD:
1554         case INTEL_PT_MNT:
1555         case INTEL_PT_MODE_EXEC:
1556         case INTEL_PT_MODE_TSX:
1557         case INTEL_PT_MTC:
1558         case INTEL_PT_FUP:
1559         case INTEL_PT_CYC:
1560         case INTEL_PT_CBR:
1561         case INTEL_PT_TSC:
1562         case INTEL_PT_TMA:
1563         case INTEL_PT_PIP:
1564         case INTEL_PT_VMCS:
1565         case INTEL_PT_PSB:
1566         case INTEL_PT_PSBEND:
1567         case INTEL_PT_PTWRITE:
1568         case INTEL_PT_PTWRITE_IP:
1569         case INTEL_PT_EXSTOP:
1570         case INTEL_PT_EXSTOP_IP:
1571         case INTEL_PT_MWAIT:
1572         case INTEL_PT_PWRE:
1573         case INTEL_PT_PWRX:
1574         case INTEL_PT_BBP:
1575         case INTEL_PT_BIP:
1576         case INTEL_PT_BEP:
1577         case INTEL_PT_BEP_IP:
1578         case INTEL_PT_CFE:
1579         case INTEL_PT_CFE_IP:
1580         case INTEL_PT_EVD:
1581                 break;
1582
1583         case INTEL_PT_TNT:
1584                 nr_bits = data->bit_countdown;
1585                 if (nr_bits > pkt_info->packet.count)
1586                         nr_bits = pkt_info->packet.count;
1587                 data->payload <<= nr_bits;
1588                 data->payload |= pkt_info->packet.payload >> (64 - nr_bits);
1589                 data->bit_countdown -= nr_bits;
1590                 return !data->bit_countdown;
1591
1592         case INTEL_PT_TIP_PGE:
1593         case INTEL_PT_TIP_PGD:
1594         case INTEL_PT_TIP:
1595         case INTEL_PT_BAD:
1596         case INTEL_PT_OVF:
1597         case INTEL_PT_TRACESTOP:
1598         default:
1599                 return 1;
1600         }
1601
1602         return 0;
1603 }
1604
1605 static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder)
1606 {
1607         int n = 64 - decoder->tnt.count;
1608         struct eptw_data data = {
1609                 .bit_countdown = n,
1610                 .payload = decoder->tnt.payload >> n,
1611         };
1612
1613         decoder->emulated_ptwrite = false;
1614         intel_pt_log("Emulated ptwrite detected\n");
1615
1616         intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data);
1617         if (data.bit_countdown)
1618                 return -ECONNRESET;
1619
1620         decoder->state.type = INTEL_PT_PTW;
1621         decoder->state.from_ip = decoder->ip;
1622         decoder->state.to_ip = 0;
1623         decoder->state.ptw_payload = data.payload;
1624         return 0;
1625 }
1626
1627 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1628 {
1629         struct intel_pt_insn intel_pt_insn;
1630         int err;
1631
1632         while (1) {
1633                 if (decoder->emulated_ptwrite)
1634                         return intel_pt_emulated_ptwrite(decoder);
1635                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1636                 if (err == INTEL_PT_RETURN) {
1637                         decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite;
1638                         return 0;
1639                 }
1640                 if (err) {
1641                         decoder->emulated_ptwrite = false;
1642                         return err;
1643                 }
1644
1645                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1646                         if (!decoder->return_compression) {
1647                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1648                                                 decoder->ip);
1649                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1650                                 return -ENOENT;
1651                         }
1652                         if (!decoder->ret_addr) {
1653                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1654                                                 decoder->ip);
1655                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1656                                 return -ENOENT;
1657                         }
1658                         if (!(decoder->tnt.payload & BIT63)) {
1659                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1660                                                 decoder->ip);
1661                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1662                                 return -ENOENT;
1663                         }
1664                         decoder->tnt.count -= 1;
1665                         if (decoder->tnt.count)
1666                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1667                         else
1668                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1669                         decoder->tnt.payload <<= 1;
1670                         decoder->state.from_ip = decoder->ip;
1671                         decoder->ip = decoder->ret_addr;
1672                         decoder->state.to_ip = decoder->ip;
1673                         return 0;
1674                 }
1675
1676                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1677                         /* Handle deferred TIPs */
1678                         err = intel_pt_get_next_packet(decoder);
1679                         if (err)
1680                                 return err;
1681                         if (decoder->packet.type != INTEL_PT_TIP ||
1682                             decoder->packet.count == 0) {
1683                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1684                                                 decoder->ip);
1685                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1686                                 decoder->pkt_step = 0;
1687                                 return -ENOENT;
1688                         }
1689                         intel_pt_set_last_ip(decoder);
1690                         decoder->state.from_ip = decoder->ip;
1691                         decoder->state.to_ip = decoder->last_ip;
1692                         decoder->ip = decoder->last_ip;
1693                         intel_pt_update_nr(decoder);
1694                         intel_pt_sample_iflag_chg(decoder);
1695                         return 0;
1696                 }
1697
1698                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1699                         decoder->tnt.count -= 1;
1700                         if (decoder->tnt.count)
1701                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1702                         else
1703                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1704                         if (decoder->tnt.payload & BIT63) {
1705                                 decoder->tnt.payload <<= 1;
1706                                 decoder->state.from_ip = decoder->ip;
1707                                 decoder->ip += intel_pt_insn.length +
1708                                                intel_pt_insn.rel;
1709                                 decoder->state.to_ip = decoder->ip;
1710                                 return 0;
1711                         }
1712                         /* Instruction sample for a non-taken branch */
1713                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1714                                 decoder->tnt.payload <<= 1;
1715                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1716                                 decoder->state.from_ip = decoder->ip;
1717                                 decoder->state.to_ip = 0;
1718                                 decoder->ip += intel_pt_insn.length;
1719                                 return 0;
1720                         }
1721                         decoder->sample_cyc = false;
1722                         decoder->ip += intel_pt_insn.length;
1723                         if (!decoder->tnt.count) {
1724                                 intel_pt_update_sample_time(decoder);
1725                                 return -EAGAIN;
1726                         }
1727                         decoder->tnt.payload <<= 1;
1728                         continue;
1729                 }
1730
1731                 return intel_pt_bug(decoder);
1732         }
1733 }
1734
1735 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1736 {
1737         unsigned int fup_tx_flags;
1738         int err;
1739
1740         fup_tx_flags = decoder->packet.payload &
1741                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1742         err = intel_pt_get_next_packet(decoder);
1743         if (err)
1744                 return err;
1745         if (decoder->packet.type == INTEL_PT_FUP) {
1746                 decoder->fup_tx_flags = fup_tx_flags;
1747                 decoder->set_fup_tx_flags = true;
1748                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1749                         *no_tip = true;
1750         } else {
1751                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1752                                 decoder->pos);
1753                 intel_pt_update_in_tx(decoder);
1754         }
1755         return 0;
1756 }
1757
1758 static int intel_pt_evd(struct intel_pt_decoder *decoder)
1759 {
1760         if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) {
1761                 intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos);
1762                 return -ENOSYS;
1763         }
1764         decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){
1765                 .type = decoder->packet.count,
1766                 .payload = decoder->packet.payload,
1767         };
1768         return 0;
1769 }
1770
1771 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1772 {
1773         timestamp |= (ref_timestamp & (0xffULL << 56));
1774
1775         if (timestamp < ref_timestamp) {
1776                 if (ref_timestamp - timestamp > (1ULL << 55))
1777                         timestamp += (1ULL << 56);
1778         } else {
1779                 if (timestamp - ref_timestamp > (1ULL << 55))
1780                         timestamp -= (1ULL << 56);
1781         }
1782
1783         return timestamp;
1784 }
1785
1786 /* For use only when decoder->vm_time_correlation is true */
1787 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1788                                    uint64_t timestamp)
1789 {
1790         uint64_t max_timestamp = decoder->buf_timestamp;
1791
1792         if (!max_timestamp) {
1793                 max_timestamp = decoder->last_reliable_timestamp +
1794                                 0x400000000ULL;
1795         }
1796         return timestamp >= decoder->last_reliable_timestamp &&
1797                timestamp < decoder->buf_timestamp;
1798 }
1799
1800 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1801 {
1802         uint64_t timestamp;
1803         bool bad = false;
1804
1805         decoder->have_tma = false;
1806
1807         if (decoder->ref_timestamp) {
1808                 timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1809                                             decoder->ref_timestamp);
1810                 decoder->tsc_timestamp = timestamp;
1811                 decoder->timestamp = timestamp;
1812                 decoder->ref_timestamp = 0;
1813                 decoder->timestamp_insn_cnt = 0;
1814         } else if (decoder->timestamp) {
1815                 timestamp = decoder->packet.payload |
1816                             (decoder->timestamp & (0xffULL << 56));
1817                 decoder->tsc_timestamp = timestamp;
1818                 if (timestamp < decoder->timestamp &&
1819                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1820                         intel_pt_log_to("Suppressing backwards timestamp",
1821                                         timestamp);
1822                         timestamp = decoder->timestamp;
1823                 }
1824                 if (timestamp < decoder->timestamp) {
1825                         if (!decoder->buf_timestamp ||
1826                             (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1827                                 intel_pt_log_to("Wraparound timestamp", timestamp);
1828                                 timestamp += (1ULL << 56);
1829                                 decoder->tsc_timestamp = timestamp;
1830                         } else {
1831                                 intel_pt_log_to("Suppressing bad timestamp", timestamp);
1832                                 timestamp = decoder->timestamp;
1833                                 bad = true;
1834                         }
1835                 }
1836                 if (decoder->vm_time_correlation &&
1837                     (bad || !intel_pt_time_in_range(decoder, timestamp)) &&
1838                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1839                         p_log("Timestamp out of range");
1840                 decoder->timestamp = timestamp;
1841                 decoder->timestamp_insn_cnt = 0;
1842         }
1843
1844         if (decoder->last_packet_type == INTEL_PT_CYC) {
1845                 decoder->cyc_ref_timestamp = decoder->timestamp;
1846                 decoder->cycle_cnt = 0;
1847                 decoder->have_calc_cyc_to_tsc = false;
1848                 intel_pt_calc_cyc_to_tsc(decoder, false);
1849         }
1850
1851         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1852 }
1853
1854 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1855 {
1856         intel_pt_log("ERROR: Buffer overflow\n");
1857         intel_pt_clear_tx_flags(decoder);
1858         intel_pt_set_nr(decoder);
1859         decoder->timestamp_insn_cnt = 0;
1860         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1861         decoder->state.from_ip = decoder->ip;
1862         decoder->ip = 0;
1863         decoder->pge = false;
1864         intel_pt_clear_fup_event(decoder);
1865         decoder->overflow = true;
1866         return -EOVERFLOW;
1867 }
1868
1869 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1870 {
1871         if (decoder->have_cyc)
1872                 return;
1873
1874         decoder->cyc_cnt_timestamp = decoder->timestamp;
1875         decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1876 }
1877
1878 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1879 {
1880         decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1881
1882         if (decoder->pge)
1883                 intel_pt_mtc_cyc_cnt_pge(decoder);
1884 }
1885
1886 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1887 {
1888         uint64_t tot_cyc_cnt, tsc_delta;
1889
1890         if (decoder->have_cyc)
1891                 return;
1892
1893         decoder->sample_cyc = true;
1894
1895         if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1896                 return;
1897
1898         tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1899         tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1900
1901         if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1902                 decoder->tot_cyc_cnt = tot_cyc_cnt;
1903 }
1904
1905 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1906 {
1907         uint32_t ctc = decoder->packet.payload;
1908         uint32_t fc = decoder->packet.count;
1909         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1910
1911         if (!decoder->tsc_ctc_ratio_d)
1912                 return;
1913
1914         if (decoder->pge && !decoder->in_psb)
1915                 intel_pt_mtc_cyc_cnt_pge(decoder);
1916         else
1917                 intel_pt_mtc_cyc_cnt_upd(decoder);
1918
1919         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1920         decoder->last_ctc = ctc - ctc_rem;
1921         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1922         if (decoder->tsc_ctc_mult) {
1923                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1924         } else {
1925                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1926                                                   decoder->tsc_ctc_ratio_n,
1927                                                   decoder->tsc_ctc_ratio_d);
1928         }
1929         decoder->ctc_delta = 0;
1930         decoder->have_tma = true;
1931         decoder->fixup_last_mtc = true;
1932         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1933                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1934 }
1935
1936 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1937 {
1938         uint64_t timestamp;
1939         uint32_t mtc, mtc_delta;
1940
1941         if (!decoder->have_tma)
1942                 return;
1943
1944         mtc = decoder->packet.payload;
1945
1946         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1947                 decoder->fixup_last_mtc = false;
1948                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1949                                         &decoder->last_mtc);
1950         }
1951
1952         if (mtc > decoder->last_mtc)
1953                 mtc_delta = mtc - decoder->last_mtc;
1954         else
1955                 mtc_delta = mtc + 256 - decoder->last_mtc;
1956
1957         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1958
1959         if (decoder->tsc_ctc_mult) {
1960                 timestamp = decoder->ctc_timestamp +
1961                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1962         } else {
1963                 timestamp = decoder->ctc_timestamp +
1964                             multdiv(decoder->ctc_delta,
1965                                     decoder->tsc_ctc_ratio_n,
1966                                     decoder->tsc_ctc_ratio_d);
1967         }
1968
1969         if (timestamp < decoder->timestamp)
1970                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1971                              timestamp, decoder->timestamp);
1972         else
1973                 decoder->timestamp = timestamp;
1974
1975         intel_pt_mtc_cyc_cnt_upd(decoder);
1976
1977         decoder->timestamp_insn_cnt = 0;
1978         decoder->last_mtc = mtc;
1979
1980         if (decoder->last_packet_type == INTEL_PT_CYC) {
1981                 decoder->cyc_ref_timestamp = decoder->timestamp;
1982                 decoder->cycle_cnt = 0;
1983                 decoder->have_calc_cyc_to_tsc = false;
1984                 intel_pt_calc_cyc_to_tsc(decoder, true);
1985         }
1986
1987         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1988 }
1989
1990 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1991 {
1992         unsigned int cbr = decoder->packet.payload & 0xff;
1993
1994         decoder->cbr_payload = decoder->packet.payload;
1995
1996         if (decoder->cbr == cbr)
1997                 return;
1998
1999         decoder->cbr = cbr;
2000         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
2001
2002         intel_pt_mtc_cyc_cnt_cbr(decoder);
2003 }
2004
2005 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
2006 {
2007         uint64_t timestamp = decoder->cyc_ref_timestamp;
2008
2009         decoder->have_cyc = true;
2010
2011         decoder->cycle_cnt += decoder->packet.payload;
2012         if (decoder->pge)
2013                 decoder->tot_cyc_cnt += decoder->packet.payload;
2014         decoder->sample_cyc = true;
2015
2016         if (!decoder->cyc_ref_timestamp)
2017                 return;
2018
2019         if (decoder->have_calc_cyc_to_tsc)
2020                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
2021         else if (decoder->cbr)
2022                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
2023         else
2024                 return;
2025
2026         if (timestamp < decoder->timestamp)
2027                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
2028                              timestamp, decoder->timestamp);
2029         else
2030                 decoder->timestamp = timestamp;
2031
2032         decoder->timestamp_insn_cnt = 0;
2033
2034         intel_pt_log_to("Setting timestamp", decoder->timestamp);
2035 }
2036
2037 static void intel_pt_bbp(struct intel_pt_decoder *decoder)
2038 {
2039         if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
2040                 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
2041                 decoder->state.items.is_32_bit = false;
2042         }
2043         decoder->blk_type = decoder->packet.payload;
2044         decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
2045         if (decoder->blk_type == INTEL_PT_GP_REGS)
2046                 decoder->state.items.is_32_bit = decoder->packet.count;
2047         if (decoder->blk_type_pos < 0) {
2048                 intel_pt_log("WARNING: Unknown block type %u\n",
2049                              decoder->blk_type);
2050         } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
2051                 intel_pt_log("WARNING: Duplicate block type %u\n",
2052                              decoder->blk_type);
2053         }
2054 }
2055
2056 static void intel_pt_bip(struct intel_pt_decoder *decoder)
2057 {
2058         uint32_t id = decoder->packet.count;
2059         uint32_t bit = 1 << id;
2060         int pos = decoder->blk_type_pos;
2061
2062         if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
2063                 intel_pt_log("WARNING: Unknown block item %u type %d\n",
2064                              id, decoder->blk_type);
2065                 return;
2066         }
2067
2068         if (decoder->state.items.mask[pos] & bit) {
2069                 intel_pt_log("WARNING: Duplicate block item %u type %d\n",
2070                              id, decoder->blk_type);
2071         }
2072
2073         decoder->state.items.mask[pos] |= bit;
2074         decoder->state.items.val[pos][id] = decoder->packet.payload;
2075 }
2076
2077 /* Walk PSB+ packets when already in sync. */
2078 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
2079 {
2080         int err;
2081
2082         decoder->in_psb = true;
2083
2084         while (1) {
2085                 err = intel_pt_get_next_packet(decoder);
2086                 if (err)
2087                         goto out;
2088
2089                 switch (decoder->packet.type) {
2090                 case INTEL_PT_PSBEND:
2091                         err = 0;
2092                         goto out;
2093
2094                 case INTEL_PT_TIP_PGD:
2095                 case INTEL_PT_TIP_PGE:
2096                 case INTEL_PT_TIP:
2097                 case INTEL_PT_TNT:
2098                 case INTEL_PT_TRACESTOP:
2099                 case INTEL_PT_BAD:
2100                 case INTEL_PT_PSB:
2101                 case INTEL_PT_PTWRITE:
2102                 case INTEL_PT_PTWRITE_IP:
2103                 case INTEL_PT_EXSTOP:
2104                 case INTEL_PT_EXSTOP_IP:
2105                 case INTEL_PT_MWAIT:
2106                 case INTEL_PT_PWRE:
2107                 case INTEL_PT_PWRX:
2108                 case INTEL_PT_BBP:
2109                 case INTEL_PT_BIP:
2110                 case INTEL_PT_BEP:
2111                 case INTEL_PT_BEP_IP:
2112                 case INTEL_PT_CFE:
2113                 case INTEL_PT_CFE_IP:
2114                 case INTEL_PT_EVD:
2115                         decoder->have_tma = false;
2116                         intel_pt_log("ERROR: Unexpected packet\n");
2117                         err = -EAGAIN;
2118                         goto out;
2119
2120                 case INTEL_PT_OVF:
2121                         err = intel_pt_overflow(decoder);
2122                         goto out;
2123
2124                 case INTEL_PT_TSC:
2125                         intel_pt_calc_tsc_timestamp(decoder);
2126                         break;
2127
2128                 case INTEL_PT_TMA:
2129                         intel_pt_calc_tma(decoder);
2130                         break;
2131
2132                 case INTEL_PT_CBR:
2133                         intel_pt_calc_cbr(decoder);
2134                         break;
2135
2136                 case INTEL_PT_MODE_EXEC:
2137                         intel_pt_mode_exec_status(decoder);
2138                         break;
2139
2140                 case INTEL_PT_PIP:
2141                         intel_pt_set_pip(decoder);
2142                         break;
2143
2144                 case INTEL_PT_FUP:
2145                         decoder->pge = true;
2146                         if (decoder->packet.count) {
2147                                 intel_pt_set_last_ip(decoder);
2148                                 decoder->psb_ip = decoder->last_ip;
2149                         }
2150                         break;
2151
2152                 case INTEL_PT_MODE_TSX:
2153                         intel_pt_update_in_tx(decoder);
2154                         break;
2155
2156                 case INTEL_PT_MTC:
2157                         intel_pt_calc_mtc_timestamp(decoder);
2158                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2159                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2160                         break;
2161
2162                 case INTEL_PT_CYC:
2163                         intel_pt_calc_cyc_timestamp(decoder);
2164                         break;
2165
2166                 case INTEL_PT_VMCS:
2167                 case INTEL_PT_MNT:
2168                 case INTEL_PT_PAD:
2169                 default:
2170                         break;
2171                 }
2172         }
2173 out:
2174         decoder->in_psb = false;
2175
2176         return err;
2177 }
2178
2179 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
2180 {
2181         int err;
2182
2183         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
2184                 decoder->tx_flags = 0;
2185                 decoder->state.flags &= ~INTEL_PT_IN_TX;
2186                 decoder->state.flags |= INTEL_PT_ABORT_TX;
2187         } else {
2188                 decoder->state.flags |= INTEL_PT_ASYNC;
2189         }
2190
2191         while (1) {
2192                 err = intel_pt_get_next_packet(decoder);
2193                 if (err)
2194                         return err;
2195
2196                 switch (decoder->packet.type) {
2197                 case INTEL_PT_TNT:
2198                 case INTEL_PT_FUP:
2199                 case INTEL_PT_TRACESTOP:
2200                 case INTEL_PT_PSB:
2201                 case INTEL_PT_TSC:
2202                 case INTEL_PT_TMA:
2203                 case INTEL_PT_MODE_TSX:
2204                 case INTEL_PT_BAD:
2205                 case INTEL_PT_PSBEND:
2206                 case INTEL_PT_PTWRITE:
2207                 case INTEL_PT_PTWRITE_IP:
2208                 case INTEL_PT_EXSTOP:
2209                 case INTEL_PT_EXSTOP_IP:
2210                 case INTEL_PT_MWAIT:
2211                 case INTEL_PT_PWRE:
2212                 case INTEL_PT_PWRX:
2213                 case INTEL_PT_BBP:
2214                 case INTEL_PT_BIP:
2215                 case INTEL_PT_BEP:
2216                 case INTEL_PT_BEP_IP:
2217                 case INTEL_PT_CFE:
2218                 case INTEL_PT_CFE_IP:
2219                 case INTEL_PT_EVD:
2220                         intel_pt_log("ERROR: Missing TIP after FUP\n");
2221                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
2222                         decoder->pkt_step = 0;
2223                         return -ENOENT;
2224
2225                 case INTEL_PT_CBR:
2226                         intel_pt_calc_cbr(decoder);
2227                         break;
2228
2229                 case INTEL_PT_OVF:
2230                         return intel_pt_overflow(decoder);
2231
2232                 case INTEL_PT_TIP_PGD:
2233                         decoder->state.from_ip = decoder->ip;
2234                         if (decoder->packet.count == 0) {
2235                                 decoder->state.to_ip = 0;
2236                         } else {
2237                                 intel_pt_set_ip(decoder);
2238                                 decoder->state.to_ip = decoder->ip;
2239                         }
2240                         decoder->pge = false;
2241                         decoder->continuous_period = false;
2242                         decoder->state.type |= INTEL_PT_TRACE_END;
2243                         intel_pt_update_nr(decoder);
2244                         return 0;
2245
2246                 case INTEL_PT_TIP_PGE:
2247                         decoder->pge = true;
2248                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
2249                                      decoder->ip);
2250                         decoder->state.from_ip = 0;
2251                         if (decoder->packet.count == 0) {
2252                                 decoder->state.to_ip = 0;
2253                         } else {
2254                                 intel_pt_set_ip(decoder);
2255                                 decoder->state.to_ip = decoder->ip;
2256                         }
2257                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2258                         intel_pt_mtc_cyc_cnt_pge(decoder);
2259                         intel_pt_set_nr(decoder);
2260                         return 0;
2261
2262                 case INTEL_PT_TIP:
2263                         decoder->state.from_ip = decoder->ip;
2264                         if (decoder->packet.count == 0) {
2265                                 decoder->state.to_ip = 0;
2266                         } else {
2267                                 intel_pt_set_ip(decoder);
2268                                 decoder->state.to_ip = decoder->ip;
2269                         }
2270                         intel_pt_update_nr(decoder);
2271                         intel_pt_sample_iflag_chg(decoder);
2272                         return 0;
2273
2274                 case INTEL_PT_PIP:
2275                         intel_pt_update_pip(decoder);
2276                         break;
2277
2278                 case INTEL_PT_MTC:
2279                         intel_pt_calc_mtc_timestamp(decoder);
2280                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2281                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2282                         break;
2283
2284                 case INTEL_PT_CYC:
2285                         intel_pt_calc_cyc_timestamp(decoder);
2286                         break;
2287
2288                 case INTEL_PT_MODE_EXEC:
2289                         intel_pt_mode_exec(decoder);
2290                         break;
2291
2292                 case INTEL_PT_VMCS:
2293                 case INTEL_PT_MNT:
2294                 case INTEL_PT_PAD:
2295                         break;
2296
2297                 default:
2298                         return intel_pt_bug(decoder);
2299                 }
2300         }
2301 }
2302
2303 static int intel_pt_resample(struct intel_pt_decoder *decoder)
2304 {
2305         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2306         decoder->state.type = INTEL_PT_INSTRUCTION;
2307         decoder->state.from_ip = decoder->ip;
2308         decoder->state.to_ip = 0;
2309         return 0;
2310 }
2311
2312 struct intel_pt_vm_tsc_info {
2313         struct intel_pt_pkt pip_packet;
2314         struct intel_pt_pkt vmcs_packet;
2315         struct intel_pt_pkt tma_packet;
2316         bool tsc, pip, vmcs, tma, psbend;
2317         uint64_t ctc_delta;
2318         uint64_t last_ctc;
2319         int max_lookahead;
2320 };
2321
2322 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2323 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2324 {
2325         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2326
2327         switch (pkt_info->packet.type) {
2328         case INTEL_PT_PAD:
2329         case INTEL_PT_MNT:
2330         case INTEL_PT_MODE_EXEC:
2331         case INTEL_PT_MODE_TSX:
2332         case INTEL_PT_MTC:
2333         case INTEL_PT_FUP:
2334         case INTEL_PT_CYC:
2335         case INTEL_PT_CBR:
2336                 break;
2337
2338         case INTEL_PT_TSC:
2339                 data->tsc = true;
2340                 break;
2341
2342         case INTEL_PT_TMA:
2343                 data->tma_packet = pkt_info->packet;
2344                 data->tma = true;
2345                 break;
2346
2347         case INTEL_PT_PIP:
2348                 data->pip_packet = pkt_info->packet;
2349                 data->pip = true;
2350                 break;
2351
2352         case INTEL_PT_VMCS:
2353                 data->vmcs_packet = pkt_info->packet;
2354                 data->vmcs = true;
2355                 break;
2356
2357         case INTEL_PT_PSBEND:
2358                 data->psbend = true;
2359                 return 1;
2360
2361         case INTEL_PT_TIP_PGE:
2362         case INTEL_PT_PTWRITE:
2363         case INTEL_PT_PTWRITE_IP:
2364         case INTEL_PT_EXSTOP:
2365         case INTEL_PT_EXSTOP_IP:
2366         case INTEL_PT_MWAIT:
2367         case INTEL_PT_PWRE:
2368         case INTEL_PT_PWRX:
2369         case INTEL_PT_BBP:
2370         case INTEL_PT_BIP:
2371         case INTEL_PT_BEP:
2372         case INTEL_PT_BEP_IP:
2373         case INTEL_PT_OVF:
2374         case INTEL_PT_BAD:
2375         case INTEL_PT_TNT:
2376         case INTEL_PT_TIP_PGD:
2377         case INTEL_PT_TIP:
2378         case INTEL_PT_PSB:
2379         case INTEL_PT_TRACESTOP:
2380         case INTEL_PT_CFE:
2381         case INTEL_PT_CFE_IP:
2382         case INTEL_PT_EVD:
2383         default:
2384                 return 1;
2385         }
2386
2387         return 0;
2388 }
2389
2390 struct intel_pt_ovf_fup_info {
2391         int max_lookahead;
2392         bool found;
2393 };
2394
2395 /* Lookahead to detect a FUP packet after OVF */
2396 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2397 {
2398         struct intel_pt_ovf_fup_info *data = pkt_info->data;
2399
2400         if (pkt_info->packet.type == INTEL_PT_CYC ||
2401             pkt_info->packet.type == INTEL_PT_MTC ||
2402             pkt_info->packet.type == INTEL_PT_TSC)
2403                 return !--(data->max_lookahead);
2404         data->found = pkt_info->packet.type == INTEL_PT_FUP;
2405         return 1;
2406 }
2407
2408 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2409 {
2410         struct intel_pt_ovf_fup_info data = {
2411                 .max_lookahead = 16,
2412                 .found = false,
2413         };
2414
2415         intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2416         return data.found;
2417 }
2418
2419 /* Lookahead and get the TMA packet after TSC */
2420 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2421 {
2422         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2423
2424         if (pkt_info->packet.type == INTEL_PT_CYC ||
2425             pkt_info->packet.type == INTEL_PT_MTC)
2426                 return !--(data->max_lookahead);
2427
2428         if (pkt_info->packet.type == INTEL_PT_TMA) {
2429                 data->tma_packet = pkt_info->packet;
2430                 data->tma = true;
2431         }
2432         return 1;
2433 }
2434
2435 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2436 {
2437         if (decoder->tsc_ctc_mult)
2438                 return ctc * decoder->tsc_ctc_mult;
2439         else
2440                 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2441 }
2442
2443 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2444                                            uint32_t ctc,
2445                                            uint32_t fc,
2446                                            uint64_t last_ctc_timestamp,
2447                                            uint64_t ctc_delta,
2448                                            uint32_t last_ctc)
2449 {
2450         /* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2451         uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2452         /*
2453          * Number of CTC ticks from there until current TMA packet. We would
2454          * expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2455          * past an MTC, so a sign-extended value is used.
2456          */
2457         uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2458         /* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2459         uint64_t new_ctc_delta = ctc_delta + delta;
2460         uint64_t expected_tsc;
2461
2462         /*
2463          * Convert CTC ticks to TSC ticks, add the starting point
2464          * (last_ctc_timestamp) and the fast counter from the TMA packet.
2465          */
2466         expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2467
2468         if (intel_pt_enable_logging) {
2469                 intel_pt_log_x64(last_mtc_ctc);
2470                 intel_pt_log_x32(last_ctc);
2471                 intel_pt_log_x64(ctc_delta);
2472                 intel_pt_log_x64(delta);
2473                 intel_pt_log_x32(ctc);
2474                 intel_pt_log_x64(new_ctc_delta);
2475                 intel_pt_log_x64(last_ctc_timestamp);
2476                 intel_pt_log_x32(fc);
2477                 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2478                 intel_pt_log_x64(expected_tsc);
2479         }
2480
2481         return expected_tsc;
2482 }
2483
2484 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2485                                       struct intel_pt_vm_tsc_info *data)
2486 {
2487         uint32_t ctc = data->tma_packet.payload;
2488         uint32_t fc = data->tma_packet.count;
2489
2490         return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2491                                           decoder->ctc_timestamp,
2492                                           data->ctc_delta, data->last_ctc);
2493 }
2494
2495 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2496                                       struct intel_pt_vmcs_info *vmcs_info)
2497 {
2498         uint64_t payload = decoder->packet.payload;
2499
2500         /* VMX adds the TSC Offset, so subtract to get host TSC */
2501         decoder->packet.payload -= vmcs_info->tsc_offset;
2502         /* TSC packet has only 7 bytes */
2503         decoder->packet.payload &= SEVEN_BYTES;
2504
2505         /*
2506          * The buffer is mmapped from the data file, so this also updates the
2507          * data file.
2508          */
2509         if (!decoder->vm_tm_corr_dry_run)
2510                 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2511
2512         intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2513                      "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2514                      payload, decoder->packet.payload, vmcs_info->vmcs,
2515                      vmcs_info->tsc_offset);
2516 }
2517
2518 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2519                                              uint64_t tsc_offset)
2520 {
2521         struct intel_pt_vmcs_info vmcs_info = {
2522                 .vmcs = NO_VMCS,
2523                 .tsc_offset = tsc_offset
2524         };
2525
2526         intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2527 }
2528
2529 static inline bool in_vm(uint64_t pip_payload)
2530 {
2531         return pip_payload & 1;
2532 }
2533
2534 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2535 {
2536         return pip_packet->payload & 1;
2537 }
2538
2539 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2540 {
2541         p_log("VMCS: %#" PRIx64 "  TSC Offset %#" PRIx64,
2542               vmcs_info->vmcs, vmcs_info->tsc_offset);
2543 }
2544
2545 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2546                                     struct intel_pt_vm_tsc_info *data)
2547 {
2548         memset(data, 0, sizeof(*data));
2549         data->ctc_delta = decoder->ctc_delta;
2550         data->last_ctc = decoder->last_ctc;
2551         intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2552         if (data->tsc && !data->psbend)
2553                 p_log("ERROR: PSB without PSBEND");
2554         decoder->in_psb = data->psbend;
2555 }
2556
2557 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2558                                           struct intel_pt_vm_tsc_info *data,
2559                                           struct intel_pt_vmcs_info *vmcs_info,
2560                                           uint64_t host_tsc)
2561 {
2562         if (!decoder->in_psb) {
2563                 /* Can't happen */
2564                 p_log("ERROR: First TSC is not in PSB+");
2565         }
2566
2567         if (data->pip) {
2568                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2569                         if (vmcs_info && vmcs_info->tsc_offset) {
2570                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2571                                 decoder->vm_tm_corr_reliable = true;
2572                         } else {
2573                                 p_log("ERROR: First TSC, unknown TSC Offset");
2574                         }
2575                 } else { /* Host */
2576                         decoder->vm_tm_corr_reliable = true;
2577                 }
2578         } else { /* Host or Guest */
2579                 decoder->vm_tm_corr_reliable = false;
2580                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2581                         /* Assume Host */
2582                 } else {
2583                         /* Assume Guest */
2584                         if (vmcs_info && vmcs_info->tsc_offset)
2585                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2586                         else
2587                                 p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2588                 }
2589         }
2590 }
2591
2592 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2593                                     struct intel_pt_vm_tsc_info *data)
2594 {
2595         struct intel_pt_vmcs_info *vmcs_info;
2596         uint64_t tsc_offset = 0;
2597         uint64_t vmcs;
2598         bool reliable = true;
2599         uint64_t expected_tsc;
2600         uint64_t host_tsc;
2601         uint64_t ref_timestamp;
2602
2603         bool assign = false;
2604         bool assign_reliable = false;
2605
2606         /* Already have 'data' for the in_psb case */
2607         if (!decoder->in_psb) {
2608                 memset(data, 0, sizeof(*data));
2609                 data->ctc_delta = decoder->ctc_delta;
2610                 data->last_ctc = decoder->last_ctc;
2611                 data->max_lookahead = 16;
2612                 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2613                 if (decoder->pge) {
2614                         data->pip = true;
2615                         data->pip_packet.payload = decoder->pip_payload;
2616                 }
2617         }
2618
2619         /* Calculations depend on having TMA packets */
2620         if (!data->tma) {
2621                 p_log("ERROR: TSC without TMA");
2622                 return;
2623         }
2624
2625         vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2626         if (vmcs == NO_VMCS)
2627                 vmcs = 0;
2628
2629         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2630
2631         ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2632         host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2633
2634         if (!decoder->ctc_timestamp) {
2635                 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2636                 return;
2637         }
2638
2639         expected_tsc = intel_pt_expected_tsc(decoder, data);
2640
2641         tsc_offset = host_tsc - expected_tsc;
2642
2643         /* Determine if TSC is from Host or Guest */
2644         if (data->pip) {
2645                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2646                         if (!vmcs_info) {
2647                                 /* PIP NR=1 without VMCS cannot happen */
2648                                 p_log("ERROR: Missing VMCS");
2649                                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2650                                 decoder->vm_tm_corr_reliable = false;
2651                                 return;
2652                         }
2653                 } else { /* Host */
2654                         decoder->last_reliable_timestamp = host_tsc;
2655                         decoder->vm_tm_corr_reliable = true;
2656                         return;
2657                 }
2658         } else { /* Host or Guest */
2659                 reliable = false; /* Host/Guest is a guess, so not reliable */
2660                 if (decoder->in_psb) {
2661                         if (!tsc_offset)
2662                                 return; /* Zero TSC Offset, assume Host */
2663                         /*
2664                          * TSC packet has only 7 bytes of TSC. We have no
2665                          * information about the Guest's 8th byte, but it
2666                          * doesn't matter because we only need 7 bytes.
2667                          * Here, since the 8th byte is unreliable and
2668                          * irrelevant, compare only 7 byes.
2669                          */
2670                         if (vmcs_info &&
2671                             (tsc_offset & SEVEN_BYTES) ==
2672                             (vmcs_info->tsc_offset & SEVEN_BYTES)) {
2673                                 /* Same TSC Offset as last VMCS, assume Guest */
2674                                 goto guest;
2675                         }
2676                 }
2677                 /*
2678                  * Check if the host_tsc is within the expected range.
2679                  * Note, we could narrow the range more by looking ahead for
2680                  * the next host TSC in the same buffer, but we don't bother to
2681                  * do that because this is probably good enough.
2682                  */
2683                 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2684                         /* Within expected range for Host TSC, assume Host */
2685                         decoder->vm_tm_corr_reliable = false;
2686                         return;
2687                 }
2688         }
2689
2690 guest: /* Assuming Guest */
2691
2692         /* Determine whether to assign TSC Offset */
2693         if (vmcs_info && vmcs_info->vmcs) {
2694                 if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2695                         assign = false;
2696                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2697                            decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2698                         /* Continuous tracing, TSC in a PSB is not a time loss */
2699                         assign = true;
2700                         assign_reliable = true;
2701                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2702                         /*
2703                          * Unlikely to be a time loss TSC in a PSB which is not
2704                          * at the start of a buffer.
2705                          */
2706                         assign = true;
2707                         assign_reliable = false;
2708                 }
2709         }
2710
2711         /* Record VMCS TSC Offset */
2712         if (assign && (vmcs_info->tsc_offset != tsc_offset ||
2713                        vmcs_info->reliable != assign_reliable)) {
2714                 bool print = vmcs_info->tsc_offset != tsc_offset;
2715
2716                 vmcs_info->tsc_offset = tsc_offset;
2717                 vmcs_info->reliable = assign_reliable;
2718                 if (print)
2719                         intel_pt_print_vmcs_info(vmcs_info);
2720         }
2721
2722         /* Determine what TSC Offset to use */
2723         if (vmcs_info && vmcs_info->tsc_offset) {
2724                 if (!vmcs_info->reliable)
2725                         reliable = false;
2726                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2727         } else {
2728                 reliable = false;
2729                 if (vmcs_info) {
2730                         if (!vmcs_info->error_printed) {
2731                                 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2732                                       vmcs_info->vmcs);
2733                                 vmcs_info->error_printed = true;
2734                         }
2735                 } else {
2736                         if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2737                                 p_log("ERROR: Unknown VMCS");
2738                 }
2739                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2740         }
2741
2742         decoder->vm_tm_corr_reliable = reliable;
2743 }
2744
2745 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2746 {
2747         uint64_t host_tsc = decoder->packet.payload;
2748         uint64_t guest_tsc = decoder->packet.payload;
2749         struct intel_pt_vmcs_info *vmcs_info;
2750         uint64_t vmcs;
2751
2752         vmcs = decoder->vmcs;
2753         if (vmcs == NO_VMCS)
2754                 vmcs = 0;
2755
2756         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2757
2758         if (decoder->pge) {
2759                 if (in_vm(decoder->pip_payload)) { /* Guest */
2760                         if (!vmcs_info) {
2761                                 /* PIP NR=1 without VMCS cannot happen */
2762                                 p_log("ERROR: Missing VMCS");
2763                         }
2764                 } else { /* Host */
2765                         return;
2766                 }
2767         } else { /* Host or Guest */
2768                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2769                         /* Within expected range for Host TSC, assume Host */
2770                         return;
2771                 }
2772         }
2773
2774         if (vmcs_info) {
2775                 /* Translate Guest TSC to Host TSC */
2776                 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2777                 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2778                 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2779                              "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2780                              guest_tsc, host_tsc, vmcs_info->vmcs,
2781                              vmcs_info->tsc_offset);
2782                 if (!intel_pt_time_in_range(decoder, host_tsc) &&
2783                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2784                         p_log("Timestamp out of range");
2785         } else {
2786                 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2787                         p_log("ERROR: Unknown VMCS");
2788                 host_tsc = decoder->timestamp;
2789         }
2790
2791         decoder->packet.payload = host_tsc;
2792
2793         if (!decoder->vm_tm_corr_dry_run)
2794                 memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2795 }
2796
2797 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2798 {
2799         struct intel_pt_vm_tsc_info data = { .psbend = false };
2800         bool pge;
2801         int err;
2802
2803         if (decoder->in_psb)
2804                 intel_pt_vm_tm_corr_psb(decoder, &data);
2805
2806         while (1) {
2807                 err = intel_pt_get_next_packet(decoder);
2808                 if (err == -ENOLINK)
2809                         continue;
2810                 if (err)
2811                         break;
2812
2813                 switch (decoder->packet.type) {
2814                 case INTEL_PT_TIP_PGD:
2815                         decoder->pge = false;
2816                         decoder->vm_tm_corr_continuous = false;
2817                         break;
2818
2819                 case INTEL_PT_TNT:
2820                 case INTEL_PT_TIP:
2821                 case INTEL_PT_TIP_PGE:
2822                         decoder->pge = true;
2823                         break;
2824
2825                 case INTEL_PT_OVF:
2826                         decoder->in_psb = false;
2827                         pge = decoder->pge;
2828                         decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2829                         if (pge != decoder->pge)
2830                                 intel_pt_log("Surprising PGE change in OVF!");
2831                         if (!decoder->pge)
2832                                 decoder->vm_tm_corr_continuous = false;
2833                         break;
2834
2835                 case INTEL_PT_FUP:
2836                         if (decoder->in_psb)
2837                                 decoder->pge = true;
2838                         break;
2839
2840                 case INTEL_PT_TRACESTOP:
2841                         decoder->pge = false;
2842                         decoder->vm_tm_corr_continuous = false;
2843                         decoder->have_tma = false;
2844                         break;
2845
2846                 case INTEL_PT_PSB:
2847                         intel_pt_vm_tm_corr_psb(decoder, &data);
2848                         break;
2849
2850                 case INTEL_PT_PIP:
2851                         decoder->pip_payload = decoder->packet.payload;
2852                         break;
2853
2854                 case INTEL_PT_MTC:
2855                         intel_pt_calc_mtc_timestamp(decoder);
2856                         break;
2857
2858                 case INTEL_PT_TSC:
2859                         intel_pt_vm_tm_corr_tsc(decoder, &data);
2860                         intel_pt_calc_tsc_timestamp(decoder);
2861                         decoder->vm_tm_corr_same_buf = true;
2862                         decoder->vm_tm_corr_continuous = decoder->pge;
2863                         break;
2864
2865                 case INTEL_PT_TMA:
2866                         intel_pt_calc_tma(decoder);
2867                         break;
2868
2869                 case INTEL_PT_CYC:
2870                         intel_pt_calc_cyc_timestamp(decoder);
2871                         break;
2872
2873                 case INTEL_PT_CBR:
2874                         intel_pt_calc_cbr(decoder);
2875                         break;
2876
2877                 case INTEL_PT_PSBEND:
2878                         decoder->in_psb = false;
2879                         data.psbend = false;
2880                         break;
2881
2882                 case INTEL_PT_VMCS:
2883                         if (decoder->packet.payload != NO_VMCS)
2884                                 decoder->vmcs = decoder->packet.payload;
2885                         break;
2886
2887                 case INTEL_PT_BBP:
2888                         decoder->blk_type = decoder->packet.payload;
2889                         break;
2890
2891                 case INTEL_PT_BIP:
2892                         if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2893                             decoder->packet.count == 2)
2894                                 intel_pt_vm_tm_corr_pebs_tsc(decoder);
2895                         break;
2896
2897                 case INTEL_PT_BEP:
2898                 case INTEL_PT_BEP_IP:
2899                         decoder->blk_type = 0;
2900                         break;
2901
2902                 case INTEL_PT_CFE:
2903                 case INTEL_PT_CFE_IP:
2904                 case INTEL_PT_EVD:
2905                 case INTEL_PT_MODE_EXEC:
2906                 case INTEL_PT_MODE_TSX:
2907                 case INTEL_PT_MNT:
2908                 case INTEL_PT_PAD:
2909                 case INTEL_PT_PTWRITE_IP:
2910                 case INTEL_PT_PTWRITE:
2911                 case INTEL_PT_MWAIT:
2912                 case INTEL_PT_PWRE:
2913                 case INTEL_PT_EXSTOP_IP:
2914                 case INTEL_PT_EXSTOP:
2915                 case INTEL_PT_PWRX:
2916                 case INTEL_PT_BAD: /* Does not happen */
2917                 default:
2918                         break;
2919                 }
2920         }
2921
2922         return err;
2923 }
2924
2925 #define HOP_PROCESS     0
2926 #define HOP_IGNORE      1
2927 #define HOP_RETURN      2
2928 #define HOP_AGAIN       3
2929
2930 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2931
2932 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2933 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
2934 {
2935         *err = 0;
2936
2937         /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2938         if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2939                 *err = intel_pt_scan_for_psb(decoder);
2940                 if (*err)
2941                         return HOP_RETURN;
2942         }
2943
2944         switch (decoder->packet.type) {
2945         case INTEL_PT_TNT:
2946                 return HOP_IGNORE;
2947
2948         case INTEL_PT_TIP_PGD:
2949                 decoder->pge = false;
2950                 if (!decoder->packet.count) {
2951                         intel_pt_set_nr(decoder);
2952                         return HOP_IGNORE;
2953                 }
2954                 intel_pt_set_ip(decoder);
2955                 decoder->state.type |= INTEL_PT_TRACE_END;
2956                 decoder->state.from_ip = 0;
2957                 decoder->state.to_ip = decoder->ip;
2958                 intel_pt_update_nr(decoder);
2959                 return HOP_RETURN;
2960
2961         case INTEL_PT_TIP:
2962                 if (!decoder->packet.count) {
2963                         intel_pt_set_nr(decoder);
2964                         return HOP_IGNORE;
2965                 }
2966                 intel_pt_set_ip(decoder);
2967                 decoder->state.type = INTEL_PT_INSTRUCTION;
2968                 decoder->state.from_ip = decoder->ip;
2969                 decoder->state.to_ip = 0;
2970                 intel_pt_update_nr(decoder);
2971                 intel_pt_sample_iflag_chg(decoder);
2972                 return HOP_RETURN;
2973
2974         case INTEL_PT_FUP:
2975                 if (!decoder->packet.count)
2976                         return HOP_IGNORE;
2977                 intel_pt_set_ip(decoder);
2978                 if (decoder->set_fup_mwait || decoder->set_fup_pwre)
2979                         *no_tip = true;
2980                 if (!decoder->branch_enable || !decoder->pge)
2981                         *no_tip = true;
2982                 if (*no_tip) {
2983                         decoder->state.type = INTEL_PT_INSTRUCTION;
2984                         decoder->state.from_ip = decoder->ip;
2985                         decoder->state.to_ip = 0;
2986                         intel_pt_fup_event(decoder, *no_tip);
2987                         return HOP_RETURN;
2988                 }
2989                 intel_pt_fup_event(decoder, *no_tip);
2990                 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH;
2991                 *err = intel_pt_walk_fup_tip(decoder);
2992                 if (!*err && decoder->state.to_ip)
2993                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2994                 return HOP_RETURN;
2995
2996         case INTEL_PT_PSB:
2997                 decoder->state.psb_offset = decoder->pos;
2998                 decoder->psb_ip = 0;
2999                 decoder->last_ip = 0;
3000                 decoder->have_last_ip = true;
3001                 *err = intel_pt_walk_psbend(decoder);
3002                 if (*err == -EAGAIN)
3003                         return HOP_AGAIN;
3004                 if (*err)
3005                         return HOP_RETURN;
3006                 decoder->state.type = INTEL_PT_PSB_EVT;
3007                 if (decoder->psb_ip) {
3008                         decoder->state.type |= INTEL_PT_INSTRUCTION;
3009                         decoder->ip = decoder->psb_ip;
3010                 }
3011                 decoder->state.from_ip = decoder->psb_ip;
3012                 decoder->state.to_ip = 0;
3013                 return HOP_RETURN;
3014
3015         case INTEL_PT_BAD:
3016         case INTEL_PT_PAD:
3017         case INTEL_PT_TIP_PGE:
3018         case INTEL_PT_TSC:
3019         case INTEL_PT_TMA:
3020         case INTEL_PT_MODE_EXEC:
3021         case INTEL_PT_MODE_TSX:
3022         case INTEL_PT_MTC:
3023         case INTEL_PT_CYC:
3024         case INTEL_PT_VMCS:
3025         case INTEL_PT_PSBEND:
3026         case INTEL_PT_CBR:
3027         case INTEL_PT_TRACESTOP:
3028         case INTEL_PT_PIP:
3029         case INTEL_PT_OVF:
3030         case INTEL_PT_MNT:
3031         case INTEL_PT_PTWRITE:
3032         case INTEL_PT_PTWRITE_IP:
3033         case INTEL_PT_EXSTOP:
3034         case INTEL_PT_EXSTOP_IP:
3035         case INTEL_PT_MWAIT:
3036         case INTEL_PT_PWRE:
3037         case INTEL_PT_PWRX:
3038         case INTEL_PT_BBP:
3039         case INTEL_PT_BIP:
3040         case INTEL_PT_BEP:
3041         case INTEL_PT_BEP_IP:
3042         case INTEL_PT_CFE:
3043         case INTEL_PT_CFE_IP:
3044         case INTEL_PT_EVD:
3045         default:
3046                 return HOP_PROCESS;
3047         }
3048 }
3049
3050 struct intel_pt_psb_info {
3051         struct intel_pt_pkt fup_packet;
3052         bool fup;
3053         int after_psbend;
3054 };
3055
3056 /* Lookahead and get the FUP packet from PSB+ */
3057 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
3058 {
3059         struct intel_pt_psb_info *data = pkt_info->data;
3060
3061         switch (pkt_info->packet.type) {
3062         case INTEL_PT_PAD:
3063         case INTEL_PT_MNT:
3064         case INTEL_PT_TSC:
3065         case INTEL_PT_TMA:
3066         case INTEL_PT_MODE_EXEC:
3067         case INTEL_PT_MODE_TSX:
3068         case INTEL_PT_MTC:
3069         case INTEL_PT_CYC:
3070         case INTEL_PT_VMCS:
3071         case INTEL_PT_CBR:
3072         case INTEL_PT_PIP:
3073                 if (data->after_psbend) {
3074                         data->after_psbend -= 1;
3075                         if (!data->after_psbend)
3076                                 return 1;
3077                 }
3078                 break;
3079
3080         case INTEL_PT_FUP:
3081                 if (data->after_psbend)
3082                         return 1;
3083                 if (data->fup || pkt_info->packet.count == 0)
3084                         return 1;
3085                 data->fup_packet = pkt_info->packet;
3086                 data->fup = true;
3087                 break;
3088
3089         case INTEL_PT_PSBEND:
3090                 if (!data->fup)
3091                         return 1;
3092                 /* Keep going to check for a TIP.PGE */
3093                 data->after_psbend = 6;
3094                 break;
3095
3096         case INTEL_PT_TIP_PGE:
3097                 /* Ignore FUP in PSB+ if followed by TIP.PGE */
3098                 if (data->after_psbend)
3099                         data->fup = false;
3100                 return 1;
3101
3102         case INTEL_PT_PTWRITE:
3103         case INTEL_PT_PTWRITE_IP:
3104         case INTEL_PT_EXSTOP:
3105         case INTEL_PT_EXSTOP_IP:
3106         case INTEL_PT_MWAIT:
3107         case INTEL_PT_PWRE:
3108         case INTEL_PT_PWRX:
3109         case INTEL_PT_BBP:
3110         case INTEL_PT_BIP:
3111         case INTEL_PT_BEP:
3112         case INTEL_PT_BEP_IP:
3113         case INTEL_PT_CFE:
3114         case INTEL_PT_CFE_IP:
3115         case INTEL_PT_EVD:
3116                 if (data->after_psbend) {
3117                         data->after_psbend -= 1;
3118                         if (!data->after_psbend)
3119                                 return 1;
3120                         break;
3121                 }
3122                 return 1;
3123
3124         case INTEL_PT_OVF:
3125         case INTEL_PT_BAD:
3126         case INTEL_PT_TNT:
3127         case INTEL_PT_TIP_PGD:
3128         case INTEL_PT_TIP:
3129         case INTEL_PT_PSB:
3130         case INTEL_PT_TRACESTOP:
3131         default:
3132                 return 1;
3133         }
3134
3135         return 0;
3136 }
3137
3138 static int intel_pt_psb(struct intel_pt_decoder *decoder)
3139 {
3140         int err;
3141
3142         decoder->last_ip = 0;
3143         decoder->psb_ip = 0;
3144         decoder->have_last_ip = true;
3145         intel_pt_clear_stack(&decoder->stack);
3146         err = intel_pt_walk_psbend(decoder);
3147         if (err)
3148                 return err;
3149         decoder->state.type = INTEL_PT_PSB_EVT;
3150         decoder->state.from_ip = decoder->psb_ip;
3151         decoder->state.to_ip = 0;
3152         return 0;
3153 }
3154
3155 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
3156 {
3157         int err;
3158
3159         if (decoder->ip != decoder->last_ip) {
3160                 err = intel_pt_walk_fup(decoder);
3161                 if (!err || err != -EAGAIN)
3162                         return err;
3163         }
3164
3165         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3166         err = intel_pt_psb(decoder);
3167         if (err) {
3168                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3169                 return -ENOENT;
3170         }
3171
3172         return 0;
3173 }
3174
3175 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
3176 {
3177         struct intel_pt_psb_info data = { .fup = false };
3178
3179         if (!decoder->branch_enable)
3180                 return false;
3181
3182         intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
3183         if (!data.fup)
3184                 return false;
3185
3186         decoder->packet = data.fup_packet;
3187         intel_pt_set_last_ip(decoder);
3188         decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
3189
3190         *err = intel_pt_fup_in_psb(decoder);
3191
3192         return true;
3193 }
3194
3195 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
3196 {
3197         int last_packet_type = INTEL_PT_PAD;
3198         bool no_tip = false;
3199         int err;
3200
3201         while (1) {
3202                 err = intel_pt_get_next_packet(decoder);
3203                 if (err)
3204                         return err;
3205 next:
3206                 err = 0;
3207                 if (decoder->cyc_threshold) {
3208                         if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
3209                                 decoder->sample_cyc = false;
3210                         last_packet_type = decoder->packet.type;
3211                 }
3212
3213                 if (decoder->hop) {
3214                         switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
3215                         case HOP_IGNORE:
3216                                 continue;
3217                         case HOP_RETURN:
3218                                 return err;
3219                         case HOP_AGAIN:
3220                                 goto next;
3221                         default:
3222                                 break;
3223                         }
3224                 }
3225
3226                 switch (decoder->packet.type) {
3227                 case INTEL_PT_TNT:
3228                         if (!decoder->packet.count)
3229                                 break;
3230                         decoder->tnt = decoder->packet;
3231                         decoder->pkt_state = INTEL_PT_STATE_TNT;
3232                         err = intel_pt_walk_tnt(decoder);
3233                         if (err == -EAGAIN)
3234                                 break;
3235                         return err;
3236
3237                 case INTEL_PT_TIP_PGD:
3238                         if (decoder->packet.count != 0)
3239                                 intel_pt_set_last_ip(decoder);
3240                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
3241                         return intel_pt_walk_tip(decoder);
3242
3243                 case INTEL_PT_TIP_PGE: {
3244                         decoder->pge = true;
3245                         decoder->overflow = false;
3246                         intel_pt_mtc_cyc_cnt_pge(decoder);
3247                         intel_pt_set_nr(decoder);
3248                         if (decoder->packet.count == 0) {
3249                                 intel_pt_log_at("Skipping zero TIP.PGE",
3250                                                 decoder->pos);
3251                                 break;
3252                         }
3253                         intel_pt_sample_iflag_chg(decoder);
3254                         intel_pt_set_ip(decoder);
3255                         decoder->state.from_ip = 0;
3256                         decoder->state.to_ip = decoder->ip;
3257                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3258                         /*
3259                          * In hop mode, resample to get the to_ip as an
3260                          * "instruction" sample.
3261                          */
3262                         if (decoder->hop)
3263                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3264                         return 0;
3265                 }
3266
3267                 case INTEL_PT_OVF:
3268                         return intel_pt_overflow(decoder);
3269
3270                 case INTEL_PT_TIP:
3271                         if (decoder->packet.count != 0)
3272                                 intel_pt_set_last_ip(decoder);
3273                         decoder->pkt_state = INTEL_PT_STATE_TIP;
3274                         return intel_pt_walk_tip(decoder);
3275
3276                 case INTEL_PT_FUP:
3277                         if (decoder->packet.count == 0) {
3278                                 intel_pt_log_at("Skipping zero FUP",
3279                                                 decoder->pos);
3280                                 no_tip = false;
3281                                 break;
3282                         }
3283                         intel_pt_set_last_ip(decoder);
3284                         if (!decoder->branch_enable || !decoder->pge) {
3285                                 decoder->ip = decoder->last_ip;
3286                                 if (intel_pt_fup_event(decoder, no_tip))
3287                                         return 0;
3288                                 no_tip = false;
3289                                 break;
3290                         }
3291                         if (decoder->set_fup_mwait)
3292                                 no_tip = true;
3293                         if (no_tip)
3294                                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3295                         else
3296                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
3297                         err = intel_pt_walk_fup(decoder);
3298                         if (err != -EAGAIN)
3299                                 return err;
3300                         if (no_tip) {
3301                                 no_tip = false;
3302                                 break;
3303                         }
3304                         return intel_pt_walk_fup_tip(decoder);
3305
3306                 case INTEL_PT_TRACESTOP:
3307                         decoder->pge = false;
3308                         decoder->continuous_period = false;
3309                         intel_pt_clear_tx_flags(decoder);
3310                         decoder->have_tma = false;
3311                         break;
3312
3313                 case INTEL_PT_PSB:
3314                         decoder->state.psb_offset = decoder->pos;
3315                         decoder->psb_ip = 0;
3316                         if (intel_pt_psb_with_fup(decoder, &err))
3317                                 return err;
3318                         err = intel_pt_psb(decoder);
3319                         if (err == -EAGAIN)
3320                                 goto next;
3321                         return err;
3322
3323                 case INTEL_PT_PIP:
3324                         intel_pt_update_pip(decoder);
3325                         break;
3326
3327                 case INTEL_PT_MTC:
3328                         intel_pt_calc_mtc_timestamp(decoder);
3329                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3330                                 break;
3331                         /*
3332                          * Ensure that there has been an instruction since the
3333                          * last MTC.
3334                          */
3335                         if (!decoder->mtc_insn)
3336                                 break;
3337                         decoder->mtc_insn = false;
3338                         /* Ensure that there is a timestamp */
3339                         if (!decoder->timestamp)
3340                                 break;
3341                         decoder->state.type = INTEL_PT_INSTRUCTION;
3342                         decoder->state.from_ip = decoder->ip;
3343                         decoder->state.to_ip = 0;
3344                         decoder->mtc_insn = false;
3345                         return 0;
3346
3347                 case INTEL_PT_TSC:
3348                         intel_pt_calc_tsc_timestamp(decoder);
3349                         break;
3350
3351                 case INTEL_PT_TMA:
3352                         intel_pt_calc_tma(decoder);
3353                         break;
3354
3355                 case INTEL_PT_CYC:
3356                         intel_pt_calc_cyc_timestamp(decoder);
3357                         break;
3358
3359                 case INTEL_PT_CBR:
3360                         intel_pt_calc_cbr(decoder);
3361                         if (decoder->cbr != decoder->cbr_seen) {
3362                                 decoder->state.type = 0;
3363                                 return 0;
3364                         }
3365                         break;
3366
3367                 case INTEL_PT_MODE_EXEC:
3368                         intel_pt_mode_exec(decoder);
3369                         err = intel_pt_get_next_packet(decoder);
3370                         if (err)
3371                                 return err;
3372                         if (decoder->packet.type == INTEL_PT_FUP) {
3373                                 decoder->set_fup_mode_exec = true;
3374                                 no_tip = true;
3375                         }
3376                         goto next;
3377
3378                 case INTEL_PT_MODE_TSX:
3379                         /* MODE_TSX need not be followed by FUP */
3380                         if (!decoder->pge || decoder->in_psb) {
3381                                 intel_pt_update_in_tx(decoder);
3382                                 break;
3383                         }
3384                         err = intel_pt_mode_tsx(decoder, &no_tip);
3385                         if (err)
3386                                 return err;
3387                         goto next;
3388
3389                 case INTEL_PT_BAD: /* Does not happen */
3390                         return intel_pt_bug(decoder);
3391
3392                 case INTEL_PT_PSBEND:
3393                 case INTEL_PT_VMCS:
3394                 case INTEL_PT_MNT:
3395                 case INTEL_PT_PAD:
3396                         break;
3397
3398                 case INTEL_PT_PTWRITE_IP:
3399                         decoder->fup_ptw_payload = decoder->packet.payload;
3400                         err = intel_pt_get_next_packet(decoder);
3401                         if (err)
3402                                 return err;
3403                         if (decoder->packet.type == INTEL_PT_FUP) {
3404                                 decoder->set_fup_ptw = true;
3405                                 no_tip = true;
3406                         } else {
3407                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3408                                                 decoder->pos);
3409                         }
3410                         goto next;
3411
3412                 case INTEL_PT_PTWRITE:
3413                         decoder->state.type = INTEL_PT_PTW;
3414                         decoder->state.from_ip = decoder->ip;
3415                         decoder->state.to_ip = 0;
3416                         decoder->state.ptw_payload = decoder->packet.payload;
3417                         return 0;
3418
3419                 case INTEL_PT_MWAIT:
3420                         decoder->fup_mwait_payload = decoder->packet.payload;
3421                         decoder->set_fup_mwait = true;
3422                         break;
3423
3424                 case INTEL_PT_PWRE:
3425                         if (decoder->set_fup_mwait) {
3426                                 decoder->fup_pwre_payload =
3427                                                         decoder->packet.payload;
3428                                 decoder->set_fup_pwre = true;
3429                                 break;
3430                         }
3431                         decoder->state.type = INTEL_PT_PWR_ENTRY;
3432                         decoder->state.from_ip = decoder->ip;
3433                         decoder->state.to_ip = 0;
3434                         decoder->state.pwrx_payload = decoder->packet.payload;
3435                         return 0;
3436
3437                 case INTEL_PT_EXSTOP_IP:
3438                         err = intel_pt_get_next_packet(decoder);
3439                         if (err)
3440                                 return err;
3441                         if (decoder->packet.type == INTEL_PT_FUP) {
3442                                 decoder->set_fup_exstop = true;
3443                                 no_tip = true;
3444                         } else {
3445                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3446                                                 decoder->pos);
3447                         }
3448                         goto next;
3449
3450                 case INTEL_PT_EXSTOP:
3451                         decoder->state.type = INTEL_PT_EX_STOP;
3452                         decoder->state.from_ip = decoder->ip;
3453                         decoder->state.to_ip = 0;
3454                         return 0;
3455
3456                 case INTEL_PT_PWRX:
3457                         decoder->state.type = INTEL_PT_PWR_EXIT;
3458                         decoder->state.from_ip = decoder->ip;
3459                         decoder->state.to_ip = 0;
3460                         decoder->state.pwrx_payload = decoder->packet.payload;
3461                         return 0;
3462
3463                 case INTEL_PT_BBP:
3464                         intel_pt_bbp(decoder);
3465                         break;
3466
3467                 case INTEL_PT_BIP:
3468                         intel_pt_bip(decoder);
3469                         break;
3470
3471                 case INTEL_PT_BEP:
3472                         decoder->state.type = INTEL_PT_BLK_ITEMS;
3473                         decoder->state.from_ip = decoder->ip;
3474                         decoder->state.to_ip = 0;
3475                         return 0;
3476
3477                 case INTEL_PT_BEP_IP:
3478                         err = intel_pt_get_next_packet(decoder);
3479                         if (err)
3480                                 return err;
3481                         if (decoder->packet.type == INTEL_PT_FUP) {
3482                                 decoder->set_fup_bep = true;
3483                                 no_tip = true;
3484                         } else {
3485                                 intel_pt_log_at("ERROR: Missing FUP after BEP",
3486                                                 decoder->pos);
3487                         }
3488                         goto next;
3489
3490                 case INTEL_PT_CFE:
3491                         decoder->fup_cfe_pkt = decoder->packet;
3492                         decoder->set_fup_cfe = true;
3493                         if (!decoder->pge) {
3494                                 intel_pt_fup_event(decoder, true);
3495                                 return 0;
3496                         }
3497                         break;
3498
3499                 case INTEL_PT_CFE_IP:
3500                         decoder->fup_cfe_pkt = decoder->packet;
3501                         err = intel_pt_get_next_packet(decoder);
3502                         if (err)
3503                                 return err;
3504                         if (decoder->packet.type == INTEL_PT_FUP) {
3505                                 decoder->set_fup_cfe_ip = true;
3506                                 no_tip = true;
3507                         } else {
3508                                 intel_pt_log_at("ERROR: Missing FUP after CFE",
3509                                                 decoder->pos);
3510                         }
3511                         goto next;
3512
3513                 case INTEL_PT_EVD:
3514                         err = intel_pt_evd(decoder);
3515                         if (err)
3516                                 return err;
3517                         break;
3518
3519                 default:
3520                         return intel_pt_bug(decoder);
3521                 }
3522         }
3523 }
3524
3525 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3526 {
3527         return decoder->packet.count &&
3528                (decoder->have_last_ip || decoder->packet.count == 3 ||
3529                 decoder->packet.count == 6);
3530 }
3531
3532 /* Walk PSB+ packets to get in sync. */
3533 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3534 {
3535         int err;
3536
3537         decoder->in_psb = true;
3538
3539         while (1) {
3540                 err = intel_pt_get_next_packet(decoder);
3541                 if (err)
3542                         goto out;
3543
3544                 switch (decoder->packet.type) {
3545                 case INTEL_PT_TIP_PGD:
3546                         decoder->continuous_period = false;
3547                         __fallthrough;
3548                 case INTEL_PT_TIP_PGE:
3549                 case INTEL_PT_TIP:
3550                 case INTEL_PT_PTWRITE:
3551                 case INTEL_PT_PTWRITE_IP:
3552                 case INTEL_PT_EXSTOP:
3553                 case INTEL_PT_EXSTOP_IP:
3554                 case INTEL_PT_MWAIT:
3555                 case INTEL_PT_PWRE:
3556                 case INTEL_PT_PWRX:
3557                 case INTEL_PT_BBP:
3558                 case INTEL_PT_BIP:
3559                 case INTEL_PT_BEP:
3560                 case INTEL_PT_BEP_IP:
3561                 case INTEL_PT_CFE:
3562                 case INTEL_PT_CFE_IP:
3563                 case INTEL_PT_EVD:
3564                         intel_pt_log("ERROR: Unexpected packet\n");
3565                         err = -ENOENT;
3566                         goto out;
3567
3568                 case INTEL_PT_FUP:
3569                         decoder->pge = true;
3570                         if (intel_pt_have_ip(decoder)) {
3571                                 uint64_t current_ip = decoder->ip;
3572
3573                                 intel_pt_set_ip(decoder);
3574                                 decoder->psb_ip = decoder->ip;
3575                                 if (current_ip)
3576                                         intel_pt_log_to("Setting IP",
3577                                                         decoder->ip);
3578                         }
3579                         break;
3580
3581                 case INTEL_PT_MTC:
3582                         intel_pt_calc_mtc_timestamp(decoder);
3583                         break;
3584
3585                 case INTEL_PT_TSC:
3586                         intel_pt_calc_tsc_timestamp(decoder);
3587                         break;
3588
3589                 case INTEL_PT_TMA:
3590                         intel_pt_calc_tma(decoder);
3591                         break;
3592
3593                 case INTEL_PT_CYC:
3594                         intel_pt_calc_cyc_timestamp(decoder);
3595                         break;
3596
3597                 case INTEL_PT_CBR:
3598                         intel_pt_calc_cbr(decoder);
3599                         break;
3600
3601                 case INTEL_PT_PIP:
3602                         intel_pt_set_pip(decoder);
3603                         break;
3604
3605                 case INTEL_PT_MODE_EXEC:
3606                         intel_pt_mode_exec_status(decoder);
3607                         break;
3608
3609                 case INTEL_PT_MODE_TSX:
3610                         intel_pt_update_in_tx(decoder);
3611                         break;
3612
3613                 case INTEL_PT_TRACESTOP:
3614                         decoder->pge = false;
3615                         decoder->continuous_period = false;
3616                         intel_pt_clear_tx_flags(decoder);
3617                         __fallthrough;
3618
3619                 case INTEL_PT_TNT:
3620                         decoder->have_tma = false;
3621                         intel_pt_log("ERROR: Unexpected packet\n");
3622                         if (decoder->ip)
3623                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
3624                         else
3625                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3626                         err = -ENOENT;
3627                         goto out;
3628
3629                 case INTEL_PT_BAD: /* Does not happen */
3630                         err = intel_pt_bug(decoder);
3631                         goto out;
3632
3633                 case INTEL_PT_OVF:
3634                         err = intel_pt_overflow(decoder);
3635                         goto out;
3636
3637                 case INTEL_PT_PSBEND:
3638                         err = 0;
3639                         goto out;
3640
3641                 case INTEL_PT_PSB:
3642                 case INTEL_PT_VMCS:
3643                 case INTEL_PT_MNT:
3644                 case INTEL_PT_PAD:
3645                 default:
3646                         break;
3647                 }
3648         }
3649 out:
3650         decoder->in_psb = false;
3651
3652         return err;
3653 }
3654
3655 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3656 {
3657         int err;
3658
3659         while (1) {
3660                 err = intel_pt_get_next_packet(decoder);
3661                 if (err)
3662                         return err;
3663
3664                 switch (decoder->packet.type) {
3665                 case INTEL_PT_TIP_PGD:
3666                         decoder->continuous_period = false;
3667                         decoder->pge = false;
3668                         if (intel_pt_have_ip(decoder))
3669                                 intel_pt_set_ip(decoder);
3670                         if (!decoder->ip)
3671                                 break;
3672                         decoder->state.type |= INTEL_PT_TRACE_END;
3673                         return 0;
3674
3675                 case INTEL_PT_TIP_PGE:
3676                         decoder->pge = true;
3677                         intel_pt_mtc_cyc_cnt_pge(decoder);
3678                         if (intel_pt_have_ip(decoder))
3679                                 intel_pt_set_ip(decoder);
3680                         if (!decoder->ip)
3681                                 break;
3682                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3683                         return 0;
3684
3685                 case INTEL_PT_TIP:
3686                         decoder->pge = true;
3687                         if (intel_pt_have_ip(decoder))
3688                                 intel_pt_set_ip(decoder);
3689                         if (!decoder->ip)
3690                                 break;
3691                         return 0;
3692
3693                 case INTEL_PT_FUP:
3694                         if (intel_pt_have_ip(decoder))
3695                                 intel_pt_set_ip(decoder);
3696                         if (decoder->ip)
3697                                 return 0;
3698                         break;
3699
3700                 case INTEL_PT_MTC:
3701                         intel_pt_calc_mtc_timestamp(decoder);
3702                         break;
3703
3704                 case INTEL_PT_TSC:
3705                         intel_pt_calc_tsc_timestamp(decoder);
3706                         break;
3707
3708                 case INTEL_PT_TMA:
3709                         intel_pt_calc_tma(decoder);
3710                         break;
3711
3712                 case INTEL_PT_CYC:
3713                         intel_pt_calc_cyc_timestamp(decoder);
3714                         break;
3715
3716                 case INTEL_PT_CBR:
3717                         intel_pt_calc_cbr(decoder);
3718                         break;
3719
3720                 case INTEL_PT_PIP:
3721                         intel_pt_set_pip(decoder);
3722                         break;
3723
3724                 case INTEL_PT_MODE_EXEC:
3725                         intel_pt_mode_exec_status(decoder);
3726                         break;
3727
3728                 case INTEL_PT_MODE_TSX:
3729                         intel_pt_update_in_tx(decoder);
3730                         break;
3731
3732                 case INTEL_PT_OVF:
3733                         return intel_pt_overflow(decoder);
3734
3735                 case INTEL_PT_BAD: /* Does not happen */
3736                         return intel_pt_bug(decoder);
3737
3738                 case INTEL_PT_TRACESTOP:
3739                         decoder->pge = false;
3740                         decoder->continuous_period = false;
3741                         intel_pt_clear_tx_flags(decoder);
3742                         decoder->have_tma = false;
3743                         break;
3744
3745                 case INTEL_PT_PSB:
3746                         decoder->state.psb_offset = decoder->pos;
3747                         decoder->psb_ip = 0;
3748                         decoder->last_ip = 0;
3749                         decoder->have_last_ip = true;
3750                         intel_pt_clear_stack(&decoder->stack);
3751                         err = intel_pt_walk_psb(decoder);
3752                         if (err)
3753                                 return err;
3754                         decoder->state.type = INTEL_PT_PSB_EVT;
3755                         decoder->state.from_ip = decoder->psb_ip;
3756                         decoder->state.to_ip = 0;
3757                         return 0;
3758
3759                 case INTEL_PT_TNT:
3760                 case INTEL_PT_PSBEND:
3761                 case INTEL_PT_VMCS:
3762                 case INTEL_PT_MNT:
3763                 case INTEL_PT_PAD:
3764                 case INTEL_PT_PTWRITE:
3765                 case INTEL_PT_PTWRITE_IP:
3766                 case INTEL_PT_EXSTOP:
3767                 case INTEL_PT_EXSTOP_IP:
3768                 case INTEL_PT_MWAIT:
3769                 case INTEL_PT_PWRE:
3770                 case INTEL_PT_PWRX:
3771                 case INTEL_PT_BBP:
3772                 case INTEL_PT_BIP:
3773                 case INTEL_PT_BEP:
3774                 case INTEL_PT_BEP_IP:
3775                 case INTEL_PT_CFE:
3776                 case INTEL_PT_CFE_IP:
3777                 case INTEL_PT_EVD:
3778                 default:
3779                         break;
3780                 }
3781         }
3782 }
3783
3784 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3785 {
3786         int err;
3787
3788         intel_pt_clear_fup_event(decoder);
3789         decoder->overflow = false;
3790
3791         if (!decoder->branch_enable) {
3792                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3793                 decoder->state.type = 0; /* Do not have a sample */
3794                 return 0;
3795         }
3796
3797         intel_pt_log("Scanning for full IP\n");
3798         err = intel_pt_walk_to_ip(decoder);
3799         if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3800                 return err;
3801
3802         /* In hop mode, resample to get the to_ip as an "instruction" sample */
3803         if (decoder->hop)
3804                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3805         else
3806                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3807
3808         decoder->state.from_ip = 0;
3809         decoder->state.to_ip = decoder->ip;
3810         intel_pt_log_to("Setting IP", decoder->ip);
3811
3812         return 0;
3813 }
3814
3815 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3816 {
3817         const unsigned char *end = decoder->buf + decoder->len;
3818         size_t i;
3819
3820         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3821                 if (i > decoder->len)
3822                         continue;
3823                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3824                         return i;
3825         }
3826         return 0;
3827 }
3828
3829 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3830 {
3831         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3832         const char *psb = INTEL_PT_PSB_STR;
3833
3834         if (rest_psb > decoder->len ||
3835             memcmp(decoder->buf, psb + part_psb, rest_psb))
3836                 return 0;
3837
3838         return rest_psb;
3839 }
3840
3841 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3842                                   int part_psb)
3843 {
3844         int rest_psb, ret;
3845
3846         decoder->pos += decoder->len;
3847         decoder->len = 0;
3848
3849         ret = intel_pt_get_next_data(decoder, false);
3850         if (ret)
3851                 return ret;
3852
3853         rest_psb = intel_pt_rest_psb(decoder, part_psb);
3854         if (!rest_psb)
3855                 return 0;
3856
3857         decoder->pos -= part_psb;
3858         decoder->next_buf = decoder->buf + rest_psb;
3859         decoder->next_len = decoder->len - rest_psb;
3860         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3861         decoder->buf = decoder->temp_buf;
3862         decoder->len = INTEL_PT_PSB_LEN;
3863
3864         return 0;
3865 }
3866
3867 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3868 {
3869         unsigned char *next;
3870         int ret;
3871
3872         intel_pt_log("Scanning for PSB\n");
3873         while (1) {
3874                 if (!decoder->len) {
3875                         ret = intel_pt_get_next_data(decoder, false);
3876                         if (ret)
3877                                 return ret;
3878                 }
3879
3880                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3881                               INTEL_PT_PSB_LEN);
3882                 if (!next) {
3883                         int part_psb;
3884
3885                         part_psb = intel_pt_part_psb(decoder);
3886                         if (part_psb) {
3887                                 ret = intel_pt_get_split_psb(decoder, part_psb);
3888                                 if (ret)
3889                                         return ret;
3890                         } else {
3891                                 decoder->pos += decoder->len;
3892                                 decoder->len = 0;
3893                         }
3894                         continue;
3895                 }
3896
3897                 decoder->pkt_step = next - decoder->buf;
3898                 return intel_pt_get_next_packet(decoder);
3899         }
3900 }
3901
3902 static int intel_pt_sync(struct intel_pt_decoder *decoder)
3903 {
3904         int err;
3905
3906         decoder->pge = false;
3907         decoder->continuous_period = false;
3908         decoder->have_last_ip = false;
3909         decoder->last_ip = 0;
3910         decoder->psb_ip = 0;
3911         decoder->ip = 0;
3912         intel_pt_clear_stack(&decoder->stack);
3913
3914         err = intel_pt_scan_for_psb(decoder);
3915         if (err)
3916                 return err;
3917
3918         if (decoder->vm_time_correlation) {
3919                 decoder->in_psb = true;
3920                 if (!decoder->timestamp)
3921                         decoder->timestamp = 1;
3922                 decoder->state.type = 0;
3923                 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3924                 return 0;
3925         }
3926
3927         decoder->have_last_ip = true;
3928         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3929
3930         err = intel_pt_walk_psb(decoder);
3931         if (err)
3932                 return err;
3933
3934         decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3935         decoder->state.from_ip = decoder->psb_ip;
3936         decoder->state.to_ip = 0;
3937
3938         if (decoder->ip) {
3939                 /*
3940                  * In hop mode, resample to get the PSB FUP ip as an
3941                  * "instruction" sample.
3942                  */
3943                 if (decoder->hop)
3944                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3945                 else
3946                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3947         }
3948
3949         return 0;
3950 }
3951
3952 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3953 {
3954         uint64_t est = decoder->sample_insn_cnt << 1;
3955
3956         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
3957                 goto out;
3958
3959         est *= decoder->max_non_turbo_ratio;
3960         est /= decoder->cbr;
3961 out:
3962         return decoder->sample_timestamp + est;
3963 }
3964
3965 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
3966 {
3967         int err;
3968
3969         do {
3970                 decoder->state.type = INTEL_PT_BRANCH;
3971                 decoder->state.flags = 0;
3972
3973                 switch (decoder->pkt_state) {
3974                 case INTEL_PT_STATE_NO_PSB:
3975                         err = intel_pt_sync(decoder);
3976                         break;
3977                 case INTEL_PT_STATE_NO_IP:
3978                         decoder->have_last_ip = false;
3979                         decoder->last_ip = 0;
3980                         decoder->ip = 0;
3981                         __fallthrough;
3982                 case INTEL_PT_STATE_ERR_RESYNC:
3983                         err = intel_pt_sync_ip(decoder);
3984                         break;
3985                 case INTEL_PT_STATE_IN_SYNC:
3986                         err = intel_pt_walk_trace(decoder);
3987                         break;
3988                 case INTEL_PT_STATE_TNT:
3989                 case INTEL_PT_STATE_TNT_CONT:
3990                         err = intel_pt_walk_tnt(decoder);
3991                         if (err == -EAGAIN)
3992                                 err = intel_pt_walk_trace(decoder);
3993                         break;
3994                 case INTEL_PT_STATE_TIP:
3995                 case INTEL_PT_STATE_TIP_PGD:
3996                         err = intel_pt_walk_tip(decoder);
3997                         break;
3998                 case INTEL_PT_STATE_FUP:
3999                         err = intel_pt_walk_fup(decoder);
4000                         if (err == -EAGAIN)
4001                                 err = intel_pt_walk_fup_tip(decoder);
4002                         break;
4003                 case INTEL_PT_STATE_FUP_NO_TIP:
4004                         err = intel_pt_walk_fup(decoder);
4005                         if (err == -EAGAIN)
4006                                 err = intel_pt_walk_trace(decoder);
4007                         break;
4008                 case INTEL_PT_STATE_FUP_IN_PSB:
4009                         err = intel_pt_fup_in_psb(decoder);
4010                         break;
4011                 case INTEL_PT_STATE_RESAMPLE:
4012                         err = intel_pt_resample(decoder);
4013                         break;
4014                 case INTEL_PT_STATE_VM_TIME_CORRELATION:
4015                         err = intel_pt_vm_time_correlation(decoder);
4016                         break;
4017                 default:
4018                         err = intel_pt_bug(decoder);
4019                         break;
4020                 }
4021         } while (err == -ENOLINK);
4022
4023         if (err) {
4024                 decoder->state.err = intel_pt_ext_err(err);
4025                 if (err != -EOVERFLOW)
4026                         decoder->state.from_ip = decoder->ip;
4027                 intel_pt_update_sample_time(decoder);
4028                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
4029                 intel_pt_set_nr(decoder);
4030         } else {
4031                 decoder->state.err = 0;
4032                 if (decoder->cbr != decoder->cbr_seen) {
4033                         decoder->cbr_seen = decoder->cbr;
4034                         if (!decoder->state.type) {
4035                                 decoder->state.from_ip = decoder->ip;
4036                                 decoder->state.to_ip = 0;
4037                         }
4038                         decoder->state.type |= INTEL_PT_CBR_CHG;
4039                         decoder->state.cbr_payload = decoder->cbr_payload;
4040                         decoder->state.cbr = decoder->cbr;
4041                 }
4042                 if (intel_pt_sample_time(decoder->pkt_state)) {
4043                         intel_pt_update_sample_time(decoder);
4044                         if (decoder->sample_cyc) {
4045                                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
4046                                 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
4047                                 decoder->sample_cyc = false;
4048                         }
4049                 }
4050                 /*
4051                  * When using only TSC/MTC to compute cycles, IPC can be
4052                  * sampled as soon as the cycle count changes.
4053                  */
4054                 if (!decoder->have_cyc)
4055                         decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
4056         }
4057
4058          /* Let PSB event always have TSC timestamp */
4059         if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
4060                 decoder->sample_timestamp = decoder->tsc_timestamp;
4061
4062         decoder->state.from_nr = decoder->nr;
4063         decoder->state.to_nr = decoder->next_nr;
4064         decoder->nr = decoder->next_nr;
4065
4066         decoder->state.timestamp = decoder->sample_timestamp;
4067         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
4068         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
4069         decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
4070
4071         return &decoder->state;
4072 }
4073
4074 /**
4075  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
4076  * @buf: pointer to buffer pointer
4077  * @len: size of buffer
4078  *
4079  * Updates the buffer pointer to point to the start of the next PSB packet if
4080  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
4081  * @len is adjusted accordingly.
4082  *
4083  * Return: %true if a PSB packet is found, %false otherwise.
4084  */
4085 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
4086 {
4087         unsigned char *next;
4088
4089         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
4090         if (next) {
4091                 *len -= next - *buf;
4092                 *buf = next;
4093                 return true;
4094         }
4095         return false;
4096 }
4097
4098 /**
4099  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
4100  *                     packet.
4101  * @buf: pointer to buffer pointer
4102  * @len: size of buffer
4103  *
4104  * Updates the buffer pointer to point to the start of the following PSB packet
4105  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
4106  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
4107  *
4108  * Return: %true if a PSB packet is found, %false otherwise.
4109  */
4110 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
4111 {
4112         unsigned char *next;
4113
4114         if (!*len)
4115                 return false;
4116
4117         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
4118         if (next) {
4119                 *len -= next - *buf;
4120                 *buf = next;
4121                 return true;
4122         }
4123         return false;
4124 }
4125
4126 /**
4127  * intel_pt_last_psb - find the last PSB packet in a buffer.
4128  * @buf: buffer
4129  * @len: size of buffer
4130  *
4131  * This function finds the last PSB in a buffer.
4132  *
4133  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
4134  */
4135 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
4136 {
4137         const char *n = INTEL_PT_PSB_STR;
4138         unsigned char *p;
4139         size_t k;
4140
4141         if (len < INTEL_PT_PSB_LEN)
4142                 return NULL;
4143
4144         k = len - INTEL_PT_PSB_LEN + 1;
4145         while (1) {
4146                 p = memrchr(buf, n[0], k);
4147                 if (!p)
4148                         return NULL;
4149                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
4150                         return p;
4151                 k = p - buf;
4152                 if (!k)
4153                         return NULL;
4154         }
4155 }
4156
4157 /**
4158  * intel_pt_next_tsc - find and return next TSC.
4159  * @buf: buffer
4160  * @len: size of buffer
4161  * @tsc: TSC value returned
4162  * @rem: returns remaining size when TSC is found
4163  *
4164  * Find a TSC packet in @buf and return the TSC value.  This function assumes
4165  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
4166  * PSBEND packet is found.
4167  *
4168  * Return: %true if TSC is found, false otherwise.
4169  */
4170 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
4171                               size_t *rem)
4172 {
4173         enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
4174         struct intel_pt_pkt packet;
4175         int ret;
4176
4177         while (len) {
4178                 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
4179                 if (ret <= 0)
4180                         return false;
4181                 if (packet.type == INTEL_PT_TSC) {
4182                         *tsc = packet.payload;
4183                         *rem = len;
4184                         return true;
4185                 }
4186                 if (packet.type == INTEL_PT_PSBEND)
4187                         return false;
4188                 buf += ret;
4189                 len -= ret;
4190         }
4191         return false;
4192 }
4193
4194 /**
4195  * intel_pt_tsc_cmp - compare 7-byte TSCs.
4196  * @tsc1: first TSC to compare
4197  * @tsc2: second TSC to compare
4198  *
4199  * This function compares 7-byte TSC values allowing for the possibility that
4200  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
4201  * around so for that purpose this function assumes the absolute difference is
4202  * less than half the maximum difference.
4203  *
4204  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
4205  * after @tsc2.
4206  */
4207 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
4208 {
4209         const uint64_t halfway = (1ULL << 55);
4210
4211         if (tsc1 == tsc2)
4212                 return 0;
4213
4214         if (tsc1 < tsc2) {
4215                 if (tsc2 - tsc1 < halfway)
4216                         return -1;
4217                 else
4218                         return 1;
4219         } else {
4220                 if (tsc1 - tsc2 < halfway)
4221                         return 1;
4222                 else
4223                         return -1;
4224         }
4225 }
4226
4227 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
4228
4229 /**
4230  * adj_for_padding - adjust overlap to account for padding.
4231  * @buf_b: second buffer
4232  * @buf_a: first buffer
4233  * @len_a: size of first buffer
4234  *
4235  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
4236  * accordingly.
4237  *
4238  * Return: A pointer into @buf_b from where non-overlapped data starts
4239  */
4240 static unsigned char *adj_for_padding(unsigned char *buf_b,
4241                                       unsigned char *buf_a, size_t len_a)
4242 {
4243         unsigned char *p = buf_b - MAX_PADDING;
4244         unsigned char *q = buf_a + len_a - MAX_PADDING;
4245         int i;
4246
4247         for (i = MAX_PADDING; i; i--, p++, q++) {
4248                 if (*p != *q)
4249                         break;
4250         }
4251
4252         return p;
4253 }
4254
4255 /**
4256  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
4257  *                             using TSC.
4258  * @buf_a: first buffer
4259  * @len_a: size of first buffer
4260  * @buf_b: second buffer
4261  * @len_b: size of second buffer
4262  * @consecutive: returns true if there is data in buf_b that is consecutive
4263  *               to buf_a
4264  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4265  *
4266  * If the trace contains TSC we can look at the last TSC of @buf_a and the
4267  * first TSC of @buf_b in order to determine if the buffers overlap, and then
4268  * walk forward in @buf_b until a later TSC is found.  A precondition is that
4269  * @buf_a and @buf_b are positioned at a PSB.
4270  *
4271  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4272  * @buf_b + @len_b if there is no non-overlapped data.
4273  */
4274 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
4275                                                 size_t len_a,
4276                                                 unsigned char *buf_b,
4277                                                 size_t len_b, bool *consecutive,
4278                                                 bool ooo_tsc)
4279 {
4280         uint64_t tsc_a, tsc_b;
4281         unsigned char *p;
4282         size_t len, rem_a, rem_b;
4283
4284         p = intel_pt_last_psb(buf_a, len_a);
4285         if (!p)
4286                 return buf_b; /* No PSB in buf_a => no overlap */
4287
4288         len = len_a - (p - buf_a);
4289         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
4290                 /* The last PSB+ in buf_a is incomplete, so go back one more */
4291                 len_a -= len;
4292                 p = intel_pt_last_psb(buf_a, len_a);
4293                 if (!p)
4294                         return buf_b; /* No full PSB+ => assume no overlap */
4295                 len = len_a - (p - buf_a);
4296                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
4297                         return buf_b; /* No TSC in buf_a => assume no overlap */
4298         }
4299
4300         while (1) {
4301                 /* Ignore PSB+ with no TSC */
4302                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
4303                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
4304
4305                         /* Same TSC, so buffers are consecutive */
4306                         if (!cmp && rem_b >= rem_a) {
4307                                 unsigned char *start;
4308
4309                                 *consecutive = true;
4310                                 start = buf_b + len_b - (rem_b - rem_a);
4311                                 return adj_for_padding(start, buf_a, len_a);
4312                         }
4313                         if (cmp < 0 && !ooo_tsc)
4314                                 return buf_b; /* tsc_a < tsc_b => no overlap */
4315                 }
4316
4317                 if (!intel_pt_step_psb(&buf_b, &len_b))
4318                         return buf_b + len_b; /* No PSB in buf_b => no data */
4319         }
4320 }
4321
4322 /**
4323  * intel_pt_find_overlap - determine start of non-overlapped trace data.
4324  * @buf_a: first buffer
4325  * @len_a: size of first buffer
4326  * @buf_b: second buffer
4327  * @len_b: size of second buffer
4328  * @have_tsc: can use TSC packets to detect overlap
4329  * @consecutive: returns true if there is data in buf_b that is consecutive
4330  *               to buf_a
4331  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4332  *
4333  * When trace samples or snapshots are recorded there is the possibility that
4334  * the data overlaps.  Note that, for the purposes of decoding, data is only
4335  * useful if it begins with a PSB packet.
4336  *
4337  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4338  * @buf_b + @len_b if there is no non-overlapped data.
4339  */
4340 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
4341                                      unsigned char *buf_b, size_t len_b,
4342                                      bool have_tsc, bool *consecutive,
4343                                      bool ooo_tsc)
4344 {
4345         unsigned char *found;
4346
4347         /* Buffer 'b' must start at PSB so throw away everything before that */
4348         if (!intel_pt_next_psb(&buf_b, &len_b))
4349                 return buf_b + len_b; /* No PSB */
4350
4351         if (!intel_pt_next_psb(&buf_a, &len_a))
4352                 return buf_b; /* No overlap */
4353
4354         if (have_tsc) {
4355                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4356                                                   consecutive, ooo_tsc);
4357                 if (found)
4358                         return found;
4359         }
4360
4361         /*
4362          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4363          * we can ignore the first part of buffer 'a'.
4364          */
4365         while (len_b < len_a) {
4366                 if (!intel_pt_step_psb(&buf_a, &len_a))
4367                         return buf_b; /* No overlap */
4368         }
4369
4370         /* Now len_b >= len_a */
4371         while (1) {
4372                 /* Potential overlap so check the bytes */
4373                 found = memmem(buf_a, len_a, buf_b, len_a);
4374                 if (found) {
4375                         *consecutive = true;
4376                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
4377                 }
4378
4379                 /* Try again at next PSB in buffer 'a' */
4380                 if (!intel_pt_step_psb(&buf_a, &len_a))
4381                         return buf_b; /* No overlap */
4382         }
4383 }
4384
4385 /**
4386  * struct fast_forward_data - data used by intel_pt_ff_cb().
4387  * @timestamp: timestamp to fast forward towards
4388  * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4389  *                 the fast forward timestamp.
4390  */
4391 struct fast_forward_data {
4392         uint64_t timestamp;
4393         uint64_t buf_timestamp;
4394 };
4395
4396 /**
4397  * intel_pt_ff_cb - fast forward lookahead callback.
4398  * @buffer: Intel PT trace buffer
4399  * @data: opaque pointer to fast forward data (struct fast_forward_data)
4400  *
4401  * Determine if @buffer trace is past the fast forward timestamp.
4402  *
4403  * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4404  *         timestamp, and 0 otherwise.
4405  */
4406 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
4407 {
4408         struct fast_forward_data *d = data;
4409         unsigned char *buf;
4410         uint64_t tsc;
4411         size_t rem;
4412         size_t len;
4413
4414         buf = (unsigned char *)buffer->buf;
4415         len = buffer->len;
4416
4417         if (!intel_pt_next_psb(&buf, &len) ||
4418             !intel_pt_next_tsc(buf, len, &tsc, &rem))
4419                 return 0;
4420
4421         tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4422
4423         intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4424                      tsc, buffer->ref_timestamp);
4425
4426         /*
4427          * If the buffer contains a timestamp earlier that the fast forward
4428          * timestamp, then record it, else stop.
4429          */
4430         if (tsc < d->timestamp)
4431                 d->buf_timestamp = buffer->ref_timestamp;
4432         else
4433                 return 1;
4434
4435         return 0;
4436 }
4437
4438 /**
4439  * intel_pt_fast_forward - reposition decoder forwards.
4440  * @decoder: Intel PT decoder
4441  * @timestamp: timestamp to fast forward towards
4442  *
4443  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4444  *
4445  * Return: 0 on success or negative error code on failure.
4446  */
4447 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4448 {
4449         struct fast_forward_data d = { .timestamp = timestamp };
4450         unsigned char *buf;
4451         size_t len;
4452         int err;
4453
4454         intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4455
4456         /* Find buffer timestamp of buffer to fast forward to */
4457         err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4458         if (err < 0)
4459                 return err;
4460
4461         /* Walk to buffer with same buffer timestamp */
4462         if (d.buf_timestamp) {
4463                 do {
4464                         decoder->pos += decoder->len;
4465                         decoder->len = 0;
4466                         err = intel_pt_get_next_data(decoder, true);
4467                         /* -ENOLINK means non-consecutive trace */
4468                         if (err && err != -ENOLINK)
4469                                 return err;
4470                 } while (decoder->buf_timestamp != d.buf_timestamp);
4471         }
4472
4473         if (!decoder->buf)
4474                 return 0;
4475
4476         buf = (unsigned char *)decoder->buf;
4477         len = decoder->len;
4478
4479         if (!intel_pt_next_psb(&buf, &len))
4480                 return 0;
4481
4482         /*
4483          * Walk PSBs while the PSB timestamp is less than the fast forward
4484          * timestamp.
4485          */
4486         do {
4487                 uint64_t tsc;
4488                 size_t rem;
4489
4490                 if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4491                         break;
4492                 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4493                 /*
4494                  * A TSC packet can slip past MTC packets but, after fast
4495                  * forward, decoding starts at the TSC timestamp. That means
4496                  * the timestamps may not be exactly the same as the timestamps
4497                  * that would have been decoded without fast forward.
4498                  */
4499                 if (tsc < timestamp) {
4500                         intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4501                         decoder->pos += decoder->len - len;
4502                         decoder->buf = buf;
4503                         decoder->len = len;
4504                         intel_pt_reposition(decoder);
4505                 } else {
4506                         break;
4507                 }
4508         } while (intel_pt_step_psb(&buf, &len));
4509
4510         return 0;
4511 }