tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29 #include "../auxtrace.h"
  30
  31 #include "intel-pt-insn-decoder.h"
  32 #include "intel-pt-pkt-decoder.h"
  33 #include "intel-pt-decoder.h"
  34 #include "intel-pt-log.h"
  35
  36 #define INTEL_PT_BLK_SIZE 1024
  37
  38 #define BIT63 (((uint64_t)1 << 63))
  39
  40 #define INTEL_PT_RETURN 1
  41
  42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  43 #define INTEL_PT_MAX_LOOPS 10000
  44
  45 struct intel_pt_blk {
  46         struct intel_pt_blk *prev;
  47         uint64_t ip[INTEL_PT_BLK_SIZE];
  48 };
  49
  50 struct intel_pt_stack {
  51         struct intel_pt_blk *blk;
  52         struct intel_pt_blk *spare;
  53         int pos;
  54 };
  55
  56 enum intel_pt_pkt_state {
  57         INTEL_PT_STATE_NO_PSB,
  58         INTEL_PT_STATE_NO_IP,
  59         INTEL_PT_STATE_ERR_RESYNC,
  60         INTEL_PT_STATE_IN_SYNC,
  61         INTEL_PT_STATE_TNT_CONT,
  62         INTEL_PT_STATE_TNT,
  63         INTEL_PT_STATE_TIP,
  64         INTEL_PT_STATE_TIP_PGD,
  65         INTEL_PT_STATE_FUP,
  66         INTEL_PT_STATE_FUP_NO_TIP,
  67 };
  68
  69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  70 {
  71         switch (pkt_state) {
  72         case INTEL_PT_STATE_NO_PSB:
  73         case INTEL_PT_STATE_NO_IP:
  74         case INTEL_PT_STATE_ERR_RESYNC:
  75         case INTEL_PT_STATE_IN_SYNC:
  76         case INTEL_PT_STATE_TNT_CONT:
  77                 return true;
  78         case INTEL_PT_STATE_TNT:
  79         case INTEL_PT_STATE_TIP:
  80         case INTEL_PT_STATE_TIP_PGD:
  81         case INTEL_PT_STATE_FUP:
  82         case INTEL_PT_STATE_FUP_NO_TIP:
  83                 return false;
  84         default:
  85                 return true;
  86         };
  87 }
  88
  89 #ifdef INTEL_PT_STRICT
  90 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  91 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  92 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  93 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  94 #else
  95 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  96 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  97 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  98 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  99 #endif
 100
 101 struct intel_pt_decoder {
 102         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 103         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 104                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 105                          uint64_t max_insn_cnt, void *data);
 106         bool (*pgd_ip)(uint64_t ip, void *data);
 107         void *data;
 108         struct intel_pt_state state;
 109         const unsigned char *buf;
 110         size_t len;
 111         bool return_compression;
 112         bool branch_enable;
 113         bool mtc_insn;
 114         bool pge;
 115         bool have_tma;
 116         bool have_cyc;
 117         bool fixup_last_mtc;
 118         bool have_last_ip;
 119         enum intel_pt_param_flags flags;
 120         uint64_t pos;
 121         uint64_t last_ip;
 122         uint64_t ip;
 123         uint64_t cr3;
 124         uint64_t timestamp;
 125         uint64_t tsc_timestamp;
 126         uint64_t ref_timestamp;
 127         uint64_t sample_timestamp;
 128         uint64_t ret_addr;
 129         uint64_t ctc_timestamp;
 130         uint64_t ctc_delta;
 131         uint64_t cycle_cnt;
 132         uint64_t cyc_ref_timestamp;
 133         uint32_t last_mtc;
 134         uint32_t tsc_ctc_ratio_n;
 135         uint32_t tsc_ctc_ratio_d;
 136         uint32_t tsc_ctc_mult;
 137         uint32_t tsc_slip;
 138         uint32_t ctc_rem_mask;
 139         int mtc_shift;
 140         struct intel_pt_stack stack;
 141         enum intel_pt_pkt_state pkt_state;
 142         struct intel_pt_pkt packet;
 143         struct intel_pt_pkt tnt;
 144         int pkt_step;
 145         int pkt_len;
 146         int last_packet_type;
 147         unsigned int cbr;
 148         unsigned int cbr_seen;
 149         unsigned int max_non_turbo_ratio;
 150         double max_non_turbo_ratio_fp;
 151         double cbr_cyc_to_tsc;
 152         double calc_cyc_to_tsc;
 153         bool have_calc_cyc_to_tsc;
 154         int exec_mode;
 155         unsigned int insn_bytes;
 156         uint64_t period;
 157         enum intel_pt_period_type period_type;
 158         uint64_t tot_insn_cnt;
 159         uint64_t period_insn_cnt;
 160         uint64_t period_mask;
 161         uint64_t period_ticks;
 162         uint64_t last_masked_timestamp;
 163         bool continuous_period;
 164         bool overflow;
 165         bool set_fup_tx_flags;
 166         bool set_fup_ptw;
 167         bool set_fup_mwait;
 168         bool set_fup_pwre;
 169         bool set_fup_exstop;
 170         unsigned int fup_tx_flags;
 171         unsigned int tx_flags;
 172         uint64_t fup_ptw_payload;
 173         uint64_t fup_mwait_payload;
 174         uint64_t fup_pwre_payload;
 175         uint64_t cbr_payload;
 176         uint64_t timestamp_insn_cnt;
 177         uint64_t sample_insn_cnt;
 178         uint64_t stuck_ip;
 179         int no_progress;
 180         int stuck_ip_prd;
 181         int stuck_ip_cnt;
 182         const unsigned char *next_buf;
 183         size_t next_len;
 184         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 185 };
 186
 187 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 188 {
 189         int i;
 190
 191         for (i = 0; x != 1; i++)
 192                 x >>= 1;
 193
 194         return x << i;
 195 }
 196
 197 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 198 {
 199         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 200                 uint64_t period;
 201
 202                 period = intel_pt_lower_power_of_2(decoder->period);
 203                 decoder->period_mask  = ~(period - 1);
 204                 decoder->period_ticks = period;
 205         }
 206 }
 207
 208 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 209 {
 210         if (!d)
 211                 return 0;
 212         return (t / d) * n + ((t % d) * n) / d;
 213 }
 214
 215 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 216 {
 217         struct intel_pt_decoder *decoder;
 218
 219         if (!params->get_trace || !params->walk_insn)
 220                 return NULL;
 221
 222         decoder = zalloc(sizeof(struct intel_pt_decoder));
 223         if (!decoder)
 224                 return NULL;
 225
 226         decoder->get_trace          = params->get_trace;
 227         decoder->walk_insn          = params->walk_insn;
 228         decoder->pgd_ip             = params->pgd_ip;
 229         decoder->data               = params->data;
 230         decoder->return_compression = params->return_compression;
 231         decoder->branch_enable      = params->branch_enable;
 232
 233         decoder->flags              = params->flags;
 234
 235         decoder->period             = params->period;
 236         decoder->period_type        = params->period_type;
 237
 238         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 239         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 240
 241         intel_pt_setup_period(decoder);
 242
 243         decoder->mtc_shift = params->mtc_period;
 244         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 245
 246         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 247         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 248
 249         if (!decoder->tsc_ctc_ratio_n)
 250                 decoder->tsc_ctc_ratio_d = 0;
 251
 252         if (decoder->tsc_ctc_ratio_d) {
 253                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 254                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 255                                                 decoder->tsc_ctc_ratio_d;
 256         }
 257
 258         /*
 259          * A TSC packet can slip past MTC packets so that the timestamp appears
 260          * to go backwards. One estimate is that can be up to about 40 CPU
 261          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 262          * slippage an order of magnitude more to be on the safe side.
 263          */
 264         decoder->tsc_slip = 0x10000;
 265
 266         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 267         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 268         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 269         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 270         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 271
 272         return decoder;
 273 }
 274
 275 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 276 {
 277         struct intel_pt_blk *blk = stack->blk;
 278
 279         stack->blk = blk->prev;
 280         if (!stack->spare)
 281                 stack->spare = blk;
 282         else
 283                 free(blk);
 284 }
 285
 286 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 287 {
 288         if (!stack->pos) {
 289                 if (!stack->blk)
 290                         return 0;
 291                 intel_pt_pop_blk(stack);
 292                 if (!stack->blk)
 293                         return 0;
 294                 stack->pos = INTEL_PT_BLK_SIZE;
 295         }
 296         return stack->blk->ip[--stack->pos];
 297 }
 298
 299 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 300 {
 301         struct intel_pt_blk *blk;
 302
 303         if (stack->spare) {
 304                 blk = stack->spare;
 305                 stack->spare = NULL;
 306         } else {
 307                 blk = malloc(sizeof(struct intel_pt_blk));
 308                 if (!blk)
 309                         return -ENOMEM;
 310         }
 311
 312         blk->prev = stack->blk;
 313         stack->blk = blk;
 314         stack->pos = 0;
 315         return 0;
 316 }
 317
 318 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 319 {
 320         int err;
 321
 322         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 323                 err = intel_pt_alloc_blk(stack);
 324                 if (err)
 325                         return err;
 326         }
 327
 328         stack->blk->ip[stack->pos++] = ip;
 329         return 0;
 330 }
 331
 332 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 333 {
 334         while (stack->blk)
 335                 intel_pt_pop_blk(stack);
 336         stack->pos = 0;
 337 }
 338
 339 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 340 {
 341         intel_pt_clear_stack(stack);
 342         zfree(&stack->blk);
 343         zfree(&stack->spare);
 344 }
 345
 346 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 347 {
 348         intel_pt_free_stack(&decoder->stack);
 349         free(decoder);
 350 }
 351
 352 static int intel_pt_ext_err(int code)
 353 {
 354         switch (code) {
 355         case -ENOMEM:
 356                 return INTEL_PT_ERR_NOMEM;
 357         case -ENOSYS:
 358                 return INTEL_PT_ERR_INTERN;
 359         case -EBADMSG:
 360                 return INTEL_PT_ERR_BADPKT;
 361         case -ENODATA:
 362                 return INTEL_PT_ERR_NODATA;
 363         case -EILSEQ:
 364                 return INTEL_PT_ERR_NOINSN;
 365         case -ENOENT:
 366                 return INTEL_PT_ERR_MISMAT;
 367         case -EOVERFLOW:
 368                 return INTEL_PT_ERR_OVR;
 369         case -ENOSPC:
 370                 return INTEL_PT_ERR_LOST;
 371         case -ELOOP:
 372                 return INTEL_PT_ERR_NELOOP;
 373         default:
 374                 return INTEL_PT_ERR_UNK;
 375         }
 376 }
 377
 378 static const char *intel_pt_err_msgs[] = {
 379         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 380         [INTEL_PT_ERR_INTERN] = "Internal error",
 381         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 382         [INTEL_PT_ERR_NODATA] = "No more data",
 383         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 384         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 385         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 386         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 387         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 388         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 389 };
 390
 391 int intel_pt__strerror(int code, char *buf, size_t buflen)
 392 {
 393         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 394                 code = INTEL_PT_ERR_UNK;
 395         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 396         return 0;
 397 }
 398
 399 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 400                                  uint64_t last_ip)
 401 {
 402         uint64_t ip;
 403
 404         switch (packet->count) {
 405         case 1:
 406                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 407                      packet->payload;
 408                 break;
 409         case 2:
 410                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 411                      packet->payload;
 412                 break;
 413         case 3:
 414                 ip = packet->payload;
 415                 /* Sign-extend 6-byte ip */
 416                 if (ip & (uint64_t)0x800000000000ULL)
 417                         ip |= (uint64_t)0xffff000000000000ULL;
 418                 break;
 419         case 4:
 420                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 421                      packet->payload;
 422                 break;
 423         case 6:
 424                 ip = packet->payload;
 425                 break;
 426         default:
 427                 return 0;
 428         }
 429
 430         return ip;
 431 }
 432
 433 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 434 {
 435         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 436         decoder->have_last_ip = true;
 437 }
 438
 439 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 440 {
 441         intel_pt_set_last_ip(decoder);
 442         decoder->ip = decoder->last_ip;
 443 }
 444
 445 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 446 {
 447         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 448                             decoder->buf);
 449 }
 450
 451 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 452 {
 453         intel_pt_log("ERROR: Internal error\n");
 454         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 455         return -ENOSYS;
 456 }
 457
 458 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 459 {
 460         decoder->tx_flags = 0;
 461 }
 462
 463 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 464 {
 465         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 466 }
 467
 468 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 469 {
 470         intel_pt_clear_tx_flags(decoder);
 471         decoder->have_tma = false;
 472         decoder->pkt_len = 1;
 473         decoder->pkt_step = 1;
 474         intel_pt_decoder_log_packet(decoder);
 475         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 476                 intel_pt_log("ERROR: Bad packet\n");
 477                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 478         }
 479         return -EBADMSG;
 480 }
 481
 482 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 483 {
 484         struct intel_pt_buffer buffer = { .buf = 0, };
 485         int ret;
 486
 487         decoder->pkt_step = 0;
 488
 489         intel_pt_log("Getting more data\n");
 490         ret = decoder->get_trace(&buffer, decoder->data);
 491         if (ret)
 492                 return ret;
 493         decoder->buf = buffer.buf;
 494         decoder->len = buffer.len;
 495         if (!decoder->len) {
 496                 intel_pt_log("No more data\n");
 497                 return -ENODATA;
 498         }
 499         if (!buffer.consecutive) {
 500                 decoder->ip = 0;
 501                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 502                 decoder->ref_timestamp = buffer.ref_timestamp;
 503                 decoder->timestamp = 0;
 504                 decoder->have_tma = false;
 505                 decoder->state.trace_nr = buffer.trace_nr;
 506                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 507                              decoder->ref_timestamp);
 508                 return -ENOLINK;
 509         }
 510
 511         return 0;
 512 }
 513
 514 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 515 {
 516         if (!decoder->next_buf)
 517                 return intel_pt_get_data(decoder);
 518
 519         decoder->buf = decoder->next_buf;
 520         decoder->len = decoder->next_len;
 521         decoder->next_buf = 0;
 522         decoder->next_len = 0;
 523         return 0;
 524 }
 525
 526 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 527 {
 528         unsigned char *buf = decoder->temp_buf;
 529         size_t old_len, len, n;
 530         int ret;
 531
 532         old_len = decoder->len;
 533         len = decoder->len;
 534         memcpy(buf, decoder->buf, len);
 535
 536         ret = intel_pt_get_data(decoder);
 537         if (ret) {
 538                 decoder->pos += old_len;
 539                 return ret < 0 ? ret : -EINVAL;
 540         }
 541
 542         n = INTEL_PT_PKT_MAX_SZ - len;
 543         if (n > decoder->len)
 544                 n = decoder->len;
 545         memcpy(buf + len, decoder->buf, n);
 546         len += n;
 547
 548         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 549         if (ret < (int)old_len) {
 550                 decoder->next_buf = decoder->buf;
 551                 decoder->next_len = decoder->len;
 552                 decoder->buf = buf;
 553                 decoder->len = old_len;
 554                 return intel_pt_bad_packet(decoder);
 555         }
 556
 557         decoder->next_buf = decoder->buf + (ret - old_len);
 558         decoder->next_len = decoder->len - (ret - old_len);
 559
 560         decoder->buf = buf;
 561         decoder->len = ret;
 562
 563         return ret;
 564 }
 565
 566 struct intel_pt_pkt_info {
 567         struct intel_pt_decoder   *decoder;
 568         struct intel_pt_pkt       packet;
 569         uint64_t                  pos;
 570         int                       pkt_len;
 571         int                       last_packet_type;
 572         void                      *data;
 573 };
 574
 575 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 576
 577 /* Lookahead packets in current buffer */
 578 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 579                                   intel_pt_pkt_cb_t cb, void *data)
 580 {
 581         struct intel_pt_pkt_info pkt_info;
 582         const unsigned char *buf = decoder->buf;
 583         size_t len = decoder->len;
 584         int ret;
 585
 586         pkt_info.decoder          = decoder;
 587         pkt_info.pos              = decoder->pos;
 588         pkt_info.pkt_len          = decoder->pkt_step;
 589         pkt_info.last_packet_type = decoder->last_packet_type;
 590         pkt_info.data             = data;
 591
 592         while (1) {
 593                 do {
 594                         pkt_info.pos += pkt_info.pkt_len;
 595                         buf          += pkt_info.pkt_len;
 596                         len          -= pkt_info.pkt_len;
 597
 598                         if (!len)
 599                                 return INTEL_PT_NEED_MORE_BYTES;
 600
 601                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 602                         if (!ret)
 603                                 return INTEL_PT_NEED_MORE_BYTES;
 604                         if (ret < 0)
 605                                 return ret;
 606
 607                         pkt_info.pkt_len = ret;
 608                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 609
 610                 ret = cb(&pkt_info);
 611                 if (ret)
 612                         return 0;
 613
 614                 pkt_info.last_packet_type = pkt_info.packet.type;
 615         }
 616 }
 617
 618 struct intel_pt_calc_cyc_to_tsc_info {
 619         uint64_t        cycle_cnt;
 620         unsigned int    cbr;
 621         uint32_t        last_mtc;
 622         uint64_t        ctc_timestamp;
 623         uint64_t        ctc_delta;
 624         uint64_t        tsc_timestamp;
 625         uint64_t        timestamp;
 626         bool            have_tma;
 627         bool            fixup_last_mtc;
 628         bool            from_mtc;
 629         double          cbr_cyc_to_tsc;
 630 };
 631
 632 /*
 633  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 634  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 635  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 636  * packet by copying the missing bits from the current MTC assuming the least
 637  * difference between the two, and that the current MTC comes after last_mtc.
 638  */
 639 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 640                                     uint32_t *last_mtc)
 641 {
 642         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 643         uint32_t mask = ~(first_missing_bit - 1);
 644
 645         *last_mtc |= mtc & mask;
 646         if (*last_mtc >= mtc) {
 647                 *last_mtc -= first_missing_bit;
 648                 *last_mtc &= 0xff;
 649         }
 650 }
 651
 652 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 653 {
 654         struct intel_pt_decoder *decoder = pkt_info->decoder;
 655         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 656         uint64_t timestamp;
 657         double cyc_to_tsc;
 658         unsigned int cbr;
 659         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 660
 661         switch (pkt_info->packet.type) {
 662         case INTEL_PT_TNT:
 663         case INTEL_PT_TIP_PGE:
 664         case INTEL_PT_TIP:
 665         case INTEL_PT_FUP:
 666         case INTEL_PT_PSB:
 667         case INTEL_PT_PIP:
 668         case INTEL_PT_MODE_EXEC:
 669         case INTEL_PT_MODE_TSX:
 670         case INTEL_PT_PSBEND:
 671         case INTEL_PT_PAD:
 672         case INTEL_PT_VMCS:
 673         case INTEL_PT_MNT:
 674         case INTEL_PT_PTWRITE:
 675         case INTEL_PT_PTWRITE_IP:
 676                 return 0;
 677
 678         case INTEL_PT_MTC:
 679                 if (!data->have_tma)
 680                         return 0;
 681
 682                 mtc = pkt_info->packet.payload;
 683                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 684                         data->fixup_last_mtc = false;
 685                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 686                                                 &data->last_mtc);
 687                 }
 688                 if (mtc > data->last_mtc)
 689                         mtc_delta = mtc - data->last_mtc;
 690                 else
 691                         mtc_delta = mtc + 256 - data->last_mtc;
 692                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 693                 data->last_mtc = mtc;
 694
 695                 if (decoder->tsc_ctc_mult) {
 696                         timestamp = data->ctc_timestamp +
 697                                 data->ctc_delta * decoder->tsc_ctc_mult;
 698                 } else {
 699                         timestamp = data->ctc_timestamp +
 700                                 multdiv(data->ctc_delta,
 701                                         decoder->tsc_ctc_ratio_n,
 702                                         decoder->tsc_ctc_ratio_d);
 703                 }
 704
 705                 if (timestamp < data->timestamp)
 706                         return 1;
 707
 708                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 709                         data->timestamp = timestamp;
 710                         return 0;
 711                 }
 712
 713                 break;
 714
 715         case INTEL_PT_TSC:
 716                 /*
 717                  * For now, do not support using TSC packets - refer
 718                  * intel_pt_calc_cyc_to_tsc().
 719                  */
 720                 if (data->from_mtc)
 721                         return 1;
 722                 timestamp = pkt_info->packet.payload |
 723                             (data->timestamp & (0xffULL << 56));
 724                 if (data->from_mtc && timestamp < data->timestamp &&
 725                     data->timestamp - timestamp < decoder->tsc_slip)
 726                         return 1;
 727                 if (timestamp < data->timestamp)
 728                         timestamp += (1ULL << 56);
 729                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 730                         if (data->from_mtc)
 731                                 return 1;
 732                         data->tsc_timestamp = timestamp;
 733                         data->timestamp = timestamp;
 734                         return 0;
 735                 }
 736                 break;
 737
 738         case INTEL_PT_TMA:
 739                 if (data->from_mtc)
 740                         return 1;
 741
 742                 if (!decoder->tsc_ctc_ratio_d)
 743                         return 0;
 744
 745                 ctc = pkt_info->packet.payload;
 746                 fc = pkt_info->packet.count;
 747                 ctc_rem = ctc & decoder->ctc_rem_mask;
 748
 749                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 750
 751                 data->ctc_timestamp = data->tsc_timestamp - fc;
 752                 if (decoder->tsc_ctc_mult) {
 753                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 754                 } else {
 755                         data->ctc_timestamp -=
 756                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 757                                         decoder->tsc_ctc_ratio_d);
 758                 }
 759
 760                 data->ctc_delta = 0;
 761                 data->have_tma = true;
 762                 data->fixup_last_mtc = true;
 763
 764                 return 0;
 765
 766         case INTEL_PT_CYC:
 767                 data->cycle_cnt += pkt_info->packet.payload;
 768                 return 0;
 769
 770         case INTEL_PT_CBR:
 771                 cbr = pkt_info->packet.payload;
 772                 if (data->cbr && data->cbr != cbr)
 773                         return 1;
 774                 data->cbr = cbr;
 775                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 776                 return 0;
 777
 778         case INTEL_PT_TIP_PGD:
 779         case INTEL_PT_TRACESTOP:
 780         case INTEL_PT_EXSTOP:
 781         case INTEL_PT_EXSTOP_IP:
 782         case INTEL_PT_MWAIT:
 783         case INTEL_PT_PWRE:
 784         case INTEL_PT_PWRX:
 785         case INTEL_PT_OVF:
 786         case INTEL_PT_BAD: /* Does not happen */
 787         default:
 788                 return 1;
 789         }
 790
 791         if (!data->cbr && decoder->cbr) {
 792                 data->cbr = decoder->cbr;
 793                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 794         }
 795
 796         if (!data->cycle_cnt)
 797                 return 1;
 798
 799         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 800
 801         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 802             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 803                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 804                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 805                 return 1;
 806         }
 807
 808         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 809         decoder->have_calc_cyc_to_tsc = true;
 810
 811         if (data->cbr) {
 812                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 813                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 814         } else {
 815                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 816                              cyc_to_tsc, pkt_info->pos);
 817         }
 818
 819         return 1;
 820 }
 821
 822 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 823                                      bool from_mtc)
 824 {
 825         struct intel_pt_calc_cyc_to_tsc_info data = {
 826                 .cycle_cnt      = 0,
 827                 .cbr            = 0,
 828                 .last_mtc       = decoder->last_mtc,
 829                 .ctc_timestamp  = decoder->ctc_timestamp,
 830                 .ctc_delta      = decoder->ctc_delta,
 831                 .tsc_timestamp  = decoder->tsc_timestamp,
 832                 .timestamp      = decoder->timestamp,
 833                 .have_tma       = decoder->have_tma,
 834                 .fixup_last_mtc = decoder->fixup_last_mtc,
 835                 .from_mtc       = from_mtc,
 836                 .cbr_cyc_to_tsc = 0,
 837         };
 838
 839         /*
 840          * For now, do not support using TSC packets for at least the reasons:
 841          * 1) timing might have stopped
 842          * 2) TSC packets within PSB+ can slip against CYC packets
 843          */
 844         if (!from_mtc)
 845                 return;
 846
 847         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 848 }
 849
 850 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 851 {
 852         int ret;
 853
 854         decoder->last_packet_type = decoder->packet.type;
 855
 856         do {
 857                 decoder->pos += decoder->pkt_step;
 858                 decoder->buf += decoder->pkt_step;
 859                 decoder->len -= decoder->pkt_step;
 860
 861                 if (!decoder->len) {
 862                         ret = intel_pt_get_next_data(decoder);
 863                         if (ret)
 864                                 return ret;
 865                 }
 866
 867                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 868                                           &decoder->packet);
 869                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 870                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 871                         ret = intel_pt_get_split_packet(decoder);
 872                         if (ret < 0)
 873                                 return ret;
 874                 }
 875                 if (ret <= 0)
 876                         return intel_pt_bad_packet(decoder);
 877
 878                 decoder->pkt_len = ret;
 879                 decoder->pkt_step = ret;
 880                 intel_pt_decoder_log_packet(decoder);
 881         } while (decoder->packet.type == INTEL_PT_PAD);
 882
 883         return 0;
 884 }
 885
 886 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 887 {
 888         uint64_t timestamp, masked_timestamp;
 889
 890         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 891         masked_timestamp = timestamp & decoder->period_mask;
 892         if (decoder->continuous_period) {
 893                 if (masked_timestamp > decoder->last_masked_timestamp)
 894                         return 1;
 895         } else {
 896                 timestamp += 1;
 897                 masked_timestamp = timestamp & decoder->period_mask;
 898                 if (masked_timestamp > decoder->last_masked_timestamp) {
 899                         decoder->last_masked_timestamp = masked_timestamp;
 900                         decoder->continuous_period = true;
 901                 }
 902         }
 903
 904         if (masked_timestamp < decoder->last_masked_timestamp)
 905                 return decoder->period_ticks;
 906
 907         return decoder->period_ticks - (timestamp - masked_timestamp);
 908 }
 909
 910 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 911 {
 912         switch (decoder->period_type) {
 913         case INTEL_PT_PERIOD_INSTRUCTIONS:
 914                 return decoder->period - decoder->period_insn_cnt;
 915         case INTEL_PT_PERIOD_TICKS:
 916                 return intel_pt_next_period(decoder);
 917         case INTEL_PT_PERIOD_NONE:
 918         case INTEL_PT_PERIOD_MTC:
 919         default:
 920                 return 0;
 921         }
 922 }
 923
 924 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 925 {
 926         uint64_t timestamp, masked_timestamp;
 927
 928         switch (decoder->period_type) {
 929         case INTEL_PT_PERIOD_INSTRUCTIONS:
 930                 decoder->period_insn_cnt = 0;
 931                 break;
 932         case INTEL_PT_PERIOD_TICKS:
 933                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 934                 masked_timestamp = timestamp & decoder->period_mask;
 935                 if (masked_timestamp > decoder->last_masked_timestamp)
 936                         decoder->last_masked_timestamp = masked_timestamp;
 937                 else
 938                         decoder->last_masked_timestamp += decoder->period_ticks;
 939                 break;
 940         case INTEL_PT_PERIOD_NONE:
 941         case INTEL_PT_PERIOD_MTC:
 942         default:
 943                 break;
 944         }
 945
 946         decoder->state.type |= INTEL_PT_INSTRUCTION;
 947 }
 948
 949 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 950                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 951 {
 952         uint64_t max_insn_cnt, insn_cnt = 0;
 953         int err;
 954
 955         if (!decoder->mtc_insn)
 956                 decoder->mtc_insn = true;
 957
 958         max_insn_cnt = intel_pt_next_sample(decoder);
 959
 960         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 961                                  max_insn_cnt, decoder->data);
 962
 963         decoder->tot_insn_cnt += insn_cnt;
 964         decoder->timestamp_insn_cnt += insn_cnt;
 965         decoder->sample_insn_cnt += insn_cnt;
 966         decoder->period_insn_cnt += insn_cnt;
 967
 968         if (err) {
 969                 decoder->no_progress = 0;
 970                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 971                 intel_pt_log_at("ERROR: Failed to get instruction",
 972                                 decoder->ip);
 973                 if (err == -ENOENT)
 974                         return -ENOLINK;
 975                 return -EILSEQ;
 976         }
 977
 978         if (ip && decoder->ip == ip) {
 979                 err = -EAGAIN;
 980                 goto out;
 981         }
 982
 983         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 984                 intel_pt_sample_insn(decoder);
 985
 986         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 987                 decoder->state.type = INTEL_PT_INSTRUCTION;
 988                 decoder->state.from_ip = decoder->ip;
 989                 decoder->state.to_ip = 0;
 990                 decoder->ip += intel_pt_insn->length;
 991                 err = INTEL_PT_RETURN;
 992                 goto out;
 993         }
 994
 995         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 996                 /* Zero-length calls are excluded */
 997                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 998                     intel_pt_insn->rel) {
 999                         err = intel_pt_push(&decoder->stack, decoder->ip +
1000                                             intel_pt_insn->length);
1001                         if (err)
1002                                 goto out;
1003                 }
1004         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1005                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1006         }
1007
1008         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1009                 int cnt = decoder->no_progress++;
1010
1011                 decoder->state.from_ip = decoder->ip;
1012                 decoder->ip += intel_pt_insn->length +
1013                                 intel_pt_insn->rel;
1014                 decoder->state.to_ip = decoder->ip;
1015                 err = INTEL_PT_RETURN;
1016
1017                 /*
1018                  * Check for being stuck in a loop.  This can happen if a
1019                  * decoder error results in the decoder erroneously setting the
1020                  * ip to an address that is itself in an infinite loop that
1021                  * consumes no packets.  When that happens, there must be an
1022                  * unconditional branch.
1023                  */
1024                 if (cnt) {
1025                         if (cnt == 1) {
1026                                 decoder->stuck_ip = decoder->state.to_ip;
1027                                 decoder->stuck_ip_prd = 1;
1028                                 decoder->stuck_ip_cnt = 1;
1029                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
1030                                    decoder->state.to_ip == decoder->stuck_ip) {
1031                                 intel_pt_log_at("ERROR: Never-ending loop",
1032                                                 decoder->state.to_ip);
1033                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1034                                 err = -ELOOP;
1035                                 goto out;
1036                         } else if (!--decoder->stuck_ip_cnt) {
1037                                 decoder->stuck_ip_prd += 1;
1038                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1039                                 decoder->stuck_ip = decoder->state.to_ip;
1040                         }
1041                 }
1042                 goto out_no_progress;
1043         }
1044 out:
1045         decoder->no_progress = 0;
1046 out_no_progress:
1047         decoder->state.insn_op = intel_pt_insn->op;
1048         decoder->state.insn_len = intel_pt_insn->length;
1049         memcpy(decoder->state.insn, intel_pt_insn->buf,
1050                INTEL_PT_INSN_BUF_SZ);
1051
1052         if (decoder->tx_flags & INTEL_PT_IN_TX)
1053                 decoder->state.flags |= INTEL_PT_IN_TX;
1054
1055         return err;
1056 }
1057
1058 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1059 {
1060         bool ret = false;
1061
1062         if (decoder->set_fup_tx_flags) {
1063                 decoder->set_fup_tx_flags = false;
1064                 decoder->tx_flags = decoder->fup_tx_flags;
1065                 decoder->state.type = INTEL_PT_TRANSACTION;
1066                 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1067                         decoder->state.type |= INTEL_PT_BRANCH;
1068                 decoder->state.from_ip = decoder->ip;
1069                 decoder->state.to_ip = 0;
1070                 decoder->state.flags = decoder->fup_tx_flags;
1071                 return true;
1072         }
1073         if (decoder->set_fup_ptw) {
1074                 decoder->set_fup_ptw = false;
1075                 decoder->state.type = INTEL_PT_PTW;
1076                 decoder->state.flags |= INTEL_PT_FUP_IP;
1077                 decoder->state.from_ip = decoder->ip;
1078                 decoder->state.to_ip = 0;
1079                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1080                 return true;
1081         }
1082         if (decoder->set_fup_mwait) {
1083                 decoder->set_fup_mwait = false;
1084                 decoder->state.type = INTEL_PT_MWAIT_OP;
1085                 decoder->state.from_ip = decoder->ip;
1086                 decoder->state.to_ip = 0;
1087                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1088                 ret = true;
1089         }
1090         if (decoder->set_fup_pwre) {
1091                 decoder->set_fup_pwre = false;
1092                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1093                 decoder->state.type &= ~INTEL_PT_BRANCH;
1094                 decoder->state.from_ip = decoder->ip;
1095                 decoder->state.to_ip = 0;
1096                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1097                 ret = true;
1098         }
1099         if (decoder->set_fup_exstop) {
1100                 decoder->set_fup_exstop = false;
1101                 decoder->state.type |= INTEL_PT_EX_STOP;
1102                 decoder->state.type &= ~INTEL_PT_BRANCH;
1103                 decoder->state.flags |= INTEL_PT_FUP_IP;
1104                 decoder->state.from_ip = decoder->ip;
1105                 decoder->state.to_ip = 0;
1106                 ret = true;
1107         }
1108         return ret;
1109 }
1110
1111 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1112                                           struct intel_pt_insn *intel_pt_insn,
1113                                           uint64_t ip, int err)
1114 {
1115         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1116                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1117                ip == decoder->ip + intel_pt_insn->length;
1118 }
1119
1120 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1121 {
1122         struct intel_pt_insn intel_pt_insn;
1123         uint64_t ip;
1124         int err;
1125
1126         ip = decoder->last_ip;
1127
1128         while (1) {
1129                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1130                 if (err == INTEL_PT_RETURN)
1131                         return 0;
1132                 if (err == -EAGAIN ||
1133                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1134                         bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1135
1136                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1137                         if (intel_pt_fup_event(decoder) && no_tip)
1138                                 return 0;
1139                         return -EAGAIN;
1140                 }
1141                 decoder->set_fup_tx_flags = false;
1142                 if (err)
1143                         return err;
1144
1145                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1146                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1147                                         decoder->ip);
1148                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1149                         return -ENOENT;
1150                 }
1151
1152                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1153                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1154                                         decoder->ip);
1155                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1156                         return -ENOENT;
1157                 }
1158
1159                 intel_pt_bug(decoder);
1160         }
1161 }
1162
1163 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1164 {
1165         struct intel_pt_insn intel_pt_insn;
1166         int err;
1167
1168         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1169         if (err == INTEL_PT_RETURN &&
1170             decoder->pgd_ip &&
1171             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1172             (decoder->state.type & INTEL_PT_BRANCH) &&
1173             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1174                 /* Unconditional branch leaving filter region */
1175                 decoder->no_progress = 0;
1176                 decoder->pge = false;
1177                 decoder->continuous_period = false;
1178                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1179                 decoder->state.to_ip = 0;
1180                 return 0;
1181         }
1182         if (err == INTEL_PT_RETURN)
1183                 return 0;
1184         if (err)
1185                 return err;
1186
1187         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1188                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1189                         decoder->pge = false;
1190                         decoder->continuous_period = false;
1191                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1192                         decoder->state.from_ip = decoder->ip;
1193                         decoder->state.to_ip = 0;
1194                         if (decoder->packet.count != 0)
1195                                 decoder->ip = decoder->last_ip;
1196                 } else {
1197                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1198                         decoder->state.from_ip = decoder->ip;
1199                         if (decoder->packet.count == 0) {
1200                                 decoder->state.to_ip = 0;
1201                         } else {
1202                                 decoder->state.to_ip = decoder->last_ip;
1203                                 decoder->ip = decoder->last_ip;
1204                         }
1205                 }
1206                 return 0;
1207         }
1208
1209         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1210                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1211                                  intel_pt_insn.rel;
1212
1213                 if (decoder->pgd_ip &&
1214                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1215                     decoder->pgd_ip(to_ip, decoder->data)) {
1216                         /* Conditional branch leaving filter region */
1217                         decoder->pge = false;
1218                         decoder->continuous_period = false;
1219                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1220                         decoder->ip = to_ip;
1221                         decoder->state.from_ip = decoder->ip;
1222                         decoder->state.to_ip = 0;
1223                         return 0;
1224                 }
1225                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1226                                 decoder->ip);
1227                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1228                 return -ENOENT;
1229         }
1230
1231         return intel_pt_bug(decoder);
1232 }
1233
1234 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1235 {
1236         struct intel_pt_insn intel_pt_insn;
1237         int err;
1238
1239         while (1) {
1240                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1241                 if (err == INTEL_PT_RETURN)
1242                         return 0;
1243                 if (err)
1244                         return err;
1245
1246                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1247                         if (!decoder->return_compression) {
1248                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1249                                                 decoder->ip);
1250                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1251                                 return -ENOENT;
1252                         }
1253                         if (!decoder->ret_addr) {
1254                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1255                                                 decoder->ip);
1256                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1257                                 return -ENOENT;
1258                         }
1259                         if (!(decoder->tnt.payload & BIT63)) {
1260                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1261                                                 decoder->ip);
1262                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1263                                 return -ENOENT;
1264                         }
1265                         decoder->tnt.count -= 1;
1266                         if (decoder->tnt.count)
1267                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1268                         else
1269                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1270                         decoder->tnt.payload <<= 1;
1271                         decoder->state.from_ip = decoder->ip;
1272                         decoder->ip = decoder->ret_addr;
1273                         decoder->state.to_ip = decoder->ip;
1274                         return 0;
1275                 }
1276
1277                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1278                         /* Handle deferred TIPs */
1279                         err = intel_pt_get_next_packet(decoder);
1280                         if (err)
1281                                 return err;
1282                         if (decoder->packet.type != INTEL_PT_TIP ||
1283                             decoder->packet.count == 0) {
1284                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1285                                                 decoder->ip);
1286                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1287                                 decoder->pkt_step = 0;
1288                                 return -ENOENT;
1289                         }
1290                         intel_pt_set_last_ip(decoder);
1291                         decoder->state.from_ip = decoder->ip;
1292                         decoder->state.to_ip = decoder->last_ip;
1293                         decoder->ip = decoder->last_ip;
1294                         return 0;
1295                 }
1296
1297                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1298                         decoder->tnt.count -= 1;
1299                         if (decoder->tnt.count)
1300                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1301                         else
1302                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1303                         if (decoder->tnt.payload & BIT63) {
1304                                 decoder->tnt.payload <<= 1;
1305                                 decoder->state.from_ip = decoder->ip;
1306                                 decoder->ip += intel_pt_insn.length +
1307                                                intel_pt_insn.rel;
1308                                 decoder->state.to_ip = decoder->ip;
1309                                 return 0;
1310                         }
1311                         /* Instruction sample for a non-taken branch */
1312                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1313                                 decoder->tnt.payload <<= 1;
1314                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1315                                 decoder->state.from_ip = decoder->ip;
1316                                 decoder->state.to_ip = 0;
1317                                 decoder->ip += intel_pt_insn.length;
1318                                 return 0;
1319                         }
1320                         decoder->ip += intel_pt_insn.length;
1321                         if (!decoder->tnt.count) {
1322                                 decoder->sample_timestamp = decoder->timestamp;
1323                                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
1324                                 return -EAGAIN;
1325                         }
1326                         decoder->tnt.payload <<= 1;
1327                         continue;
1328                 }
1329
1330                 return intel_pt_bug(decoder);
1331         }
1332 }
1333
1334 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1335 {
1336         unsigned int fup_tx_flags;
1337         int err;
1338
1339         fup_tx_flags = decoder->packet.payload &
1340                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1341         err = intel_pt_get_next_packet(decoder);
1342         if (err)
1343                 return err;
1344         if (decoder->packet.type == INTEL_PT_FUP) {
1345                 decoder->fup_tx_flags = fup_tx_flags;
1346                 decoder->set_fup_tx_flags = true;
1347                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1348                         *no_tip = true;
1349         } else {
1350                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1351                                 decoder->pos);
1352                 intel_pt_update_in_tx(decoder);
1353         }
1354         return 0;
1355 }
1356
1357 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1358 {
1359         uint64_t timestamp;
1360
1361         decoder->have_tma = false;
1362
1363         if (decoder->ref_timestamp) {
1364                 timestamp = decoder->packet.payload |
1365                             (decoder->ref_timestamp & (0xffULL << 56));
1366                 if (timestamp < decoder->ref_timestamp) {
1367                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1368                                 timestamp += (1ULL << 56);
1369                 } else {
1370                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1371                                 timestamp -= (1ULL << 56);
1372                 }
1373                 decoder->tsc_timestamp = timestamp;
1374                 decoder->timestamp = timestamp;
1375                 decoder->ref_timestamp = 0;
1376                 decoder->timestamp_insn_cnt = 0;
1377         } else if (decoder->timestamp) {
1378                 timestamp = decoder->packet.payload |
1379                             (decoder->timestamp & (0xffULL << 56));
1380                 decoder->tsc_timestamp = timestamp;
1381                 if (timestamp < decoder->timestamp &&
1382                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1383                         intel_pt_log_to("Suppressing backwards timestamp",
1384                                         timestamp);
1385                         timestamp = decoder->timestamp;
1386                 }
1387                 if (timestamp < decoder->timestamp) {
1388                         intel_pt_log_to("Wraparound timestamp", timestamp);
1389                         timestamp += (1ULL << 56);
1390                         decoder->tsc_timestamp = timestamp;
1391                 }
1392                 decoder->timestamp = timestamp;
1393                 decoder->timestamp_insn_cnt = 0;
1394         }
1395
1396         if (decoder->last_packet_type == INTEL_PT_CYC) {
1397                 decoder->cyc_ref_timestamp = decoder->timestamp;
1398                 decoder->cycle_cnt = 0;
1399                 decoder->have_calc_cyc_to_tsc = false;
1400                 intel_pt_calc_cyc_to_tsc(decoder, false);
1401         }
1402
1403         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1404 }
1405
1406 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1407 {
1408         intel_pt_log("ERROR: Buffer overflow\n");
1409         intel_pt_clear_tx_flags(decoder);
1410         decoder->timestamp_insn_cnt = 0;
1411         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1412         decoder->overflow = true;
1413         return -EOVERFLOW;
1414 }
1415
1416 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1417 {
1418         uint32_t ctc = decoder->packet.payload;
1419         uint32_t fc = decoder->packet.count;
1420         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1421
1422         if (!decoder->tsc_ctc_ratio_d)
1423                 return;
1424
1425         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1426         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1427         if (decoder->tsc_ctc_mult) {
1428                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1429         } else {
1430                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1431                                                   decoder->tsc_ctc_ratio_n,
1432                                                   decoder->tsc_ctc_ratio_d);
1433         }
1434         decoder->ctc_delta = 0;
1435         decoder->have_tma = true;
1436         decoder->fixup_last_mtc = true;
1437         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1438                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1439 }
1440
1441 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1442 {
1443         uint64_t timestamp;
1444         uint32_t mtc, mtc_delta;
1445
1446         if (!decoder->have_tma)
1447                 return;
1448
1449         mtc = decoder->packet.payload;
1450
1451         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1452                 decoder->fixup_last_mtc = false;
1453                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1454                                         &decoder->last_mtc);
1455         }
1456
1457         if (mtc > decoder->last_mtc)
1458                 mtc_delta = mtc - decoder->last_mtc;
1459         else
1460                 mtc_delta = mtc + 256 - decoder->last_mtc;
1461
1462         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1463
1464         if (decoder->tsc_ctc_mult) {
1465                 timestamp = decoder->ctc_timestamp +
1466                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1467         } else {
1468                 timestamp = decoder->ctc_timestamp +
1469                             multdiv(decoder->ctc_delta,
1470                                     decoder->tsc_ctc_ratio_n,
1471                                     decoder->tsc_ctc_ratio_d);
1472         }
1473
1474         if (timestamp < decoder->timestamp)
1475                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1476                              timestamp, decoder->timestamp);
1477         else
1478                 decoder->timestamp = timestamp;
1479
1480         decoder->timestamp_insn_cnt = 0;
1481         decoder->last_mtc = mtc;
1482
1483         if (decoder->last_packet_type == INTEL_PT_CYC) {
1484                 decoder->cyc_ref_timestamp = decoder->timestamp;
1485                 decoder->cycle_cnt = 0;
1486                 decoder->have_calc_cyc_to_tsc = false;
1487                 intel_pt_calc_cyc_to_tsc(decoder, true);
1488         }
1489 }
1490
1491 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1492 {
1493         unsigned int cbr = decoder->packet.payload & 0xff;
1494
1495         decoder->cbr_payload = decoder->packet.payload;
1496
1497         if (decoder->cbr == cbr)
1498                 return;
1499
1500         decoder->cbr = cbr;
1501         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1502 }
1503
1504 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1505 {
1506         uint64_t timestamp = decoder->cyc_ref_timestamp;
1507
1508         decoder->have_cyc = true;
1509
1510         decoder->cycle_cnt += decoder->packet.payload;
1511
1512         if (!decoder->cyc_ref_timestamp)
1513                 return;
1514
1515         if (decoder->have_calc_cyc_to_tsc)
1516                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1517         else if (decoder->cbr)
1518                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1519         else
1520                 return;
1521
1522         if (timestamp < decoder->timestamp)
1523                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1524                              timestamp, decoder->timestamp);
1525         else
1526                 decoder->timestamp = timestamp;
1527
1528         decoder->timestamp_insn_cnt = 0;
1529 }
1530
1531 /* Walk PSB+ packets when already in sync. */
1532 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1533 {
1534         int err;
1535
1536         while (1) {
1537                 err = intel_pt_get_next_packet(decoder);
1538                 if (err)
1539                         return err;
1540
1541                 switch (decoder->packet.type) {
1542                 case INTEL_PT_PSBEND:
1543                         return 0;
1544
1545                 case INTEL_PT_TIP_PGD:
1546                 case INTEL_PT_TIP_PGE:
1547                 case INTEL_PT_TIP:
1548                 case INTEL_PT_TNT:
1549                 case INTEL_PT_TRACESTOP:
1550                 case INTEL_PT_BAD:
1551                 case INTEL_PT_PSB:
1552                 case INTEL_PT_PTWRITE:
1553                 case INTEL_PT_PTWRITE_IP:
1554                 case INTEL_PT_EXSTOP:
1555                 case INTEL_PT_EXSTOP_IP:
1556                 case INTEL_PT_MWAIT:
1557                 case INTEL_PT_PWRE:
1558                 case INTEL_PT_PWRX:
1559                         decoder->have_tma = false;
1560                         intel_pt_log("ERROR: Unexpected packet\n");
1561                         return -EAGAIN;
1562
1563                 case INTEL_PT_OVF:
1564                         return intel_pt_overflow(decoder);
1565
1566                 case INTEL_PT_TSC:
1567                         intel_pt_calc_tsc_timestamp(decoder);
1568                         break;
1569
1570                 case INTEL_PT_TMA:
1571                         intel_pt_calc_tma(decoder);
1572                         break;
1573
1574                 case INTEL_PT_CBR:
1575                         intel_pt_calc_cbr(decoder);
1576                         break;
1577
1578                 case INTEL_PT_MODE_EXEC:
1579                         decoder->exec_mode = decoder->packet.payload;
1580                         break;
1581
1582                 case INTEL_PT_PIP:
1583                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1584                         break;
1585
1586                 case INTEL_PT_FUP:
1587                         decoder->pge = true;
1588                         if (decoder->packet.count)
1589                                 intel_pt_set_last_ip(decoder);
1590                         break;
1591
1592                 case INTEL_PT_MODE_TSX:
1593                         intel_pt_update_in_tx(decoder);
1594                         break;
1595
1596                 case INTEL_PT_MTC:
1597                         intel_pt_calc_mtc_timestamp(decoder);
1598                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1599                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1600                         break;
1601
1602                 case INTEL_PT_CYC:
1603                         intel_pt_calc_cyc_timestamp(decoder);
1604                         break;
1605
1606                 case INTEL_PT_VMCS:
1607                 case INTEL_PT_MNT:
1608                 case INTEL_PT_PAD:
1609                 default:
1610                         break;
1611                 }
1612         }
1613 }
1614
1615 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1616 {
1617         int err;
1618
1619         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1620                 decoder->tx_flags = 0;
1621                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1622                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1623         } else {
1624                 decoder->state.flags |= INTEL_PT_ASYNC;
1625         }
1626
1627         while (1) {
1628                 err = intel_pt_get_next_packet(decoder);
1629                 if (err)
1630                         return err;
1631
1632                 switch (decoder->packet.type) {
1633                 case INTEL_PT_TNT:
1634                 case INTEL_PT_FUP:
1635                 case INTEL_PT_TRACESTOP:
1636                 case INTEL_PT_PSB:
1637                 case INTEL_PT_TSC:
1638                 case INTEL_PT_TMA:
1639                 case INTEL_PT_MODE_TSX:
1640                 case INTEL_PT_BAD:
1641                 case INTEL_PT_PSBEND:
1642                 case INTEL_PT_PTWRITE:
1643                 case INTEL_PT_PTWRITE_IP:
1644                 case INTEL_PT_EXSTOP:
1645                 case INTEL_PT_EXSTOP_IP:
1646                 case INTEL_PT_MWAIT:
1647                 case INTEL_PT_PWRE:
1648                 case INTEL_PT_PWRX:
1649                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1650                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1651                         decoder->pkt_step = 0;
1652                         return -ENOENT;
1653
1654                 case INTEL_PT_CBR:
1655                         intel_pt_calc_cbr(decoder);
1656                         break;
1657
1658                 case INTEL_PT_OVF:
1659                         return intel_pt_overflow(decoder);
1660
1661                 case INTEL_PT_TIP_PGD:
1662                         decoder->state.from_ip = decoder->ip;
1663                         decoder->state.to_ip = 0;
1664                         if (decoder->packet.count != 0) {
1665                                 intel_pt_set_ip(decoder);
1666                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1667                                              decoder->ip);
1668                         }
1669                         decoder->pge = false;
1670                         decoder->continuous_period = false;
1671                         return 0;
1672
1673                 case INTEL_PT_TIP_PGE:
1674                         decoder->pge = true;
1675                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1676                                      decoder->ip);
1677                         decoder->state.from_ip = 0;
1678                         if (decoder->packet.count == 0) {
1679                                 decoder->state.to_ip = 0;
1680                         } else {
1681                                 intel_pt_set_ip(decoder);
1682                                 decoder->state.to_ip = decoder->ip;
1683                         }
1684                         return 0;
1685
1686                 case INTEL_PT_TIP:
1687                         decoder->state.from_ip = decoder->ip;
1688                         if (decoder->packet.count == 0) {
1689                                 decoder->state.to_ip = 0;
1690                         } else {
1691                                 intel_pt_set_ip(decoder);
1692                                 decoder->state.to_ip = decoder->ip;
1693                         }
1694                         return 0;
1695
1696                 case INTEL_PT_PIP:
1697                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1698                         break;
1699
1700                 case INTEL_PT_MTC:
1701                         intel_pt_calc_mtc_timestamp(decoder);
1702                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1703                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1704                         break;
1705
1706                 case INTEL_PT_CYC:
1707                         intel_pt_calc_cyc_timestamp(decoder);
1708                         break;
1709
1710                 case INTEL_PT_MODE_EXEC:
1711                         decoder->exec_mode = decoder->packet.payload;
1712                         break;
1713
1714                 case INTEL_PT_VMCS:
1715                 case INTEL_PT_MNT:
1716                 case INTEL_PT_PAD:
1717                         break;
1718
1719                 default:
1720                         return intel_pt_bug(decoder);
1721                 }
1722         }
1723 }
1724
1725 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1726 {
1727         bool no_tip = false;
1728         int err;
1729
1730         while (1) {
1731                 err = intel_pt_get_next_packet(decoder);
1732                 if (err)
1733                         return err;
1734 next:
1735                 switch (decoder->packet.type) {
1736                 case INTEL_PT_TNT:
1737                         if (!decoder->packet.count)
1738                                 break;
1739                         decoder->tnt = decoder->packet;
1740                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1741                         err = intel_pt_walk_tnt(decoder);
1742                         if (err == -EAGAIN)
1743                                 break;
1744                         return err;
1745
1746                 case INTEL_PT_TIP_PGD:
1747                         if (decoder->packet.count != 0)
1748                                 intel_pt_set_last_ip(decoder);
1749                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1750                         return intel_pt_walk_tip(decoder);
1751
1752                 case INTEL_PT_TIP_PGE: {
1753                         decoder->pge = true;
1754                         if (decoder->packet.count == 0) {
1755                                 intel_pt_log_at("Skipping zero TIP.PGE",
1756                                                 decoder->pos);
1757                                 break;
1758                         }
1759                         intel_pt_set_ip(decoder);
1760                         decoder->state.from_ip = 0;
1761                         decoder->state.to_ip = decoder->ip;
1762                         return 0;
1763                 }
1764
1765                 case INTEL_PT_OVF:
1766                         return intel_pt_overflow(decoder);
1767
1768                 case INTEL_PT_TIP:
1769                         if (decoder->packet.count != 0)
1770                                 intel_pt_set_last_ip(decoder);
1771                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1772                         return intel_pt_walk_tip(decoder);
1773
1774                 case INTEL_PT_FUP:
1775                         if (decoder->packet.count == 0) {
1776                                 intel_pt_log_at("Skipping zero FUP",
1777                                                 decoder->pos);
1778                                 no_tip = false;
1779                                 break;
1780                         }
1781                         intel_pt_set_last_ip(decoder);
1782                         if (!decoder->branch_enable) {
1783                                 decoder->ip = decoder->last_ip;
1784                                 if (intel_pt_fup_event(decoder))
1785                                         return 0;
1786                                 no_tip = false;
1787                                 break;
1788                         }
1789                         if (decoder->set_fup_mwait)
1790                                 no_tip = true;
1791                         if (no_tip)
1792                                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
1793                         else
1794                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
1795                         err = intel_pt_walk_fup(decoder);
1796                         if (err != -EAGAIN)
1797                                 return err;
1798                         if (no_tip) {
1799                                 no_tip = false;
1800                                 break;
1801                         }
1802                         return intel_pt_walk_fup_tip(decoder);
1803
1804                 case INTEL_PT_TRACESTOP:
1805                         decoder->pge = false;
1806                         decoder->continuous_period = false;
1807                         intel_pt_clear_tx_flags(decoder);
1808                         decoder->have_tma = false;
1809                         break;
1810
1811                 case INTEL_PT_PSB:
1812                         decoder->last_ip = 0;
1813                         decoder->have_last_ip = true;
1814                         intel_pt_clear_stack(&decoder->stack);
1815                         err = intel_pt_walk_psbend(decoder);
1816                         if (err == -EAGAIN)
1817                                 goto next;
1818                         if (err)
1819                                 return err;
1820                         break;
1821
1822                 case INTEL_PT_PIP:
1823                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1824                         break;
1825
1826                 case INTEL_PT_MTC:
1827                         intel_pt_calc_mtc_timestamp(decoder);
1828                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1829                                 break;
1830                         /*
1831                          * Ensure that there has been an instruction since the
1832                          * last MTC.
1833                          */
1834                         if (!decoder->mtc_insn)
1835                                 break;
1836                         decoder->mtc_insn = false;
1837                         /* Ensure that there is a timestamp */
1838                         if (!decoder->timestamp)
1839                                 break;
1840                         decoder->state.type = INTEL_PT_INSTRUCTION;
1841                         decoder->state.from_ip = decoder->ip;
1842                         decoder->state.to_ip = 0;
1843                         decoder->mtc_insn = false;
1844                         return 0;
1845
1846                 case INTEL_PT_TSC:
1847                         intel_pt_calc_tsc_timestamp(decoder);
1848                         break;
1849
1850                 case INTEL_PT_TMA:
1851                         intel_pt_calc_tma(decoder);
1852                         break;
1853
1854                 case INTEL_PT_CYC:
1855                         intel_pt_calc_cyc_timestamp(decoder);
1856                         break;
1857
1858                 case INTEL_PT_CBR:
1859                         intel_pt_calc_cbr(decoder);
1860                         if (!decoder->branch_enable &&
1861                             decoder->cbr != decoder->cbr_seen) {
1862                                 decoder->cbr_seen = decoder->cbr;
1863                                 decoder->state.type = INTEL_PT_CBR_CHG;
1864                                 decoder->state.from_ip = decoder->ip;
1865                                 decoder->state.to_ip = 0;
1866                                 decoder->state.cbr_payload =
1867                                                         decoder->packet.payload;
1868                                 return 0;
1869                         }
1870                         break;
1871
1872                 case INTEL_PT_MODE_EXEC:
1873                         decoder->exec_mode = decoder->packet.payload;
1874                         break;
1875
1876                 case INTEL_PT_MODE_TSX:
1877                         /* MODE_TSX need not be followed by FUP */
1878                         if (!decoder->pge) {
1879                                 intel_pt_update_in_tx(decoder);
1880                                 break;
1881                         }
1882                         err = intel_pt_mode_tsx(decoder, &no_tip);
1883                         if (err)
1884                                 return err;
1885                         goto next;
1886
1887                 case INTEL_PT_BAD: /* Does not happen */
1888                         return intel_pt_bug(decoder);
1889
1890                 case INTEL_PT_PSBEND:
1891                 case INTEL_PT_VMCS:
1892                 case INTEL_PT_MNT:
1893                 case INTEL_PT_PAD:
1894                         break;
1895
1896                 case INTEL_PT_PTWRITE_IP:
1897                         decoder->fup_ptw_payload = decoder->packet.payload;
1898                         err = intel_pt_get_next_packet(decoder);
1899                         if (err)
1900                                 return err;
1901                         if (decoder->packet.type == INTEL_PT_FUP) {
1902                                 decoder->set_fup_ptw = true;
1903                                 no_tip = true;
1904                         } else {
1905                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1906                                                 decoder->pos);
1907                         }
1908                         goto next;
1909
1910                 case INTEL_PT_PTWRITE:
1911                         decoder->state.type = INTEL_PT_PTW;
1912                         decoder->state.from_ip = decoder->ip;
1913                         decoder->state.to_ip = 0;
1914                         decoder->state.ptw_payload = decoder->packet.payload;
1915                         return 0;
1916
1917                 case INTEL_PT_MWAIT:
1918                         decoder->fup_mwait_payload = decoder->packet.payload;
1919                         decoder->set_fup_mwait = true;
1920                         break;
1921
1922                 case INTEL_PT_PWRE:
1923                         if (decoder->set_fup_mwait) {
1924                                 decoder->fup_pwre_payload =
1925                                                         decoder->packet.payload;
1926                                 decoder->set_fup_pwre = true;
1927                                 break;
1928                         }
1929                         decoder->state.type = INTEL_PT_PWR_ENTRY;
1930                         decoder->state.from_ip = decoder->ip;
1931                         decoder->state.to_ip = 0;
1932                         decoder->state.pwrx_payload = decoder->packet.payload;
1933                         return 0;
1934
1935                 case INTEL_PT_EXSTOP_IP:
1936                         err = intel_pt_get_next_packet(decoder);
1937                         if (err)
1938                                 return err;
1939                         if (decoder->packet.type == INTEL_PT_FUP) {
1940                                 decoder->set_fup_exstop = true;
1941                                 no_tip = true;
1942                         } else {
1943                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1944                                                 decoder->pos);
1945                         }
1946                         goto next;
1947
1948                 case INTEL_PT_EXSTOP:
1949                         decoder->state.type = INTEL_PT_EX_STOP;
1950                         decoder->state.from_ip = decoder->ip;
1951                         decoder->state.to_ip = 0;
1952                         return 0;
1953
1954                 case INTEL_PT_PWRX:
1955                         decoder->state.type = INTEL_PT_PWR_EXIT;
1956                         decoder->state.from_ip = decoder->ip;
1957                         decoder->state.to_ip = 0;
1958                         decoder->state.pwrx_payload = decoder->packet.payload;
1959                         return 0;
1960
1961                 default:
1962                         return intel_pt_bug(decoder);
1963                 }
1964         }
1965 }
1966
1967 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1968 {
1969         return decoder->packet.count &&
1970                (decoder->have_last_ip || decoder->packet.count == 3 ||
1971                 decoder->packet.count == 6);
1972 }
1973
1974 /* Walk PSB+ packets to get in sync. */
1975 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1976 {
1977         int err;
1978
1979         while (1) {
1980                 err = intel_pt_get_next_packet(decoder);
1981                 if (err)
1982                         return err;
1983
1984                 switch (decoder->packet.type) {
1985                 case INTEL_PT_TIP_PGD:
1986                         decoder->continuous_period = false;
1987                         __fallthrough;
1988                 case INTEL_PT_TIP_PGE:
1989                 case INTEL_PT_TIP:
1990                 case INTEL_PT_PTWRITE:
1991                 case INTEL_PT_PTWRITE_IP:
1992                 case INTEL_PT_EXSTOP:
1993                 case INTEL_PT_EXSTOP_IP:
1994                 case INTEL_PT_MWAIT:
1995                 case INTEL_PT_PWRE:
1996                 case INTEL_PT_PWRX:
1997                         intel_pt_log("ERROR: Unexpected packet\n");
1998                         return -ENOENT;
1999
2000                 case INTEL_PT_FUP:
2001                         decoder->pge = true;
2002                         if (intel_pt_have_ip(decoder)) {
2003                                 uint64_t current_ip = decoder->ip;
2004
2005                                 intel_pt_set_ip(decoder);
2006                                 if (current_ip)
2007                                         intel_pt_log_to("Setting IP",
2008                                                         decoder->ip);
2009                         }
2010                         break;
2011
2012                 case INTEL_PT_MTC:
2013                         intel_pt_calc_mtc_timestamp(decoder);
2014                         break;
2015
2016                 case INTEL_PT_TSC:
2017                         intel_pt_calc_tsc_timestamp(decoder);
2018                         break;
2019
2020                 case INTEL_PT_TMA:
2021                         intel_pt_calc_tma(decoder);
2022                         break;
2023
2024                 case INTEL_PT_CYC:
2025                         intel_pt_calc_cyc_timestamp(decoder);
2026                         break;
2027
2028                 case INTEL_PT_CBR:
2029                         intel_pt_calc_cbr(decoder);
2030                         break;
2031
2032                 case INTEL_PT_PIP:
2033                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2034                         break;
2035
2036                 case INTEL_PT_MODE_EXEC:
2037                         decoder->exec_mode = decoder->packet.payload;
2038                         break;
2039
2040                 case INTEL_PT_MODE_TSX:
2041                         intel_pt_update_in_tx(decoder);
2042                         break;
2043
2044                 case INTEL_PT_TRACESTOP:
2045                         decoder->pge = false;
2046                         decoder->continuous_period = false;
2047                         intel_pt_clear_tx_flags(decoder);
2048                         __fallthrough;
2049
2050                 case INTEL_PT_TNT:
2051                         decoder->have_tma = false;
2052                         intel_pt_log("ERROR: Unexpected packet\n");
2053                         if (decoder->ip)
2054                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2055                         else
2056                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2057                         return -ENOENT;
2058
2059                 case INTEL_PT_BAD: /* Does not happen */
2060                         return intel_pt_bug(decoder);
2061
2062                 case INTEL_PT_OVF:
2063                         return intel_pt_overflow(decoder);
2064
2065                 case INTEL_PT_PSBEND:
2066                         return 0;
2067
2068                 case INTEL_PT_PSB:
2069                 case INTEL_PT_VMCS:
2070                 case INTEL_PT_MNT:
2071                 case INTEL_PT_PAD:
2072                 default:
2073                         break;
2074                 }
2075         }
2076 }
2077
2078 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2079 {
2080         int err;
2081
2082         while (1) {
2083                 err = intel_pt_get_next_packet(decoder);
2084                 if (err)
2085                         return err;
2086
2087                 switch (decoder->packet.type) {
2088                 case INTEL_PT_TIP_PGD:
2089                         decoder->continuous_period = false;
2090                         __fallthrough;
2091                 case INTEL_PT_TIP_PGE:
2092                 case INTEL_PT_TIP:
2093                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
2094                         if (intel_pt_have_ip(decoder))
2095                                 intel_pt_set_ip(decoder);
2096                         if (decoder->ip)
2097                                 return 0;
2098                         break;
2099
2100                 case INTEL_PT_FUP:
2101                         if (intel_pt_have_ip(decoder))
2102                                 intel_pt_set_ip(decoder);
2103                         if (decoder->ip)
2104                                 return 0;
2105                         break;
2106
2107                 case INTEL_PT_MTC:
2108                         intel_pt_calc_mtc_timestamp(decoder);
2109                         break;
2110
2111                 case INTEL_PT_TSC:
2112                         intel_pt_calc_tsc_timestamp(decoder);
2113                         break;
2114
2115                 case INTEL_PT_TMA:
2116                         intel_pt_calc_tma(decoder);
2117                         break;
2118
2119                 case INTEL_PT_CYC:
2120                         intel_pt_calc_cyc_timestamp(decoder);
2121                         break;
2122
2123                 case INTEL_PT_CBR:
2124                         intel_pt_calc_cbr(decoder);
2125                         break;
2126
2127                 case INTEL_PT_PIP:
2128                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2129                         break;
2130
2131                 case INTEL_PT_MODE_EXEC:
2132                         decoder->exec_mode = decoder->packet.payload;
2133                         break;
2134
2135                 case INTEL_PT_MODE_TSX:
2136                         intel_pt_update_in_tx(decoder);
2137                         break;
2138
2139                 case INTEL_PT_OVF:
2140                         return intel_pt_overflow(decoder);
2141
2142                 case INTEL_PT_BAD: /* Does not happen */
2143                         return intel_pt_bug(decoder);
2144
2145                 case INTEL_PT_TRACESTOP:
2146                         decoder->pge = false;
2147                         decoder->continuous_period = false;
2148                         intel_pt_clear_tx_flags(decoder);
2149                         decoder->have_tma = false;
2150                         break;
2151
2152                 case INTEL_PT_PSB:
2153                         decoder->last_ip = 0;
2154                         decoder->have_last_ip = true;
2155                         intel_pt_clear_stack(&decoder->stack);
2156                         err = intel_pt_walk_psb(decoder);
2157                         if (err)
2158                                 return err;
2159                         if (decoder->ip) {
2160                                 /* Do not have a sample */
2161                                 decoder->state.type = 0;
2162                                 return 0;
2163                         }
2164                         break;
2165
2166                 case INTEL_PT_TNT:
2167                 case INTEL_PT_PSBEND:
2168                 case INTEL_PT_VMCS:
2169                 case INTEL_PT_MNT:
2170                 case INTEL_PT_PAD:
2171                 case INTEL_PT_PTWRITE:
2172                 case INTEL_PT_PTWRITE_IP:
2173                 case INTEL_PT_EXSTOP:
2174                 case INTEL_PT_EXSTOP_IP:
2175                 case INTEL_PT_MWAIT:
2176                 case INTEL_PT_PWRE:
2177                 case INTEL_PT_PWRX:
2178                 default:
2179                         break;
2180                 }
2181         }
2182 }
2183
2184 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2185 {
2186         int err;
2187
2188         decoder->set_fup_tx_flags = false;
2189         decoder->set_fup_ptw = false;
2190         decoder->set_fup_mwait = false;
2191         decoder->set_fup_pwre = false;
2192         decoder->set_fup_exstop = false;
2193
2194         if (!decoder->branch_enable) {
2195                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2196                 decoder->overflow = false;
2197                 decoder->state.type = 0; /* Do not have a sample */
2198                 return 0;
2199         }
2200
2201         intel_pt_log("Scanning for full IP\n");
2202         err = intel_pt_walk_to_ip(decoder);
2203         if (err)
2204                 return err;
2205
2206         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2207         decoder->overflow = false;
2208
2209         decoder->state.from_ip = 0;
2210         decoder->state.to_ip = decoder->ip;
2211         intel_pt_log_to("Setting IP", decoder->ip);
2212
2213         return 0;
2214 }
2215
2216 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2217 {
2218         const unsigned char *end = decoder->buf + decoder->len;
2219         size_t i;
2220
2221         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2222                 if (i > decoder->len)
2223                         continue;
2224                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2225                         return i;
2226         }
2227         return 0;
2228 }
2229
2230 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2231 {
2232         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2233         const char *psb = INTEL_PT_PSB_STR;
2234
2235         if (rest_psb > decoder->len ||
2236             memcmp(decoder->buf, psb + part_psb, rest_psb))
2237                 return 0;
2238
2239         return rest_psb;
2240 }
2241
2242 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2243                                   int part_psb)
2244 {
2245         int rest_psb, ret;
2246
2247         decoder->pos += decoder->len;
2248         decoder->len = 0;
2249
2250         ret = intel_pt_get_next_data(decoder);
2251         if (ret)
2252                 return ret;
2253
2254         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2255         if (!rest_psb)
2256                 return 0;
2257
2258         decoder->pos -= part_psb;
2259         decoder->next_buf = decoder->buf + rest_psb;
2260         decoder->next_len = decoder->len - rest_psb;
2261         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2262         decoder->buf = decoder->temp_buf;
2263         decoder->len = INTEL_PT_PSB_LEN;
2264
2265         return 0;
2266 }
2267
2268 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2269 {
2270         unsigned char *next;
2271         int ret;
2272
2273         intel_pt_log("Scanning for PSB\n");
2274         while (1) {
2275                 if (!decoder->len) {
2276                         ret = intel_pt_get_next_data(decoder);
2277                         if (ret)
2278                                 return ret;
2279                 }
2280
2281                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2282                               INTEL_PT_PSB_LEN);
2283                 if (!next) {
2284                         int part_psb;
2285
2286                         part_psb = intel_pt_part_psb(decoder);
2287                         if (part_psb) {
2288                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2289                                 if (ret)
2290                                         return ret;
2291                         } else {
2292                                 decoder->pos += decoder->len;
2293                                 decoder->len = 0;
2294                         }
2295                         continue;
2296                 }
2297
2298                 decoder->pkt_step = next - decoder->buf;
2299                 return intel_pt_get_next_packet(decoder);
2300         }
2301 }
2302
2303 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2304 {
2305         int err;
2306
2307         decoder->pge = false;
2308         decoder->continuous_period = false;
2309         decoder->have_last_ip = false;
2310         decoder->last_ip = 0;
2311         decoder->ip = 0;
2312         intel_pt_clear_stack(&decoder->stack);
2313
2314         err = intel_pt_scan_for_psb(decoder);
2315         if (err)
2316                 return err;
2317
2318         decoder->have_last_ip = true;
2319         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2320
2321         err = intel_pt_walk_psb(decoder);
2322         if (err)
2323                 return err;
2324
2325         if (decoder->ip) {
2326                 decoder->state.type = 0; /* Do not have a sample */
2327                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2328         } else {
2329                 return intel_pt_sync_ip(decoder);
2330         }
2331
2332         return 0;
2333 }
2334
2335 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2336 {
2337         uint64_t est = decoder->sample_insn_cnt << 1;
2338
2339         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2340                 goto out;
2341
2342         est *= decoder->max_non_turbo_ratio;
2343         est /= decoder->cbr;
2344 out:
2345         return decoder->sample_timestamp + est;
2346 }
2347
2348 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2349 {
2350         int err;
2351
2352         do {
2353                 decoder->state.type = INTEL_PT_BRANCH;
2354                 decoder->state.flags = 0;
2355
2356                 switch (decoder->pkt_state) {
2357                 case INTEL_PT_STATE_NO_PSB:
2358                         err = intel_pt_sync(decoder);
2359                         break;
2360                 case INTEL_PT_STATE_NO_IP:
2361                         decoder->have_last_ip = false;
2362                         decoder->last_ip = 0;
2363                         decoder->ip = 0;
2364                         __fallthrough;
2365                 case INTEL_PT_STATE_ERR_RESYNC:
2366                         err = intel_pt_sync_ip(decoder);
2367                         break;
2368                 case INTEL_PT_STATE_IN_SYNC:
2369                         err = intel_pt_walk_trace(decoder);
2370                         break;
2371                 case INTEL_PT_STATE_TNT:
2372                 case INTEL_PT_STATE_TNT_CONT:
2373                         err = intel_pt_walk_tnt(decoder);
2374                         if (err == -EAGAIN)
2375                                 err = intel_pt_walk_trace(decoder);
2376                         break;
2377                 case INTEL_PT_STATE_TIP:
2378                 case INTEL_PT_STATE_TIP_PGD:
2379                         err = intel_pt_walk_tip(decoder);
2380                         break;
2381                 case INTEL_PT_STATE_FUP:
2382                         err = intel_pt_walk_fup(decoder);
2383                         if (err == -EAGAIN)
2384                                 err = intel_pt_walk_fup_tip(decoder);
2385                         break;
2386                 case INTEL_PT_STATE_FUP_NO_TIP:
2387                         err = intel_pt_walk_fup(decoder);
2388                         if (err == -EAGAIN)
2389                                 err = intel_pt_walk_trace(decoder);
2390                         break;
2391                 default:
2392                         err = intel_pt_bug(decoder);
2393                         break;
2394                 }
2395         } while (err == -ENOLINK);
2396
2397         if (err) {
2398                 decoder->state.err = intel_pt_ext_err(err);
2399                 decoder->state.from_ip = decoder->ip;
2400                 decoder->sample_timestamp = decoder->timestamp;
2401                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2402         } else {
2403                 decoder->state.err = 0;
2404                 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2405                         decoder->cbr_seen = decoder->cbr;
2406                         decoder->state.type |= INTEL_PT_CBR_CHG;
2407                         decoder->state.cbr_payload = decoder->cbr_payload;
2408                 }
2409                 if (intel_pt_sample_time(decoder->pkt_state)) {
2410                         decoder->sample_timestamp = decoder->timestamp;
2411                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2412                 }
2413         }
2414
2415         decoder->state.timestamp = decoder->sample_timestamp;
2416         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2417         decoder->state.cr3 = decoder->cr3;
2418         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2419
2420         return &decoder->state;
2421 }
2422
2423 /**
2424  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2425  * @buf: pointer to buffer pointer
2426  * @len: size of buffer
2427  *
2428  * Updates the buffer pointer to point to the start of the next PSB packet if
2429  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2430  * @len is adjusted accordingly.
2431  *
2432  * Return: %true if a PSB packet is found, %false otherwise.
2433  */
2434 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2435 {
2436         unsigned char *next;
2437
2438         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2439         if (next) {
2440                 *len -= next - *buf;
2441                 *buf = next;
2442                 return true;
2443         }
2444         return false;
2445 }
2446
2447 /**
2448  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2449  *                     packet.
2450  * @buf: pointer to buffer pointer
2451  * @len: size of buffer
2452  *
2453  * Updates the buffer pointer to point to the start of the following PSB packet
2454  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2455  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2456  *
2457  * Return: %true if a PSB packet is found, %false otherwise.
2458  */
2459 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2460 {
2461         unsigned char *next;
2462
2463         if (!*len)
2464                 return false;
2465
2466         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2467         if (next) {
2468                 *len -= next - *buf;
2469                 *buf = next;
2470                 return true;
2471         }
2472         return false;
2473 }
2474
2475 /**
2476  * intel_pt_last_psb - find the last PSB packet in a buffer.
2477  * @buf: buffer
2478  * @len: size of buffer
2479  *
2480  * This function finds the last PSB in a buffer.
2481  *
2482  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2483  */
2484 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2485 {
2486         const char *n = INTEL_PT_PSB_STR;
2487         unsigned char *p;
2488         size_t k;
2489
2490         if (len < INTEL_PT_PSB_LEN)
2491                 return NULL;
2492
2493         k = len - INTEL_PT_PSB_LEN + 1;
2494         while (1) {
2495                 p = memrchr(buf, n[0], k);
2496                 if (!p)
2497                         return NULL;
2498                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2499                         return p;
2500                 k = p - buf;
2501                 if (!k)
2502                         return NULL;
2503         }
2504 }
2505
2506 /**
2507  * intel_pt_next_tsc - find and return next TSC.
2508  * @buf: buffer
2509  * @len: size of buffer
2510  * @tsc: TSC value returned
2511  * @rem: returns remaining size when TSC is found
2512  *
2513  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2514  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2515  * PSBEND packet is found.
2516  *
2517  * Return: %true if TSC is found, false otherwise.
2518  */
2519 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2520                               size_t *rem)
2521 {
2522         struct intel_pt_pkt packet;
2523         int ret;
2524
2525         while (len) {
2526                 ret = intel_pt_get_packet(buf, len, &packet);
2527                 if (ret <= 0)
2528                         return false;
2529                 if (packet.type == INTEL_PT_TSC) {
2530                         *tsc = packet.payload;
2531                         *rem = len;
2532                         return true;
2533                 }
2534                 if (packet.type == INTEL_PT_PSBEND)
2535                         return false;
2536                 buf += ret;
2537                 len -= ret;
2538         }
2539         return false;
2540 }
2541
2542 /**
2543  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2544  * @tsc1: first TSC to compare
2545  * @tsc2: second TSC to compare
2546  *
2547  * This function compares 7-byte TSC values allowing for the possibility that
2548  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2549  * around so for that purpose this function assumes the absolute difference is
2550  * less than half the maximum difference.
2551  *
2552  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2553  * after @tsc2.
2554  */
2555 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2556 {
2557         const uint64_t halfway = (1ULL << 55);
2558
2559         if (tsc1 == tsc2)
2560                 return 0;
2561
2562         if (tsc1 < tsc2) {
2563                 if (tsc2 - tsc1 < halfway)
2564                         return -1;
2565                 else
2566                         return 1;
2567         } else {
2568                 if (tsc1 - tsc2 < halfway)
2569                         return 1;
2570                 else
2571                         return -1;
2572         }
2573 }
2574
2575 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2576
2577 /**
2578  * adj_for_padding - adjust overlap to account for padding.
2579  * @buf_b: second buffer
2580  * @buf_a: first buffer
2581  * @len_a: size of first buffer
2582  *
2583  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2584  * accordingly.
2585  *
2586  * Return: A pointer into @buf_b from where non-overlapped data starts
2587  */
2588 static unsigned char *adj_for_padding(unsigned char *buf_b,
2589                                       unsigned char *buf_a, size_t len_a)
2590 {
2591         unsigned char *p = buf_b - MAX_PADDING;
2592         unsigned char *q = buf_a + len_a - MAX_PADDING;
2593         int i;
2594
2595         for (i = MAX_PADDING; i; i--, p++, q++) {
2596                 if (*p != *q)
2597                         break;
2598         }
2599
2600         return p;
2601 }
2602
2603 /**
2604  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2605  *                             using TSC.
2606  * @buf_a: first buffer
2607  * @len_a: size of first buffer
2608  * @buf_b: second buffer
2609  * @len_b: size of second buffer
2610  * @consecutive: returns true if there is data in buf_b that is consecutive
2611  *               to buf_a
2612  *
2613  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2614  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2615  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2616  * @buf_a and @buf_b are positioned at a PSB.
2617  *
2618  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2619  * @buf_b + @len_b if there is no non-overlapped data.
2620  */
2621 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2622                                                 size_t len_a,
2623                                                 unsigned char *buf_b,
2624                                                 size_t len_b, bool *consecutive)
2625 {
2626         uint64_t tsc_a, tsc_b;
2627         unsigned char *p;
2628         size_t len, rem_a, rem_b;
2629
2630         p = intel_pt_last_psb(buf_a, len_a);
2631         if (!p)
2632                 return buf_b; /* No PSB in buf_a => no overlap */
2633
2634         len = len_a - (p - buf_a);
2635         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2636                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2637                 len_a -= len;
2638                 p = intel_pt_last_psb(buf_a, len_a);
2639                 if (!p)
2640                         return buf_b; /* No full PSB+ => assume no overlap */
2641                 len = len_a - (p - buf_a);
2642                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2643                         return buf_b; /* No TSC in buf_a => assume no overlap */
2644         }
2645
2646         while (1) {
2647                 /* Ignore PSB+ with no TSC */
2648                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2649                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2650
2651                         /* Same TSC, so buffers are consecutive */
2652                         if (!cmp && rem_b >= rem_a) {
2653                                 unsigned char *start;
2654
2655                                 *consecutive = true;
2656                                 start = buf_b + len_b - (rem_b - rem_a);
2657                                 return adj_for_padding(start, buf_a, len_a);
2658                         }
2659                         if (cmp < 0)
2660                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2661                 }
2662
2663                 if (!intel_pt_step_psb(&buf_b, &len_b))
2664                         return buf_b + len_b; /* No PSB in buf_b => no data */
2665         }
2666 }
2667
2668 /**
2669  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2670  * @buf_a: first buffer
2671  * @len_a: size of first buffer
2672  * @buf_b: second buffer
2673  * @len_b: size of second buffer
2674  * @have_tsc: can use TSC packets to detect overlap
2675  * @consecutive: returns true if there is data in buf_b that is consecutive
2676  *               to buf_a
2677  *
2678  * When trace samples or snapshots are recorded there is the possibility that
2679  * the data overlaps.  Note that, for the purposes of decoding, data is only
2680  * useful if it begins with a PSB packet.
2681  *
2682  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2683  * @buf_b + @len_b if there is no non-overlapped data.
2684  */
2685 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2686                                      unsigned char *buf_b, size_t len_b,
2687                                      bool have_tsc, bool *consecutive)
2688 {
2689         unsigned char *found;
2690
2691         /* Buffer 'b' must start at PSB so throw away everything before that */
2692         if (!intel_pt_next_psb(&buf_b, &len_b))
2693                 return buf_b + len_b; /* No PSB */
2694
2695         if (!intel_pt_next_psb(&buf_a, &len_a))
2696                 return buf_b; /* No overlap */
2697
2698         if (have_tsc) {
2699                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2700                                                   consecutive);
2701                 if (found)
2702                         return found;
2703         }
2704
2705         /*
2706          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2707          * we can ignore the first part of buffer 'a'.
2708          */
2709         while (len_b < len_a) {
2710                 if (!intel_pt_step_psb(&buf_a, &len_a))
2711                         return buf_b; /* No overlap */
2712         }
2713
2714         /* Now len_b >= len_a */
2715         while (1) {
2716                 /* Potential overlap so check the bytes */
2717                 found = memmem(buf_a, len_a, buf_b, len_a);
2718                 if (found) {
2719                         *consecutive = true;
2720                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
2721                 }
2722
2723                 /* Try again at next PSB in buffer 'a' */
2724                 if (!intel_pt_step_psb(&buf_a, &len_a))
2725                         return buf_b; /* No overlap */
2726         }
2727 }