tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29 #include "../auxtrace.h"
  30
  31 #include "intel-pt-insn-decoder.h"
  32 #include "intel-pt-pkt-decoder.h"
  33 #include "intel-pt-decoder.h"
  34 #include "intel-pt-log.h"
  35
  36 #define INTEL_PT_BLK_SIZE 1024
  37
  38 #define BIT63 (((uint64_t)1 << 63))
  39
  40 #define INTEL_PT_RETURN 1
  41
  42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  43 #define INTEL_PT_MAX_LOOPS 10000
  44
  45 struct intel_pt_blk {
  46         struct intel_pt_blk *prev;
  47         uint64_t ip[INTEL_PT_BLK_SIZE];
  48 };
  49
  50 struct intel_pt_stack {
  51         struct intel_pt_blk *blk;
  52         struct intel_pt_blk *spare;
  53         int pos;
  54 };
  55
  56 enum intel_pt_pkt_state {
  57         INTEL_PT_STATE_NO_PSB,
  58         INTEL_PT_STATE_NO_IP,
  59         INTEL_PT_STATE_ERR_RESYNC,
  60         INTEL_PT_STATE_IN_SYNC,
  61         INTEL_PT_STATE_TNT_CONT,
  62         INTEL_PT_STATE_TNT,
  63         INTEL_PT_STATE_TIP,
  64         INTEL_PT_STATE_TIP_PGD,
  65         INTEL_PT_STATE_FUP,
  66         INTEL_PT_STATE_FUP_NO_TIP,
  67 };
  68
  69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  70 {
  71         switch (pkt_state) {
  72         case INTEL_PT_STATE_NO_PSB:
  73         case INTEL_PT_STATE_NO_IP:
  74         case INTEL_PT_STATE_ERR_RESYNC:
  75         case INTEL_PT_STATE_IN_SYNC:
  76         case INTEL_PT_STATE_TNT_CONT:
  77                 return true;
  78         case INTEL_PT_STATE_TNT:
  79         case INTEL_PT_STATE_TIP:
  80         case INTEL_PT_STATE_TIP_PGD:
  81         case INTEL_PT_STATE_FUP:
  82         case INTEL_PT_STATE_FUP_NO_TIP:
  83                 return false;
  84         default:
  85                 return true;
  86         };
  87 }
  88
  89 #ifdef INTEL_PT_STRICT
  90 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  91 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  92 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  93 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  94 #else
  95 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  96 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  97 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  98 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  99 #endif
 100
 101 struct intel_pt_decoder {
 102         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 103         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 104                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 105                          uint64_t max_insn_cnt, void *data);
 106         bool (*pgd_ip)(uint64_t ip, void *data);
 107         void *data;
 108         struct intel_pt_state state;
 109         const unsigned char *buf;
 110         size_t len;
 111         bool return_compression;
 112         bool branch_enable;
 113         bool mtc_insn;
 114         bool pge;
 115         bool have_tma;
 116         bool have_cyc;
 117         bool fixup_last_mtc;
 118         bool have_last_ip;
 119         enum intel_pt_param_flags flags;
 120         uint64_t pos;
 121         uint64_t last_ip;
 122         uint64_t ip;
 123         uint64_t cr3;
 124         uint64_t timestamp;
 125         uint64_t tsc_timestamp;
 126         uint64_t ref_timestamp;
 127         uint64_t sample_timestamp;
 128         uint64_t ret_addr;
 129         uint64_t ctc_timestamp;
 130         uint64_t ctc_delta;
 131         uint64_t cycle_cnt;
 132         uint64_t cyc_ref_timestamp;
 133         uint32_t last_mtc;
 134         uint32_t tsc_ctc_ratio_n;
 135         uint32_t tsc_ctc_ratio_d;
 136         uint32_t tsc_ctc_mult;
 137         uint32_t tsc_slip;
 138         uint32_t ctc_rem_mask;
 139         int mtc_shift;
 140         struct intel_pt_stack stack;
 141         enum intel_pt_pkt_state pkt_state;
 142         struct intel_pt_pkt packet;
 143         struct intel_pt_pkt tnt;
 144         int pkt_step;
 145         int pkt_len;
 146         int last_packet_type;
 147         unsigned int cbr;
 148         unsigned int cbr_seen;
 149         unsigned int max_non_turbo_ratio;
 150         double max_non_turbo_ratio_fp;
 151         double cbr_cyc_to_tsc;
 152         double calc_cyc_to_tsc;
 153         bool have_calc_cyc_to_tsc;
 154         int exec_mode;
 155         unsigned int insn_bytes;
 156         uint64_t period;
 157         enum intel_pt_period_type period_type;
 158         uint64_t tot_insn_cnt;
 159         uint64_t period_insn_cnt;
 160         uint64_t period_mask;
 161         uint64_t period_ticks;
 162         uint64_t last_masked_timestamp;
 163         bool continuous_period;
 164         bool overflow;
 165         bool set_fup_tx_flags;
 166         bool set_fup_ptw;
 167         bool set_fup_mwait;
 168         bool set_fup_pwre;
 169         bool set_fup_exstop;
 170         unsigned int fup_tx_flags;
 171         unsigned int tx_flags;
 172         uint64_t fup_ptw_payload;
 173         uint64_t fup_mwait_payload;
 174         uint64_t fup_pwre_payload;
 175         uint64_t cbr_payload;
 176         uint64_t timestamp_insn_cnt;
 177         uint64_t sample_insn_cnt;
 178         uint64_t stuck_ip;
 179         int no_progress;
 180         int stuck_ip_prd;
 181         int stuck_ip_cnt;
 182         const unsigned char *next_buf;
 183         size_t next_len;
 184         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 185 };
 186
 187 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 188 {
 189         int i;
 190
 191         for (i = 0; x != 1; i++)
 192                 x >>= 1;
 193
 194         return x << i;
 195 }
 196
 197 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 198 {
 199         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 200                 uint64_t period;
 201
 202                 period = intel_pt_lower_power_of_2(decoder->period);
 203                 decoder->period_mask  = ~(period - 1);
 204                 decoder->period_ticks = period;
 205         }
 206 }
 207
 208 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 209 {
 210         if (!d)
 211                 return 0;
 212         return (t / d) * n + ((t % d) * n) / d;
 213 }
 214
 215 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 216 {
 217         struct intel_pt_decoder *decoder;
 218
 219         if (!params->get_trace || !params->walk_insn)
 220                 return NULL;
 221
 222         decoder = zalloc(sizeof(struct intel_pt_decoder));
 223         if (!decoder)
 224                 return NULL;
 225
 226         decoder->get_trace          = params->get_trace;
 227         decoder->walk_insn          = params->walk_insn;
 228         decoder->pgd_ip             = params->pgd_ip;
 229         decoder->data               = params->data;
 230         decoder->return_compression = params->return_compression;
 231         decoder->branch_enable      = params->branch_enable;
 232
 233         decoder->flags              = params->flags;
 234
 235         decoder->period             = params->period;
 236         decoder->period_type        = params->period_type;
 237
 238         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 239         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 240
 241         intel_pt_setup_period(decoder);
 242
 243         decoder->mtc_shift = params->mtc_period;
 244         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 245
 246         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 247         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 248
 249         if (!decoder->tsc_ctc_ratio_n)
 250                 decoder->tsc_ctc_ratio_d = 0;
 251
 252         if (decoder->tsc_ctc_ratio_d) {
 253                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 254                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 255                                                 decoder->tsc_ctc_ratio_d;
 256         }
 257
 258         /*
 259          * A TSC packet can slip past MTC packets so that the timestamp appears
 260          * to go backwards. One estimate is that can be up to about 40 CPU
 261          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 262          * slippage an order of magnitude more to be on the safe side.
 263          */
 264         decoder->tsc_slip = 0x10000;
 265
 266         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 267         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 268         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 269         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 270         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 271
 272         return decoder;
 273 }
 274
 275 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 276 {
 277         struct intel_pt_blk *blk = stack->blk;
 278
 279         stack->blk = blk->prev;
 280         if (!stack->spare)
 281                 stack->spare = blk;
 282         else
 283                 free(blk);
 284 }
 285
 286 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 287 {
 288         if (!stack->pos) {
 289                 if (!stack->blk)
 290                         return 0;
 291                 intel_pt_pop_blk(stack);
 292                 if (!stack->blk)
 293                         return 0;
 294                 stack->pos = INTEL_PT_BLK_SIZE;
 295         }
 296         return stack->blk->ip[--stack->pos];
 297 }
 298
 299 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 300 {
 301         struct intel_pt_blk *blk;
 302
 303         if (stack->spare) {
 304                 blk = stack->spare;
 305                 stack->spare = NULL;
 306         } else {
 307                 blk = malloc(sizeof(struct intel_pt_blk));
 308                 if (!blk)
 309                         return -ENOMEM;
 310         }
 311
 312         blk->prev = stack->blk;
 313         stack->blk = blk;
 314         stack->pos = 0;
 315         return 0;
 316 }
 317
 318 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 319 {
 320         int err;
 321
 322         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 323                 err = intel_pt_alloc_blk(stack);
 324                 if (err)
 325                         return err;
 326         }
 327
 328         stack->blk->ip[stack->pos++] = ip;
 329         return 0;
 330 }
 331
 332 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 333 {
 334         while (stack->blk)
 335                 intel_pt_pop_blk(stack);
 336         stack->pos = 0;
 337 }
 338
 339 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 340 {
 341         intel_pt_clear_stack(stack);
 342         zfree(&stack->blk);
 343         zfree(&stack->spare);
 344 }
 345
 346 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 347 {
 348         intel_pt_free_stack(&decoder->stack);
 349         free(decoder);
 350 }
 351
 352 static int intel_pt_ext_err(int code)
 353 {
 354         switch (code) {
 355         case -ENOMEM:
 356                 return INTEL_PT_ERR_NOMEM;
 357         case -ENOSYS:
 358                 return INTEL_PT_ERR_INTERN;
 359         case -EBADMSG:
 360                 return INTEL_PT_ERR_BADPKT;
 361         case -ENODATA:
 362                 return INTEL_PT_ERR_NODATA;
 363         case -EILSEQ:
 364                 return INTEL_PT_ERR_NOINSN;
 365         case -ENOENT:
 366                 return INTEL_PT_ERR_MISMAT;
 367         case -EOVERFLOW:
 368                 return INTEL_PT_ERR_OVR;
 369         case -ENOSPC:
 370                 return INTEL_PT_ERR_LOST;
 371         case -ELOOP:
 372                 return INTEL_PT_ERR_NELOOP;
 373         default:
 374                 return INTEL_PT_ERR_UNK;
 375         }
 376 }
 377
 378 static const char *intel_pt_err_msgs[] = {
 379         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 380         [INTEL_PT_ERR_INTERN] = "Internal error",
 381         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 382         [INTEL_PT_ERR_NODATA] = "No more data",
 383         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 384         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 385         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 386         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 387         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 388         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 389 };
 390
 391 int intel_pt__strerror(int code, char *buf, size_t buflen)
 392 {
 393         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 394                 code = INTEL_PT_ERR_UNK;
 395         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 396         return 0;
 397 }
 398
 399 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 400                                  uint64_t last_ip)
 401 {
 402         uint64_t ip;
 403
 404         switch (packet->count) {
 405         case 1:
 406                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 407                      packet->payload;
 408                 break;
 409         case 2:
 410                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 411                      packet->payload;
 412                 break;
 413         case 3:
 414                 ip = packet->payload;
 415                 /* Sign-extend 6-byte ip */
 416                 if (ip & (uint64_t)0x800000000000ULL)
 417                         ip |= (uint64_t)0xffff000000000000ULL;
 418                 break;
 419         case 4:
 420                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 421                      packet->payload;
 422                 break;
 423         case 6:
 424                 ip = packet->payload;
 425                 break;
 426         default:
 427                 return 0;
 428         }
 429
 430         return ip;
 431 }
 432
 433 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 434 {
 435         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 436         decoder->have_last_ip = true;
 437 }
 438
 439 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 440 {
 441         intel_pt_set_last_ip(decoder);
 442         decoder->ip = decoder->last_ip;
 443 }
 444
 445 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 446 {
 447         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 448                             decoder->buf);
 449 }
 450
 451 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 452 {
 453         intel_pt_log("ERROR: Internal error\n");
 454         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 455         return -ENOSYS;
 456 }
 457
 458 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 459 {
 460         decoder->tx_flags = 0;
 461 }
 462
 463 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 464 {
 465         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 466 }
 467
 468 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 469 {
 470         intel_pt_clear_tx_flags(decoder);
 471         decoder->have_tma = false;
 472         decoder->pkt_len = 1;
 473         decoder->pkt_step = 1;
 474         intel_pt_decoder_log_packet(decoder);
 475         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 476                 intel_pt_log("ERROR: Bad packet\n");
 477                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 478         }
 479         return -EBADMSG;
 480 }
 481
 482 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 483 {
 484         struct intel_pt_buffer buffer = { .buf = 0, };
 485         int ret;
 486
 487         decoder->pkt_step = 0;
 488
 489         intel_pt_log("Getting more data\n");
 490         ret = decoder->get_trace(&buffer, decoder->data);
 491         if (ret)
 492                 return ret;
 493         decoder->buf = buffer.buf;
 494         decoder->len = buffer.len;
 495         if (!decoder->len) {
 496                 intel_pt_log("No more data\n");
 497                 return -ENODATA;
 498         }
 499         if (!buffer.consecutive) {
 500                 decoder->ip = 0;
 501                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 502                 decoder->ref_timestamp = buffer.ref_timestamp;
 503                 decoder->timestamp = 0;
 504                 decoder->have_tma = false;
 505                 decoder->state.trace_nr = buffer.trace_nr;
 506                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 507                              decoder->ref_timestamp);
 508                 return -ENOLINK;
 509         }
 510
 511         return 0;
 512 }
 513
 514 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 515 {
 516         if (!decoder->next_buf)
 517                 return intel_pt_get_data(decoder);
 518
 519         decoder->buf = decoder->next_buf;
 520         decoder->len = decoder->next_len;
 521         decoder->next_buf = 0;
 522         decoder->next_len = 0;
 523         return 0;
 524 }
 525
 526 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 527 {
 528         unsigned char *buf = decoder->temp_buf;
 529         size_t old_len, len, n;
 530         int ret;
 531
 532         old_len = decoder->len;
 533         len = decoder->len;
 534         memcpy(buf, decoder->buf, len);
 535
 536         ret = intel_pt_get_data(decoder);
 537         if (ret) {
 538                 decoder->pos += old_len;
 539                 return ret < 0 ? ret : -EINVAL;
 540         }
 541
 542         n = INTEL_PT_PKT_MAX_SZ - len;
 543         if (n > decoder->len)
 544                 n = decoder->len;
 545         memcpy(buf + len, decoder->buf, n);
 546         len += n;
 547
 548         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 549         if (ret < (int)old_len) {
 550                 decoder->next_buf = decoder->buf;
 551                 decoder->next_len = decoder->len;
 552                 decoder->buf = buf;
 553                 decoder->len = old_len;
 554                 return intel_pt_bad_packet(decoder);
 555         }
 556
 557         decoder->next_buf = decoder->buf + (ret - old_len);
 558         decoder->next_len = decoder->len - (ret - old_len);
 559
 560         decoder->buf = buf;
 561         decoder->len = ret;
 562
 563         return ret;
 564 }
 565
 566 struct intel_pt_pkt_info {
 567         struct intel_pt_decoder   *decoder;
 568         struct intel_pt_pkt       packet;
 569         uint64_t                  pos;
 570         int                       pkt_len;
 571         int                       last_packet_type;
 572         void                      *data;
 573 };
 574
 575 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 576
 577 /* Lookahead packets in current buffer */
 578 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 579                                   intel_pt_pkt_cb_t cb, void *data)
 580 {
 581         struct intel_pt_pkt_info pkt_info;
 582         const unsigned char *buf = decoder->buf;
 583         size_t len = decoder->len;
 584         int ret;
 585
 586         pkt_info.decoder          = decoder;
 587         pkt_info.pos              = decoder->pos;
 588         pkt_info.pkt_len          = decoder->pkt_step;
 589         pkt_info.last_packet_type = decoder->last_packet_type;
 590         pkt_info.data             = data;
 591
 592         while (1) {
 593                 do {
 594                         pkt_info.pos += pkt_info.pkt_len;
 595                         buf          += pkt_info.pkt_len;
 596                         len          -= pkt_info.pkt_len;
 597
 598                         if (!len)
 599                                 return INTEL_PT_NEED_MORE_BYTES;
 600
 601                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 602                         if (!ret)
 603                                 return INTEL_PT_NEED_MORE_BYTES;
 604                         if (ret < 0)
 605                                 return ret;
 606
 607                         pkt_info.pkt_len = ret;
 608                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 609
 610                 ret = cb(&pkt_info);
 611                 if (ret)
 612                         return 0;
 613
 614                 pkt_info.last_packet_type = pkt_info.packet.type;
 615         }
 616 }
 617
 618 struct intel_pt_calc_cyc_to_tsc_info {
 619         uint64_t        cycle_cnt;
 620         unsigned int    cbr;
 621         uint32_t        last_mtc;
 622         uint64_t        ctc_timestamp;
 623         uint64_t        ctc_delta;
 624         uint64_t        tsc_timestamp;
 625         uint64_t        timestamp;
 626         bool            have_tma;
 627         bool            fixup_last_mtc;
 628         bool            from_mtc;
 629         double          cbr_cyc_to_tsc;
 630 };
 631
 632 /*
 633  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 634  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 635  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 636  * packet by copying the missing bits from the current MTC assuming the least
 637  * difference between the two, and that the current MTC comes after last_mtc.
 638  */
 639 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 640                                     uint32_t *last_mtc)
 641 {
 642         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 643         uint32_t mask = ~(first_missing_bit - 1);
 644
 645         *last_mtc |= mtc & mask;
 646         if (*last_mtc >= mtc) {
 647                 *last_mtc -= first_missing_bit;
 648                 *last_mtc &= 0xff;
 649         }
 650 }
 651
 652 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 653 {
 654         struct intel_pt_decoder *decoder = pkt_info->decoder;
 655         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 656         uint64_t timestamp;
 657         double cyc_to_tsc;
 658         unsigned int cbr;
 659         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 660
 661         switch (pkt_info->packet.type) {
 662         case INTEL_PT_TNT:
 663         case INTEL_PT_TIP_PGE:
 664         case INTEL_PT_TIP:
 665         case INTEL_PT_FUP:
 666         case INTEL_PT_PSB:
 667         case INTEL_PT_PIP:
 668         case INTEL_PT_MODE_EXEC:
 669         case INTEL_PT_MODE_TSX:
 670         case INTEL_PT_PSBEND:
 671         case INTEL_PT_PAD:
 672         case INTEL_PT_VMCS:
 673         case INTEL_PT_MNT:
 674         case INTEL_PT_PTWRITE:
 675         case INTEL_PT_PTWRITE_IP:
 676                 return 0;
 677
 678         case INTEL_PT_MTC:
 679                 if (!data->have_tma)
 680                         return 0;
 681
 682                 mtc = pkt_info->packet.payload;
 683                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 684                         data->fixup_last_mtc = false;
 685                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 686                                                 &data->last_mtc);
 687                 }
 688                 if (mtc > data->last_mtc)
 689                         mtc_delta = mtc - data->last_mtc;
 690                 else
 691                         mtc_delta = mtc + 256 - data->last_mtc;
 692                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 693                 data->last_mtc = mtc;
 694
 695                 if (decoder->tsc_ctc_mult) {
 696                         timestamp = data->ctc_timestamp +
 697                                 data->ctc_delta * decoder->tsc_ctc_mult;
 698                 } else {
 699                         timestamp = data->ctc_timestamp +
 700                                 multdiv(data->ctc_delta,
 701                                         decoder->tsc_ctc_ratio_n,
 702                                         decoder->tsc_ctc_ratio_d);
 703                 }
 704
 705                 if (timestamp < data->timestamp)
 706                         return 1;
 707
 708                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 709                         data->timestamp = timestamp;
 710                         return 0;
 711                 }
 712
 713                 break;
 714
 715         case INTEL_PT_TSC:
 716                 /*
 717                  * For now, do not support using TSC packets - refer
 718                  * intel_pt_calc_cyc_to_tsc().
 719                  */
 720                 if (data->from_mtc)
 721                         return 1;
 722                 timestamp = pkt_info->packet.payload |
 723                             (data->timestamp & (0xffULL << 56));
 724                 if (data->from_mtc && timestamp < data->timestamp &&
 725                     data->timestamp - timestamp < decoder->tsc_slip)
 726                         return 1;
 727                 if (timestamp < data->timestamp)
 728                         timestamp += (1ULL << 56);
 729                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 730                         if (data->from_mtc)
 731                                 return 1;
 732                         data->tsc_timestamp = timestamp;
 733                         data->timestamp = timestamp;
 734                         return 0;
 735                 }
 736                 break;
 737
 738         case INTEL_PT_TMA:
 739                 if (data->from_mtc)
 740                         return 1;
 741
 742                 if (!decoder->tsc_ctc_ratio_d)
 743                         return 0;
 744
 745                 ctc = pkt_info->packet.payload;
 746                 fc = pkt_info->packet.count;
 747                 ctc_rem = ctc & decoder->ctc_rem_mask;
 748
 749                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 750
 751                 data->ctc_timestamp = data->tsc_timestamp - fc;
 752                 if (decoder->tsc_ctc_mult) {
 753                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 754                 } else {
 755                         data->ctc_timestamp -=
 756                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 757                                         decoder->tsc_ctc_ratio_d);
 758                 }
 759
 760                 data->ctc_delta = 0;
 761                 data->have_tma = true;
 762                 data->fixup_last_mtc = true;
 763
 764                 return 0;
 765
 766         case INTEL_PT_CYC:
 767                 data->cycle_cnt += pkt_info->packet.payload;
 768                 return 0;
 769
 770         case INTEL_PT_CBR:
 771                 cbr = pkt_info->packet.payload;
 772                 if (data->cbr && data->cbr != cbr)
 773                         return 1;
 774                 data->cbr = cbr;
 775                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 776                 return 0;
 777
 778         case INTEL_PT_TIP_PGD:
 779         case INTEL_PT_TRACESTOP:
 780         case INTEL_PT_EXSTOP:
 781         case INTEL_PT_EXSTOP_IP:
 782         case INTEL_PT_MWAIT:
 783         case INTEL_PT_PWRE:
 784         case INTEL_PT_PWRX:
 785         case INTEL_PT_OVF:
 786         case INTEL_PT_BAD: /* Does not happen */
 787         default:
 788                 return 1;
 789         }
 790
 791         if (!data->cbr && decoder->cbr) {
 792                 data->cbr = decoder->cbr;
 793                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 794         }
 795
 796         if (!data->cycle_cnt)
 797                 return 1;
 798
 799         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 800
 801         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 802             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 803                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 804                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 805                 return 1;
 806         }
 807
 808         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 809         decoder->have_calc_cyc_to_tsc = true;
 810
 811         if (data->cbr) {
 812                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 813                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 814         } else {
 815                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 816                              cyc_to_tsc, pkt_info->pos);
 817         }
 818
 819         return 1;
 820 }
 821
 822 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 823                                      bool from_mtc)
 824 {
 825         struct intel_pt_calc_cyc_to_tsc_info data = {
 826                 .cycle_cnt      = 0,
 827                 .cbr            = 0,
 828                 .last_mtc       = decoder->last_mtc,
 829                 .ctc_timestamp  = decoder->ctc_timestamp,
 830                 .ctc_delta      = decoder->ctc_delta,
 831                 .tsc_timestamp  = decoder->tsc_timestamp,
 832                 .timestamp      = decoder->timestamp,
 833                 .have_tma       = decoder->have_tma,
 834                 .fixup_last_mtc = decoder->fixup_last_mtc,
 835                 .from_mtc       = from_mtc,
 836                 .cbr_cyc_to_tsc = 0,
 837         };
 838
 839         /*
 840          * For now, do not support using TSC packets for at least the reasons:
 841          * 1) timing might have stopped
 842          * 2) TSC packets within PSB+ can slip against CYC packets
 843          */
 844         if (!from_mtc)
 845                 return;
 846
 847         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 848 }
 849
 850 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 851 {
 852         int ret;
 853
 854         decoder->last_packet_type = decoder->packet.type;
 855
 856         do {
 857                 decoder->pos += decoder->pkt_step;
 858                 decoder->buf += decoder->pkt_step;
 859                 decoder->len -= decoder->pkt_step;
 860
 861                 if (!decoder->len) {
 862                         ret = intel_pt_get_next_data(decoder);
 863                         if (ret)
 864                                 return ret;
 865                 }
 866
 867                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 868                                           &decoder->packet);
 869                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 870                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 871                         ret = intel_pt_get_split_packet(decoder);
 872                         if (ret < 0)
 873                                 return ret;
 874                 }
 875                 if (ret <= 0)
 876                         return intel_pt_bad_packet(decoder);
 877
 878                 decoder->pkt_len = ret;
 879                 decoder->pkt_step = ret;
 880                 intel_pt_decoder_log_packet(decoder);
 881         } while (decoder->packet.type == INTEL_PT_PAD);
 882
 883         return 0;
 884 }
 885
 886 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 887 {
 888         uint64_t timestamp, masked_timestamp;
 889
 890         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 891         masked_timestamp = timestamp & decoder->period_mask;
 892         if (decoder->continuous_period) {
 893                 if (masked_timestamp > decoder->last_masked_timestamp)
 894                         return 1;
 895         } else {
 896                 timestamp += 1;
 897                 masked_timestamp = timestamp & decoder->period_mask;
 898                 if (masked_timestamp > decoder->last_masked_timestamp) {
 899                         decoder->last_masked_timestamp = masked_timestamp;
 900                         decoder->continuous_period = true;
 901                 }
 902         }
 903
 904         if (masked_timestamp < decoder->last_masked_timestamp)
 905                 return decoder->period_ticks;
 906
 907         return decoder->period_ticks - (timestamp - masked_timestamp);
 908 }
 909
 910 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 911 {
 912         switch (decoder->period_type) {
 913         case INTEL_PT_PERIOD_INSTRUCTIONS:
 914                 return decoder->period - decoder->period_insn_cnt;
 915         case INTEL_PT_PERIOD_TICKS:
 916                 return intel_pt_next_period(decoder);
 917         case INTEL_PT_PERIOD_NONE:
 918         case INTEL_PT_PERIOD_MTC:
 919         default:
 920                 return 0;
 921         }
 922 }
 923
 924 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 925 {
 926         uint64_t timestamp, masked_timestamp;
 927
 928         switch (decoder->period_type) {
 929         case INTEL_PT_PERIOD_INSTRUCTIONS:
 930                 decoder->period_insn_cnt = 0;
 931                 break;
 932         case INTEL_PT_PERIOD_TICKS:
 933                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 934                 masked_timestamp = timestamp & decoder->period_mask;
 935                 if (masked_timestamp > decoder->last_masked_timestamp)
 936                         decoder->last_masked_timestamp = masked_timestamp;
 937                 else
 938                         decoder->last_masked_timestamp += decoder->period_ticks;
 939                 break;
 940         case INTEL_PT_PERIOD_NONE:
 941         case INTEL_PT_PERIOD_MTC:
 942         default:
 943                 break;
 944         }
 945
 946         decoder->state.type |= INTEL_PT_INSTRUCTION;
 947 }
 948
 949 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 950                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 951 {
 952         uint64_t max_insn_cnt, insn_cnt = 0;
 953         int err;
 954
 955         if (!decoder->mtc_insn)
 956                 decoder->mtc_insn = true;
 957
 958         max_insn_cnt = intel_pt_next_sample(decoder);
 959
 960         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 961                                  max_insn_cnt, decoder->data);
 962
 963         decoder->tot_insn_cnt += insn_cnt;
 964         decoder->timestamp_insn_cnt += insn_cnt;
 965         decoder->sample_insn_cnt += insn_cnt;
 966         decoder->period_insn_cnt += insn_cnt;
 967
 968         if (err) {
 969                 decoder->no_progress = 0;
 970                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 971                 intel_pt_log_at("ERROR: Failed to get instruction",
 972                                 decoder->ip);
 973                 if (err == -ENOENT)
 974                         return -ENOLINK;
 975                 return -EILSEQ;
 976         }
 977
 978         if (ip && decoder->ip == ip) {
 979                 err = -EAGAIN;
 980                 goto out;
 981         }
 982
 983         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 984                 intel_pt_sample_insn(decoder);
 985
 986         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 987                 decoder->state.type = INTEL_PT_INSTRUCTION;
 988                 decoder->state.from_ip = decoder->ip;
 989                 decoder->state.to_ip = 0;
 990                 decoder->ip += intel_pt_insn->length;
 991                 err = INTEL_PT_RETURN;
 992                 goto out;
 993         }
 994
 995         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 996                 /* Zero-length calls are excluded */
 997                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 998                     intel_pt_insn->rel) {
 999                         err = intel_pt_push(&decoder->stack, decoder->ip +
1000                                             intel_pt_insn->length);
1001                         if (err)
1002                                 goto out;
1003                 }
1004         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1005                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1006         }
1007
1008         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1009                 int cnt = decoder->no_progress++;
1010
1011                 decoder->state.from_ip = decoder->ip;
1012                 decoder->ip += intel_pt_insn->length +
1013                                 intel_pt_insn->rel;
1014                 decoder->state.to_ip = decoder->ip;
1015                 err = INTEL_PT_RETURN;
1016
1017                 /*
1018                  * Check for being stuck in a loop.  This can happen if a
1019                  * decoder error results in the decoder erroneously setting the
1020                  * ip to an address that is itself in an infinite loop that
1021                  * consumes no packets.  When that happens, there must be an
1022                  * unconditional branch.
1023                  */
1024                 if (cnt) {
1025                         if (cnt == 1) {
1026                                 decoder->stuck_ip = decoder->state.to_ip;
1027                                 decoder->stuck_ip_prd = 1;
1028                                 decoder->stuck_ip_cnt = 1;
1029                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
1030                                    decoder->state.to_ip == decoder->stuck_ip) {
1031                                 intel_pt_log_at("ERROR: Never-ending loop",
1032                                                 decoder->state.to_ip);
1033                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1034                                 err = -ELOOP;
1035                                 goto out;
1036                         } else if (!--decoder->stuck_ip_cnt) {
1037                                 decoder->stuck_ip_prd += 1;
1038                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1039                                 decoder->stuck_ip = decoder->state.to_ip;
1040                         }
1041                 }
1042                 goto out_no_progress;
1043         }
1044 out:
1045         decoder->no_progress = 0;
1046 out_no_progress:
1047         decoder->state.insn_op = intel_pt_insn->op;
1048         decoder->state.insn_len = intel_pt_insn->length;
1049         memcpy(decoder->state.insn, intel_pt_insn->buf,
1050                INTEL_PT_INSN_BUF_SZ);
1051
1052         if (decoder->tx_flags & INTEL_PT_IN_TX)
1053                 decoder->state.flags |= INTEL_PT_IN_TX;
1054
1055         return err;
1056 }
1057
1058 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1059 {
1060         bool ret = false;
1061
1062         if (decoder->set_fup_tx_flags) {
1063                 decoder->set_fup_tx_flags = false;
1064                 decoder->tx_flags = decoder->fup_tx_flags;
1065                 decoder->state.type = INTEL_PT_TRANSACTION;
1066                 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1067                         decoder->state.type |= INTEL_PT_BRANCH;
1068                 decoder->state.from_ip = decoder->ip;
1069                 decoder->state.to_ip = 0;
1070                 decoder->state.flags = decoder->fup_tx_flags;
1071                 return true;
1072         }
1073         if (decoder->set_fup_ptw) {
1074                 decoder->set_fup_ptw = false;
1075                 decoder->state.type = INTEL_PT_PTW;
1076                 decoder->state.flags |= INTEL_PT_FUP_IP;
1077                 decoder->state.from_ip = decoder->ip;
1078                 decoder->state.to_ip = 0;
1079                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1080                 return true;
1081         }
1082         if (decoder->set_fup_mwait) {
1083                 decoder->set_fup_mwait = false;
1084                 decoder->state.type = INTEL_PT_MWAIT_OP;
1085                 decoder->state.from_ip = decoder->ip;
1086                 decoder->state.to_ip = 0;
1087                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1088                 ret = true;
1089         }
1090         if (decoder->set_fup_pwre) {
1091                 decoder->set_fup_pwre = false;
1092                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1093                 decoder->state.type &= ~INTEL_PT_BRANCH;
1094                 decoder->state.from_ip = decoder->ip;
1095                 decoder->state.to_ip = 0;
1096                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1097                 ret = true;
1098         }
1099         if (decoder->set_fup_exstop) {
1100                 decoder->set_fup_exstop = false;
1101                 decoder->state.type |= INTEL_PT_EX_STOP;
1102                 decoder->state.type &= ~INTEL_PT_BRANCH;
1103                 decoder->state.flags |= INTEL_PT_FUP_IP;
1104                 decoder->state.from_ip = decoder->ip;
1105                 decoder->state.to_ip = 0;
1106                 ret = true;
1107         }
1108         return ret;
1109 }
1110
1111 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1112                                           struct intel_pt_insn *intel_pt_insn,
1113                                           uint64_t ip, int err)
1114 {
1115         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1116                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1117                ip == decoder->ip + intel_pt_insn->length;
1118 }
1119
1120 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1121 {
1122         struct intel_pt_insn intel_pt_insn;
1123         uint64_t ip;
1124         int err;
1125
1126         ip = decoder->last_ip;
1127
1128         while (1) {
1129                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1130                 if (err == INTEL_PT_RETURN)
1131                         return 0;
1132                 if (err == -EAGAIN ||
1133                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1134                         bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1135
1136                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1137                         if (intel_pt_fup_event(decoder) && no_tip)
1138                                 return 0;
1139                         return -EAGAIN;
1140                 }
1141                 decoder->set_fup_tx_flags = false;
1142                 if (err)
1143                         return err;
1144
1145                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1146                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1147                                         decoder->ip);
1148                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1149                         return -ENOENT;
1150                 }
1151
1152                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1153                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1154                                         decoder->ip);
1155                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1156                         return -ENOENT;
1157                 }
1158
1159                 intel_pt_bug(decoder);
1160         }
1161 }
1162
1163 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1164 {
1165         struct intel_pt_insn intel_pt_insn;
1166         int err;
1167
1168         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1169         if (err == INTEL_PT_RETURN &&
1170             decoder->pgd_ip &&
1171             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1172             (decoder->state.type & INTEL_PT_BRANCH) &&
1173             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1174                 /* Unconditional branch leaving filter region */
1175                 decoder->no_progress = 0;
1176                 decoder->pge = false;
1177                 decoder->continuous_period = false;
1178                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1179                 decoder->state.to_ip = 0;
1180                 return 0;
1181         }
1182         if (err == INTEL_PT_RETURN)
1183                 return 0;
1184         if (err)
1185                 return err;
1186
1187         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1188                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1189                         decoder->pge = false;
1190                         decoder->continuous_period = false;
1191                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1192                         decoder->state.from_ip = decoder->ip;
1193                         decoder->state.to_ip = 0;
1194                         if (decoder->packet.count != 0)
1195                                 decoder->ip = decoder->last_ip;
1196                 } else {
1197                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1198                         decoder->state.from_ip = decoder->ip;
1199                         if (decoder->packet.count == 0) {
1200                                 decoder->state.to_ip = 0;
1201                         } else {
1202                                 decoder->state.to_ip = decoder->last_ip;
1203                                 decoder->ip = decoder->last_ip;
1204                         }
1205                 }
1206                 return 0;
1207         }
1208
1209         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1210                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1211                                  intel_pt_insn.rel;
1212
1213                 if (decoder->pgd_ip &&
1214                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1215                     decoder->pgd_ip(to_ip, decoder->data)) {
1216                         /* Conditional branch leaving filter region */
1217                         decoder->pge = false;
1218                         decoder->continuous_period = false;
1219                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1220                         decoder->ip = to_ip;
1221                         decoder->state.from_ip = decoder->ip;
1222                         decoder->state.to_ip = 0;
1223                         return 0;
1224                 }
1225                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1226                                 decoder->ip);
1227                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1228                 return -ENOENT;
1229         }
1230
1231         return intel_pt_bug(decoder);
1232 }
1233
1234 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1235 {
1236         struct intel_pt_insn intel_pt_insn;
1237         int err;
1238
1239         while (1) {
1240                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1241                 if (err == INTEL_PT_RETURN)
1242                         return 0;
1243                 if (err)
1244                         return err;
1245
1246                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1247                         if (!decoder->return_compression) {
1248                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1249                                                 decoder->ip);
1250                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1251                                 return -ENOENT;
1252                         }
1253                         if (!decoder->ret_addr) {
1254                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1255                                                 decoder->ip);
1256                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1257                                 return -ENOENT;
1258                         }
1259                         if (!(decoder->tnt.payload & BIT63)) {
1260                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1261                                                 decoder->ip);
1262                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1263                                 return -ENOENT;
1264                         }
1265                         decoder->tnt.count -= 1;
1266                         if (decoder->tnt.count)
1267                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1268                         else
1269                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1270                         decoder->tnt.payload <<= 1;
1271                         decoder->state.from_ip = decoder->ip;
1272                         decoder->ip = decoder->ret_addr;
1273                         decoder->state.to_ip = decoder->ip;
1274                         return 0;
1275                 }
1276
1277                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1278                         /* Handle deferred TIPs */
1279                         err = intel_pt_get_next_packet(decoder);
1280                         if (err)
1281                                 return err;
1282                         if (decoder->packet.type != INTEL_PT_TIP ||
1283                             decoder->packet.count == 0) {
1284                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1285                                                 decoder->ip);
1286                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1287                                 decoder->pkt_step = 0;
1288                                 return -ENOENT;
1289                         }
1290                         intel_pt_set_last_ip(decoder);
1291                         decoder->state.from_ip = decoder->ip;
1292                         decoder->state.to_ip = decoder->last_ip;
1293                         decoder->ip = decoder->last_ip;
1294                         return 0;
1295                 }
1296
1297                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1298                         decoder->tnt.count -= 1;
1299                         if (decoder->tnt.count)
1300                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1301                         else
1302                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1303                         if (decoder->tnt.payload & BIT63) {
1304                                 decoder->tnt.payload <<= 1;
1305                                 decoder->state.from_ip = decoder->ip;
1306                                 decoder->ip += intel_pt_insn.length +
1307                                                intel_pt_insn.rel;
1308                                 decoder->state.to_ip = decoder->ip;
1309                                 return 0;
1310                         }
1311                         /* Instruction sample for a non-taken branch */
1312                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1313                                 decoder->tnt.payload <<= 1;
1314                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1315                                 decoder->state.from_ip = decoder->ip;
1316                                 decoder->state.to_ip = 0;
1317                                 decoder->ip += intel_pt_insn.length;
1318                                 return 0;
1319                         }
1320                         decoder->ip += intel_pt_insn.length;
1321                         if (!decoder->tnt.count) {
1322                                 decoder->sample_timestamp = decoder->timestamp;
1323                                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
1324                                 return -EAGAIN;
1325                         }
1326                         decoder->tnt.payload <<= 1;
1327                         continue;
1328                 }
1329
1330                 return intel_pt_bug(decoder);
1331         }
1332 }
1333
1334 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1335 {
1336         unsigned int fup_tx_flags;
1337         int err;
1338
1339         fup_tx_flags = decoder->packet.payload &
1340                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1341         err = intel_pt_get_next_packet(decoder);
1342         if (err)
1343                 return err;
1344         if (decoder->packet.type == INTEL_PT_FUP) {
1345                 decoder->fup_tx_flags = fup_tx_flags;
1346                 decoder->set_fup_tx_flags = true;
1347                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1348                         *no_tip = true;
1349         } else {
1350                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1351                                 decoder->pos);
1352                 intel_pt_update_in_tx(decoder);
1353         }
1354         return 0;
1355 }
1356
1357 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1358 {
1359         uint64_t timestamp;
1360
1361         decoder->have_tma = false;
1362
1363         if (decoder->ref_timestamp) {
1364                 timestamp = decoder->packet.payload |
1365                             (decoder->ref_timestamp & (0xffULL << 56));
1366                 if (timestamp < decoder->ref_timestamp) {
1367                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1368                                 timestamp += (1ULL << 56);
1369                 } else {
1370                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1371                                 timestamp -= (1ULL << 56);
1372                 }
1373                 decoder->tsc_timestamp = timestamp;
1374                 decoder->timestamp = timestamp;
1375                 decoder->ref_timestamp = 0;
1376                 decoder->timestamp_insn_cnt = 0;
1377         } else if (decoder->timestamp) {
1378                 timestamp = decoder->packet.payload |
1379                             (decoder->timestamp & (0xffULL << 56));
1380                 decoder->tsc_timestamp = timestamp;
1381                 if (timestamp < decoder->timestamp &&
1382                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1383                         intel_pt_log_to("Suppressing backwards timestamp",
1384                                         timestamp);
1385                         timestamp = decoder->timestamp;
1386                 }
1387                 if (timestamp < decoder->timestamp) {
1388                         intel_pt_log_to("Wraparound timestamp", timestamp);
1389                         timestamp += (1ULL << 56);
1390                         decoder->tsc_timestamp = timestamp;
1391                 }
1392                 decoder->timestamp = timestamp;
1393                 decoder->timestamp_insn_cnt = 0;
1394         }
1395
1396         if (decoder->last_packet_type == INTEL_PT_CYC) {
1397                 decoder->cyc_ref_timestamp = decoder->timestamp;
1398                 decoder->cycle_cnt = 0;
1399                 decoder->have_calc_cyc_to_tsc = false;
1400                 intel_pt_calc_cyc_to_tsc(decoder, false);
1401         }
1402
1403         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1404 }
1405
1406 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1407 {
1408         intel_pt_log("ERROR: Buffer overflow\n");
1409         intel_pt_clear_tx_flags(decoder);
1410         decoder->timestamp_insn_cnt = 0;
1411         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1412         decoder->overflow = true;
1413         return -EOVERFLOW;
1414 }
1415
1416 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1417 {
1418         uint32_t ctc = decoder->packet.payload;
1419         uint32_t fc = decoder->packet.count;
1420         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1421
1422         if (!decoder->tsc_ctc_ratio_d)
1423                 return;
1424
1425         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1426         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1427         if (decoder->tsc_ctc_mult) {
1428                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1429         } else {
1430                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1431                                                   decoder->tsc_ctc_ratio_n,
1432                                                   decoder->tsc_ctc_ratio_d);
1433         }
1434         decoder->ctc_delta = 0;
1435         decoder->have_tma = true;
1436         decoder->fixup_last_mtc = true;
1437         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1438                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1439 }
1440
1441 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1442 {
1443         uint64_t timestamp;
1444         uint32_t mtc, mtc_delta;
1445
1446         if (!decoder->have_tma)
1447                 return;
1448
1449         mtc = decoder->packet.payload;
1450
1451         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1452                 decoder->fixup_last_mtc = false;
1453                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1454                                         &decoder->last_mtc);
1455         }
1456
1457         if (mtc > decoder->last_mtc)
1458                 mtc_delta = mtc - decoder->last_mtc;
1459         else
1460                 mtc_delta = mtc + 256 - decoder->last_mtc;
1461
1462         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1463
1464         if (decoder->tsc_ctc_mult) {
1465                 timestamp = decoder->ctc_timestamp +
1466                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1467         } else {
1468                 timestamp = decoder->ctc_timestamp +
1469                             multdiv(decoder->ctc_delta,
1470                                     decoder->tsc_ctc_ratio_n,
1471                                     decoder->tsc_ctc_ratio_d);
1472         }
1473
1474         if (timestamp < decoder->timestamp)
1475                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1476                              timestamp, decoder->timestamp);
1477         else
1478                 decoder->timestamp = timestamp;
1479
1480         decoder->timestamp_insn_cnt = 0;
1481         decoder->last_mtc = mtc;
1482
1483         if (decoder->last_packet_type == INTEL_PT_CYC) {
1484                 decoder->cyc_ref_timestamp = decoder->timestamp;
1485                 decoder->cycle_cnt = 0;
1486                 decoder->have_calc_cyc_to_tsc = false;
1487                 intel_pt_calc_cyc_to_tsc(decoder, true);
1488         }
1489 }
1490
1491 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1492 {
1493         unsigned int cbr = decoder->packet.payload & 0xff;
1494
1495         decoder->cbr_payload = decoder->packet.payload;
1496
1497         if (decoder->cbr == cbr)
1498                 return;
1499
1500         decoder->cbr = cbr;
1501         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1502         decoder->cyc_ref_timestamp = decoder->timestamp;
1503         decoder->cycle_cnt = 0;
1504 }
1505
1506 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1507 {
1508         uint64_t timestamp = decoder->cyc_ref_timestamp;
1509
1510         decoder->have_cyc = true;
1511
1512         decoder->cycle_cnt += decoder->packet.payload;
1513
1514         if (!decoder->cyc_ref_timestamp)
1515                 return;
1516
1517         if (decoder->have_calc_cyc_to_tsc)
1518                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1519         else if (decoder->cbr)
1520                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1521         else
1522                 return;
1523
1524         if (timestamp < decoder->timestamp)
1525                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1526                              timestamp, decoder->timestamp);
1527         else
1528                 decoder->timestamp = timestamp;
1529
1530         decoder->timestamp_insn_cnt = 0;
1531 }
1532
1533 /* Walk PSB+ packets when already in sync. */
1534 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1535 {
1536         int err;
1537
1538         while (1) {
1539                 err = intel_pt_get_next_packet(decoder);
1540                 if (err)
1541                         return err;
1542
1543                 switch (decoder->packet.type) {
1544                 case INTEL_PT_PSBEND:
1545                         return 0;
1546
1547                 case INTEL_PT_TIP_PGD:
1548                 case INTEL_PT_TIP_PGE:
1549                 case INTEL_PT_TIP:
1550                 case INTEL_PT_TNT:
1551                 case INTEL_PT_TRACESTOP:
1552                 case INTEL_PT_BAD:
1553                 case INTEL_PT_PSB:
1554                 case INTEL_PT_PTWRITE:
1555                 case INTEL_PT_PTWRITE_IP:
1556                 case INTEL_PT_EXSTOP:
1557                 case INTEL_PT_EXSTOP_IP:
1558                 case INTEL_PT_MWAIT:
1559                 case INTEL_PT_PWRE:
1560                 case INTEL_PT_PWRX:
1561                         decoder->have_tma = false;
1562                         intel_pt_log("ERROR: Unexpected packet\n");
1563                         return -EAGAIN;
1564
1565                 case INTEL_PT_OVF:
1566                         return intel_pt_overflow(decoder);
1567
1568                 case INTEL_PT_TSC:
1569                         intel_pt_calc_tsc_timestamp(decoder);
1570                         break;
1571
1572                 case INTEL_PT_TMA:
1573                         intel_pt_calc_tma(decoder);
1574                         break;
1575
1576                 case INTEL_PT_CBR:
1577                         intel_pt_calc_cbr(decoder);
1578                         break;
1579
1580                 case INTEL_PT_MODE_EXEC:
1581                         decoder->exec_mode = decoder->packet.payload;
1582                         break;
1583
1584                 case INTEL_PT_PIP:
1585                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1586                         break;
1587
1588                 case INTEL_PT_FUP:
1589                         decoder->pge = true;
1590                         if (decoder->packet.count)
1591                                 intel_pt_set_last_ip(decoder);
1592                         break;
1593
1594                 case INTEL_PT_MODE_TSX:
1595                         intel_pt_update_in_tx(decoder);
1596                         break;
1597
1598                 case INTEL_PT_MTC:
1599                         intel_pt_calc_mtc_timestamp(decoder);
1600                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1601                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1602                         break;
1603
1604                 case INTEL_PT_CYC:
1605                         intel_pt_calc_cyc_timestamp(decoder);
1606                         break;
1607
1608                 case INTEL_PT_VMCS:
1609                 case INTEL_PT_MNT:
1610                 case INTEL_PT_PAD:
1611                 default:
1612                         break;
1613                 }
1614         }
1615 }
1616
1617 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1618 {
1619         int err;
1620
1621         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1622                 decoder->tx_flags = 0;
1623                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1624                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1625         } else {
1626                 decoder->state.flags |= INTEL_PT_ASYNC;
1627         }
1628
1629         while (1) {
1630                 err = intel_pt_get_next_packet(decoder);
1631                 if (err)
1632                         return err;
1633
1634                 switch (decoder->packet.type) {
1635                 case INTEL_PT_TNT:
1636                 case INTEL_PT_FUP:
1637                 case INTEL_PT_TRACESTOP:
1638                 case INTEL_PT_PSB:
1639                 case INTEL_PT_TSC:
1640                 case INTEL_PT_TMA:
1641                 case INTEL_PT_MODE_TSX:
1642                 case INTEL_PT_BAD:
1643                 case INTEL_PT_PSBEND:
1644                 case INTEL_PT_PTWRITE:
1645                 case INTEL_PT_PTWRITE_IP:
1646                 case INTEL_PT_EXSTOP:
1647                 case INTEL_PT_EXSTOP_IP:
1648                 case INTEL_PT_MWAIT:
1649                 case INTEL_PT_PWRE:
1650                 case INTEL_PT_PWRX:
1651                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1652                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1653                         decoder->pkt_step = 0;
1654                         return -ENOENT;
1655
1656                 case INTEL_PT_CBR:
1657                         intel_pt_calc_cbr(decoder);
1658                         break;
1659
1660                 case INTEL_PT_OVF:
1661                         return intel_pt_overflow(decoder);
1662
1663                 case INTEL_PT_TIP_PGD:
1664                         decoder->state.from_ip = decoder->ip;
1665                         decoder->state.to_ip = 0;
1666                         if (decoder->packet.count != 0) {
1667                                 intel_pt_set_ip(decoder);
1668                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1669                                              decoder->ip);
1670                         }
1671                         decoder->pge = false;
1672                         decoder->continuous_period = false;
1673                         return 0;
1674
1675                 case INTEL_PT_TIP_PGE:
1676                         decoder->pge = true;
1677                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1678                                      decoder->ip);
1679                         decoder->state.from_ip = 0;
1680                         if (decoder->packet.count == 0) {
1681                                 decoder->state.to_ip = 0;
1682                         } else {
1683                                 intel_pt_set_ip(decoder);
1684                                 decoder->state.to_ip = decoder->ip;
1685                         }
1686                         return 0;
1687
1688                 case INTEL_PT_TIP:
1689                         decoder->state.from_ip = decoder->ip;
1690                         if (decoder->packet.count == 0) {
1691                                 decoder->state.to_ip = 0;
1692                         } else {
1693                                 intel_pt_set_ip(decoder);
1694                                 decoder->state.to_ip = decoder->ip;
1695                         }
1696                         return 0;
1697
1698                 case INTEL_PT_PIP:
1699                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1700                         break;
1701
1702                 case INTEL_PT_MTC:
1703                         intel_pt_calc_mtc_timestamp(decoder);
1704                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1705                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1706                         break;
1707
1708                 case INTEL_PT_CYC:
1709                         intel_pt_calc_cyc_timestamp(decoder);
1710                         break;
1711
1712                 case INTEL_PT_MODE_EXEC:
1713                         decoder->exec_mode = decoder->packet.payload;
1714                         break;
1715
1716                 case INTEL_PT_VMCS:
1717                 case INTEL_PT_MNT:
1718                 case INTEL_PT_PAD:
1719                         break;
1720
1721                 default:
1722                         return intel_pt_bug(decoder);
1723                 }
1724         }
1725 }
1726
1727 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1728 {
1729         bool no_tip = false;
1730         int err;
1731
1732         while (1) {
1733                 err = intel_pt_get_next_packet(decoder);
1734                 if (err)
1735                         return err;
1736 next:
1737                 switch (decoder->packet.type) {
1738                 case INTEL_PT_TNT:
1739                         if (!decoder->packet.count)
1740                                 break;
1741                         decoder->tnt = decoder->packet;
1742                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1743                         err = intel_pt_walk_tnt(decoder);
1744                         if (err == -EAGAIN)
1745                                 break;
1746                         return err;
1747
1748                 case INTEL_PT_TIP_PGD:
1749                         if (decoder->packet.count != 0)
1750                                 intel_pt_set_last_ip(decoder);
1751                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1752                         return intel_pt_walk_tip(decoder);
1753
1754                 case INTEL_PT_TIP_PGE: {
1755                         decoder->pge = true;
1756                         if (decoder->packet.count == 0) {
1757                                 intel_pt_log_at("Skipping zero TIP.PGE",
1758                                                 decoder->pos);
1759                                 break;
1760                         }
1761                         intel_pt_set_ip(decoder);
1762                         decoder->state.from_ip = 0;
1763                         decoder->state.to_ip = decoder->ip;
1764                         return 0;
1765                 }
1766
1767                 case INTEL_PT_OVF:
1768                         return intel_pt_overflow(decoder);
1769
1770                 case INTEL_PT_TIP:
1771                         if (decoder->packet.count != 0)
1772                                 intel_pt_set_last_ip(decoder);
1773                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1774                         return intel_pt_walk_tip(decoder);
1775
1776                 case INTEL_PT_FUP:
1777                         if (decoder->packet.count == 0) {
1778                                 intel_pt_log_at("Skipping zero FUP",
1779                                                 decoder->pos);
1780                                 no_tip = false;
1781                                 break;
1782                         }
1783                         intel_pt_set_last_ip(decoder);
1784                         if (!decoder->branch_enable) {
1785                                 decoder->ip = decoder->last_ip;
1786                                 if (intel_pt_fup_event(decoder))
1787                                         return 0;
1788                                 no_tip = false;
1789                                 break;
1790                         }
1791                         if (decoder->set_fup_mwait)
1792                                 no_tip = true;
1793                         if (no_tip)
1794                                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
1795                         else
1796                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
1797                         err = intel_pt_walk_fup(decoder);
1798                         if (err != -EAGAIN)
1799                                 return err;
1800                         if (no_tip) {
1801                                 no_tip = false;
1802                                 break;
1803                         }
1804                         return intel_pt_walk_fup_tip(decoder);
1805
1806                 case INTEL_PT_TRACESTOP:
1807                         decoder->pge = false;
1808                         decoder->continuous_period = false;
1809                         intel_pt_clear_tx_flags(decoder);
1810                         decoder->have_tma = false;
1811                         break;
1812
1813                 case INTEL_PT_PSB:
1814                         decoder->last_ip = 0;
1815                         decoder->have_last_ip = true;
1816                         intel_pt_clear_stack(&decoder->stack);
1817                         err = intel_pt_walk_psbend(decoder);
1818                         if (err == -EAGAIN)
1819                                 goto next;
1820                         if (err)
1821                                 return err;
1822                         break;
1823
1824                 case INTEL_PT_PIP:
1825                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1826                         break;
1827
1828                 case INTEL_PT_MTC:
1829                         intel_pt_calc_mtc_timestamp(decoder);
1830                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1831                                 break;
1832                         /*
1833                          * Ensure that there has been an instruction since the
1834                          * last MTC.
1835                          */
1836                         if (!decoder->mtc_insn)
1837                                 break;
1838                         decoder->mtc_insn = false;
1839                         /* Ensure that there is a timestamp */
1840                         if (!decoder->timestamp)
1841                                 break;
1842                         decoder->state.type = INTEL_PT_INSTRUCTION;
1843                         decoder->state.from_ip = decoder->ip;
1844                         decoder->state.to_ip = 0;
1845                         decoder->mtc_insn = false;
1846                         return 0;
1847
1848                 case INTEL_PT_TSC:
1849                         intel_pt_calc_tsc_timestamp(decoder);
1850                         break;
1851
1852                 case INTEL_PT_TMA:
1853                         intel_pt_calc_tma(decoder);
1854                         break;
1855
1856                 case INTEL_PT_CYC:
1857                         intel_pt_calc_cyc_timestamp(decoder);
1858                         break;
1859
1860                 case INTEL_PT_CBR:
1861                         intel_pt_calc_cbr(decoder);
1862                         if (!decoder->branch_enable &&
1863                             decoder->cbr != decoder->cbr_seen) {
1864                                 decoder->cbr_seen = decoder->cbr;
1865                                 decoder->state.type = INTEL_PT_CBR_CHG;
1866                                 decoder->state.from_ip = decoder->ip;
1867                                 decoder->state.to_ip = 0;
1868                                 decoder->state.cbr_payload =
1869                                                         decoder->packet.payload;
1870                                 return 0;
1871                         }
1872                         break;
1873
1874                 case INTEL_PT_MODE_EXEC:
1875                         decoder->exec_mode = decoder->packet.payload;
1876                         break;
1877
1878                 case INTEL_PT_MODE_TSX:
1879                         /* MODE_TSX need not be followed by FUP */
1880                         if (!decoder->pge) {
1881                                 intel_pt_update_in_tx(decoder);
1882                                 break;
1883                         }
1884                         err = intel_pt_mode_tsx(decoder, &no_tip);
1885                         if (err)
1886                                 return err;
1887                         goto next;
1888
1889                 case INTEL_PT_BAD: /* Does not happen */
1890                         return intel_pt_bug(decoder);
1891
1892                 case INTEL_PT_PSBEND:
1893                 case INTEL_PT_VMCS:
1894                 case INTEL_PT_MNT:
1895                 case INTEL_PT_PAD:
1896                         break;
1897
1898                 case INTEL_PT_PTWRITE_IP:
1899                         decoder->fup_ptw_payload = decoder->packet.payload;
1900                         err = intel_pt_get_next_packet(decoder);
1901                         if (err)
1902                                 return err;
1903                         if (decoder->packet.type == INTEL_PT_FUP) {
1904                                 decoder->set_fup_ptw = true;
1905                                 no_tip = true;
1906                         } else {
1907                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1908                                                 decoder->pos);
1909                         }
1910                         goto next;
1911
1912                 case INTEL_PT_PTWRITE:
1913                         decoder->state.type = INTEL_PT_PTW;
1914                         decoder->state.from_ip = decoder->ip;
1915                         decoder->state.to_ip = 0;
1916                         decoder->state.ptw_payload = decoder->packet.payload;
1917                         return 0;
1918
1919                 case INTEL_PT_MWAIT:
1920                         decoder->fup_mwait_payload = decoder->packet.payload;
1921                         decoder->set_fup_mwait = true;
1922                         break;
1923
1924                 case INTEL_PT_PWRE:
1925                         if (decoder->set_fup_mwait) {
1926                                 decoder->fup_pwre_payload =
1927                                                         decoder->packet.payload;
1928                                 decoder->set_fup_pwre = true;
1929                                 break;
1930                         }
1931                         decoder->state.type = INTEL_PT_PWR_ENTRY;
1932                         decoder->state.from_ip = decoder->ip;
1933                         decoder->state.to_ip = 0;
1934                         decoder->state.pwrx_payload = decoder->packet.payload;
1935                         return 0;
1936
1937                 case INTEL_PT_EXSTOP_IP:
1938                         err = intel_pt_get_next_packet(decoder);
1939                         if (err)
1940                                 return err;
1941                         if (decoder->packet.type == INTEL_PT_FUP) {
1942                                 decoder->set_fup_exstop = true;
1943                                 no_tip = true;
1944                         } else {
1945                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1946                                                 decoder->pos);
1947                         }
1948                         goto next;
1949
1950                 case INTEL_PT_EXSTOP:
1951                         decoder->state.type = INTEL_PT_EX_STOP;
1952                         decoder->state.from_ip = decoder->ip;
1953                         decoder->state.to_ip = 0;
1954                         return 0;
1955
1956                 case INTEL_PT_PWRX:
1957                         decoder->state.type = INTEL_PT_PWR_EXIT;
1958                         decoder->state.from_ip = decoder->ip;
1959                         decoder->state.to_ip = 0;
1960                         decoder->state.pwrx_payload = decoder->packet.payload;
1961                         return 0;
1962
1963                 default:
1964                         return intel_pt_bug(decoder);
1965                 }
1966         }
1967 }
1968
1969 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1970 {
1971         return decoder->packet.count &&
1972                (decoder->have_last_ip || decoder->packet.count == 3 ||
1973                 decoder->packet.count == 6);
1974 }
1975
1976 /* Walk PSB+ packets to get in sync. */
1977 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1978 {
1979         int err;
1980
1981         while (1) {
1982                 err = intel_pt_get_next_packet(decoder);
1983                 if (err)
1984                         return err;
1985
1986                 switch (decoder->packet.type) {
1987                 case INTEL_PT_TIP_PGD:
1988                         decoder->continuous_period = false;
1989                         __fallthrough;
1990                 case INTEL_PT_TIP_PGE:
1991                 case INTEL_PT_TIP:
1992                 case INTEL_PT_PTWRITE:
1993                 case INTEL_PT_PTWRITE_IP:
1994                 case INTEL_PT_EXSTOP:
1995                 case INTEL_PT_EXSTOP_IP:
1996                 case INTEL_PT_MWAIT:
1997                 case INTEL_PT_PWRE:
1998                 case INTEL_PT_PWRX:
1999                         intel_pt_log("ERROR: Unexpected packet\n");
2000                         return -ENOENT;
2001
2002                 case INTEL_PT_FUP:
2003                         decoder->pge = true;
2004                         if (intel_pt_have_ip(decoder)) {
2005                                 uint64_t current_ip = decoder->ip;
2006
2007                                 intel_pt_set_ip(decoder);
2008                                 if (current_ip)
2009                                         intel_pt_log_to("Setting IP",
2010                                                         decoder->ip);
2011                         }
2012                         break;
2013
2014                 case INTEL_PT_MTC:
2015                         intel_pt_calc_mtc_timestamp(decoder);
2016                         break;
2017
2018                 case INTEL_PT_TSC:
2019                         intel_pt_calc_tsc_timestamp(decoder);
2020                         break;
2021
2022                 case INTEL_PT_TMA:
2023                         intel_pt_calc_tma(decoder);
2024                         break;
2025
2026                 case INTEL_PT_CYC:
2027                         intel_pt_calc_cyc_timestamp(decoder);
2028                         break;
2029
2030                 case INTEL_PT_CBR:
2031                         intel_pt_calc_cbr(decoder);
2032                         break;
2033
2034                 case INTEL_PT_PIP:
2035                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2036                         break;
2037
2038                 case INTEL_PT_MODE_EXEC:
2039                         decoder->exec_mode = decoder->packet.payload;
2040                         break;
2041
2042                 case INTEL_PT_MODE_TSX:
2043                         intel_pt_update_in_tx(decoder);
2044                         break;
2045
2046                 case INTEL_PT_TRACESTOP:
2047                         decoder->pge = false;
2048                         decoder->continuous_period = false;
2049                         intel_pt_clear_tx_flags(decoder);
2050                         __fallthrough;
2051
2052                 case INTEL_PT_TNT:
2053                         decoder->have_tma = false;
2054                         intel_pt_log("ERROR: Unexpected packet\n");
2055                         if (decoder->ip)
2056                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2057                         else
2058                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2059                         return -ENOENT;
2060
2061                 case INTEL_PT_BAD: /* Does not happen */
2062                         return intel_pt_bug(decoder);
2063
2064                 case INTEL_PT_OVF:
2065                         return intel_pt_overflow(decoder);
2066
2067                 case INTEL_PT_PSBEND:
2068                         return 0;
2069
2070                 case INTEL_PT_PSB:
2071                 case INTEL_PT_VMCS:
2072                 case INTEL_PT_MNT:
2073                 case INTEL_PT_PAD:
2074                 default:
2075                         break;
2076                 }
2077         }
2078 }
2079
2080 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2081 {
2082         int err;
2083
2084         while (1) {
2085                 err = intel_pt_get_next_packet(decoder);
2086                 if (err)
2087                         return err;
2088
2089                 switch (decoder->packet.type) {
2090                 case INTEL_PT_TIP_PGD:
2091                         decoder->continuous_period = false;
2092                         __fallthrough;
2093                 case INTEL_PT_TIP_PGE:
2094                 case INTEL_PT_TIP:
2095                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
2096                         if (intel_pt_have_ip(decoder))
2097                                 intel_pt_set_ip(decoder);
2098                         if (decoder->ip)
2099                                 return 0;
2100                         break;
2101
2102                 case INTEL_PT_FUP:
2103                         if (intel_pt_have_ip(decoder))
2104                                 intel_pt_set_ip(decoder);
2105                         if (decoder->ip)
2106                                 return 0;
2107                         break;
2108
2109                 case INTEL_PT_MTC:
2110                         intel_pt_calc_mtc_timestamp(decoder);
2111                         break;
2112
2113                 case INTEL_PT_TSC:
2114                         intel_pt_calc_tsc_timestamp(decoder);
2115                         break;
2116
2117                 case INTEL_PT_TMA:
2118                         intel_pt_calc_tma(decoder);
2119                         break;
2120
2121                 case INTEL_PT_CYC:
2122                         intel_pt_calc_cyc_timestamp(decoder);
2123                         break;
2124
2125                 case INTEL_PT_CBR:
2126                         intel_pt_calc_cbr(decoder);
2127                         break;
2128
2129                 case INTEL_PT_PIP:
2130                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2131                         break;
2132
2133                 case INTEL_PT_MODE_EXEC:
2134                         decoder->exec_mode = decoder->packet.payload;
2135                         break;
2136
2137                 case INTEL_PT_MODE_TSX:
2138                         intel_pt_update_in_tx(decoder);
2139                         break;
2140
2141                 case INTEL_PT_OVF:
2142                         return intel_pt_overflow(decoder);
2143
2144                 case INTEL_PT_BAD: /* Does not happen */
2145                         return intel_pt_bug(decoder);
2146
2147                 case INTEL_PT_TRACESTOP:
2148                         decoder->pge = false;
2149                         decoder->continuous_period = false;
2150                         intel_pt_clear_tx_flags(decoder);
2151                         decoder->have_tma = false;
2152                         break;
2153
2154                 case INTEL_PT_PSB:
2155                         decoder->last_ip = 0;
2156                         decoder->have_last_ip = true;
2157                         intel_pt_clear_stack(&decoder->stack);
2158                         err = intel_pt_walk_psb(decoder);
2159                         if (err)
2160                                 return err;
2161                         if (decoder->ip) {
2162                                 /* Do not have a sample */
2163                                 decoder->state.type = 0;
2164                                 return 0;
2165                         }
2166                         break;
2167
2168                 case INTEL_PT_TNT:
2169                 case INTEL_PT_PSBEND:
2170                 case INTEL_PT_VMCS:
2171                 case INTEL_PT_MNT:
2172                 case INTEL_PT_PAD:
2173                 case INTEL_PT_PTWRITE:
2174                 case INTEL_PT_PTWRITE_IP:
2175                 case INTEL_PT_EXSTOP:
2176                 case INTEL_PT_EXSTOP_IP:
2177                 case INTEL_PT_MWAIT:
2178                 case INTEL_PT_PWRE:
2179                 case INTEL_PT_PWRX:
2180                 default:
2181                         break;
2182                 }
2183         }
2184 }
2185
2186 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2187 {
2188         int err;
2189
2190         decoder->set_fup_tx_flags = false;
2191         decoder->set_fup_ptw = false;
2192         decoder->set_fup_mwait = false;
2193         decoder->set_fup_pwre = false;
2194         decoder->set_fup_exstop = false;
2195
2196         if (!decoder->branch_enable) {
2197                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2198                 decoder->overflow = false;
2199                 decoder->state.type = 0; /* Do not have a sample */
2200                 return 0;
2201         }
2202
2203         intel_pt_log("Scanning for full IP\n");
2204         err = intel_pt_walk_to_ip(decoder);
2205         if (err)
2206                 return err;
2207
2208         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2209         decoder->overflow = false;
2210
2211         decoder->state.from_ip = 0;
2212         decoder->state.to_ip = decoder->ip;
2213         intel_pt_log_to("Setting IP", decoder->ip);
2214
2215         return 0;
2216 }
2217
2218 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2219 {
2220         const unsigned char *end = decoder->buf + decoder->len;
2221         size_t i;
2222
2223         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2224                 if (i > decoder->len)
2225                         continue;
2226                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2227                         return i;
2228         }
2229         return 0;
2230 }
2231
2232 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2233 {
2234         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2235         const char *psb = INTEL_PT_PSB_STR;
2236
2237         if (rest_psb > decoder->len ||
2238             memcmp(decoder->buf, psb + part_psb, rest_psb))
2239                 return 0;
2240
2241         return rest_psb;
2242 }
2243
2244 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2245                                   int part_psb)
2246 {
2247         int rest_psb, ret;
2248
2249         decoder->pos += decoder->len;
2250         decoder->len = 0;
2251
2252         ret = intel_pt_get_next_data(decoder);
2253         if (ret)
2254                 return ret;
2255
2256         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2257         if (!rest_psb)
2258                 return 0;
2259
2260         decoder->pos -= part_psb;
2261         decoder->next_buf = decoder->buf + rest_psb;
2262         decoder->next_len = decoder->len - rest_psb;
2263         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2264         decoder->buf = decoder->temp_buf;
2265         decoder->len = INTEL_PT_PSB_LEN;
2266
2267         return 0;
2268 }
2269
2270 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2271 {
2272         unsigned char *next;
2273         int ret;
2274
2275         intel_pt_log("Scanning for PSB\n");
2276         while (1) {
2277                 if (!decoder->len) {
2278                         ret = intel_pt_get_next_data(decoder);
2279                         if (ret)
2280                                 return ret;
2281                 }
2282
2283                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2284                               INTEL_PT_PSB_LEN);
2285                 if (!next) {
2286                         int part_psb;
2287
2288                         part_psb = intel_pt_part_psb(decoder);
2289                         if (part_psb) {
2290                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2291                                 if (ret)
2292                                         return ret;
2293                         } else {
2294                                 decoder->pos += decoder->len;
2295                                 decoder->len = 0;
2296                         }
2297                         continue;
2298                 }
2299
2300                 decoder->pkt_step = next - decoder->buf;
2301                 return intel_pt_get_next_packet(decoder);
2302         }
2303 }
2304
2305 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2306 {
2307         int err;
2308
2309         decoder->pge = false;
2310         decoder->continuous_period = false;
2311         decoder->have_last_ip = false;
2312         decoder->last_ip = 0;
2313         decoder->ip = 0;
2314         intel_pt_clear_stack(&decoder->stack);
2315
2316         err = intel_pt_scan_for_psb(decoder);
2317         if (err)
2318                 return err;
2319
2320         decoder->have_last_ip = true;
2321         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2322
2323         err = intel_pt_walk_psb(decoder);
2324         if (err)
2325                 return err;
2326
2327         if (decoder->ip) {
2328                 decoder->state.type = 0; /* Do not have a sample */
2329                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2330         } else {
2331                 return intel_pt_sync_ip(decoder);
2332         }
2333
2334         return 0;
2335 }
2336
2337 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2338 {
2339         uint64_t est = decoder->sample_insn_cnt << 1;
2340
2341         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2342                 goto out;
2343
2344         est *= decoder->max_non_turbo_ratio;
2345         est /= decoder->cbr;
2346 out:
2347         return decoder->sample_timestamp + est;
2348 }
2349
2350 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2351 {
2352         int err;
2353
2354         do {
2355                 decoder->state.type = INTEL_PT_BRANCH;
2356                 decoder->state.flags = 0;
2357
2358                 switch (decoder->pkt_state) {
2359                 case INTEL_PT_STATE_NO_PSB:
2360                         err = intel_pt_sync(decoder);
2361                         break;
2362                 case INTEL_PT_STATE_NO_IP:
2363                         decoder->have_last_ip = false;
2364                         decoder->last_ip = 0;
2365                         decoder->ip = 0;
2366                         __fallthrough;
2367                 case INTEL_PT_STATE_ERR_RESYNC:
2368                         err = intel_pt_sync_ip(decoder);
2369                         break;
2370                 case INTEL_PT_STATE_IN_SYNC:
2371                         err = intel_pt_walk_trace(decoder);
2372                         break;
2373                 case INTEL_PT_STATE_TNT:
2374                 case INTEL_PT_STATE_TNT_CONT:
2375                         err = intel_pt_walk_tnt(decoder);
2376                         if (err == -EAGAIN)
2377                                 err = intel_pt_walk_trace(decoder);
2378                         break;
2379                 case INTEL_PT_STATE_TIP:
2380                 case INTEL_PT_STATE_TIP_PGD:
2381                         err = intel_pt_walk_tip(decoder);
2382                         break;
2383                 case INTEL_PT_STATE_FUP:
2384                         err = intel_pt_walk_fup(decoder);
2385                         if (err == -EAGAIN)
2386                                 err = intel_pt_walk_fup_tip(decoder);
2387                         break;
2388                 case INTEL_PT_STATE_FUP_NO_TIP:
2389                         err = intel_pt_walk_fup(decoder);
2390                         if (err == -EAGAIN)
2391                                 err = intel_pt_walk_trace(decoder);
2392                         break;
2393                 default:
2394                         err = intel_pt_bug(decoder);
2395                         break;
2396                 }
2397         } while (err == -ENOLINK);
2398
2399         if (err) {
2400                 decoder->state.err = intel_pt_ext_err(err);
2401                 decoder->state.from_ip = decoder->ip;
2402                 decoder->sample_timestamp = decoder->timestamp;
2403                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2404         } else {
2405                 decoder->state.err = 0;
2406                 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2407                         decoder->cbr_seen = decoder->cbr;
2408                         decoder->state.type |= INTEL_PT_CBR_CHG;
2409                         decoder->state.cbr_payload = decoder->cbr_payload;
2410                 }
2411                 if (intel_pt_sample_time(decoder->pkt_state)) {
2412                         decoder->sample_timestamp = decoder->timestamp;
2413                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2414                 }
2415         }
2416
2417         decoder->state.timestamp = decoder->sample_timestamp;
2418         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2419         decoder->state.cr3 = decoder->cr3;
2420         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2421
2422         return &decoder->state;
2423 }
2424
2425 /**
2426  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2427  * @buf: pointer to buffer pointer
2428  * @len: size of buffer
2429  *
2430  * Updates the buffer pointer to point to the start of the next PSB packet if
2431  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2432  * @len is adjusted accordingly.
2433  *
2434  * Return: %true if a PSB packet is found, %false otherwise.
2435  */
2436 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2437 {
2438         unsigned char *next;
2439
2440         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2441         if (next) {
2442                 *len -= next - *buf;
2443                 *buf = next;
2444                 return true;
2445         }
2446         return false;
2447 }
2448
2449 /**
2450  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2451  *                     packet.
2452  * @buf: pointer to buffer pointer
2453  * @len: size of buffer
2454  *
2455  * Updates the buffer pointer to point to the start of the following PSB packet
2456  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2457  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2458  *
2459  * Return: %true if a PSB packet is found, %false otherwise.
2460  */
2461 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2462 {
2463         unsigned char *next;
2464
2465         if (!*len)
2466                 return false;
2467
2468         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2469         if (next) {
2470                 *len -= next - *buf;
2471                 *buf = next;
2472                 return true;
2473         }
2474         return false;
2475 }
2476
2477 /**
2478  * intel_pt_last_psb - find the last PSB packet in a buffer.
2479  * @buf: buffer
2480  * @len: size of buffer
2481  *
2482  * This function finds the last PSB in a buffer.
2483  *
2484  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2485  */
2486 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2487 {
2488         const char *n = INTEL_PT_PSB_STR;
2489         unsigned char *p;
2490         size_t k;
2491
2492         if (len < INTEL_PT_PSB_LEN)
2493                 return NULL;
2494
2495         k = len - INTEL_PT_PSB_LEN + 1;
2496         while (1) {
2497                 p = memrchr(buf, n[0], k);
2498                 if (!p)
2499                         return NULL;
2500                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2501                         return p;
2502                 k = p - buf;
2503                 if (!k)
2504                         return NULL;
2505         }
2506 }
2507
2508 /**
2509  * intel_pt_next_tsc - find and return next TSC.
2510  * @buf: buffer
2511  * @len: size of buffer
2512  * @tsc: TSC value returned
2513  * @rem: returns remaining size when TSC is found
2514  *
2515  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2516  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2517  * PSBEND packet is found.
2518  *
2519  * Return: %true if TSC is found, false otherwise.
2520  */
2521 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2522                               size_t *rem)
2523 {
2524         struct intel_pt_pkt packet;
2525         int ret;
2526
2527         while (len) {
2528                 ret = intel_pt_get_packet(buf, len, &packet);
2529                 if (ret <= 0)
2530                         return false;
2531                 if (packet.type == INTEL_PT_TSC) {
2532                         *tsc = packet.payload;
2533                         *rem = len;
2534                         return true;
2535                 }
2536                 if (packet.type == INTEL_PT_PSBEND)
2537                         return false;
2538                 buf += ret;
2539                 len -= ret;
2540         }
2541         return false;
2542 }
2543
2544 /**
2545  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2546  * @tsc1: first TSC to compare
2547  * @tsc2: second TSC to compare
2548  *
2549  * This function compares 7-byte TSC values allowing for the possibility that
2550  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2551  * around so for that purpose this function assumes the absolute difference is
2552  * less than half the maximum difference.
2553  *
2554  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2555  * after @tsc2.
2556  */
2557 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2558 {
2559         const uint64_t halfway = (1ULL << 55);
2560
2561         if (tsc1 == tsc2)
2562                 return 0;
2563
2564         if (tsc1 < tsc2) {
2565                 if (tsc2 - tsc1 < halfway)
2566                         return -1;
2567                 else
2568                         return 1;
2569         } else {
2570                 if (tsc1 - tsc2 < halfway)
2571                         return 1;
2572                 else
2573                         return -1;
2574         }
2575 }
2576
2577 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2578
2579 /**
2580  * adj_for_padding - adjust overlap to account for padding.
2581  * @buf_b: second buffer
2582  * @buf_a: first buffer
2583  * @len_a: size of first buffer
2584  *
2585  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2586  * accordingly.
2587  *
2588  * Return: A pointer into @buf_b from where non-overlapped data starts
2589  */
2590 static unsigned char *adj_for_padding(unsigned char *buf_b,
2591                                       unsigned char *buf_a, size_t len_a)
2592 {
2593         unsigned char *p = buf_b - MAX_PADDING;
2594         unsigned char *q = buf_a + len_a - MAX_PADDING;
2595         int i;
2596
2597         for (i = MAX_PADDING; i; i--, p++, q++) {
2598                 if (*p != *q)
2599                         break;
2600         }
2601
2602         return p;
2603 }
2604
2605 /**
2606  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2607  *                             using TSC.
2608  * @buf_a: first buffer
2609  * @len_a: size of first buffer
2610  * @buf_b: second buffer
2611  * @len_b: size of second buffer
2612  * @consecutive: returns true if there is data in buf_b that is consecutive
2613  *               to buf_a
2614  *
2615  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2616  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2617  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2618  * @buf_a and @buf_b are positioned at a PSB.
2619  *
2620  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2621  * @buf_b + @len_b if there is no non-overlapped data.
2622  */
2623 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2624                                                 size_t len_a,
2625                                                 unsigned char *buf_b,
2626                                                 size_t len_b, bool *consecutive)
2627 {
2628         uint64_t tsc_a, tsc_b;
2629         unsigned char *p;
2630         size_t len, rem_a, rem_b;
2631
2632         p = intel_pt_last_psb(buf_a, len_a);
2633         if (!p)
2634                 return buf_b; /* No PSB in buf_a => no overlap */
2635
2636         len = len_a - (p - buf_a);
2637         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2638                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2639                 len_a -= len;
2640                 p = intel_pt_last_psb(buf_a, len_a);
2641                 if (!p)
2642                         return buf_b; /* No full PSB+ => assume no overlap */
2643                 len = len_a - (p - buf_a);
2644                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2645                         return buf_b; /* No TSC in buf_a => assume no overlap */
2646         }
2647
2648         while (1) {
2649                 /* Ignore PSB+ with no TSC */
2650                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2651                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2652
2653                         /* Same TSC, so buffers are consecutive */
2654                         if (!cmp && rem_b >= rem_a) {
2655                                 unsigned char *start;
2656
2657                                 *consecutive = true;
2658                                 start = buf_b + len_b - (rem_b - rem_a);
2659                                 return adj_for_padding(start, buf_a, len_a);
2660                         }
2661                         if (cmp < 0)
2662                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2663                 }
2664
2665                 if (!intel_pt_step_psb(&buf_b, &len_b))
2666                         return buf_b + len_b; /* No PSB in buf_b => no data */
2667         }
2668 }
2669
2670 /**
2671  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2672  * @buf_a: first buffer
2673  * @len_a: size of first buffer
2674  * @buf_b: second buffer
2675  * @len_b: size of second buffer
2676  * @have_tsc: can use TSC packets to detect overlap
2677  * @consecutive: returns true if there is data in buf_b that is consecutive
2678  *               to buf_a
2679  *
2680  * When trace samples or snapshots are recorded there is the possibility that
2681  * the data overlaps.  Note that, for the purposes of decoding, data is only
2682  * useful if it begins with a PSB packet.
2683  *
2684  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2685  * @buf_b + @len_b if there is no non-overlapped data.
2686  */
2687 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2688                                      unsigned char *buf_b, size_t len_b,
2689                                      bool have_tsc, bool *consecutive)
2690 {
2691         unsigned char *found;
2692
2693         /* Buffer 'b' must start at PSB so throw away everything before that */
2694         if (!intel_pt_next_psb(&buf_b, &len_b))
2695                 return buf_b + len_b; /* No PSB */
2696
2697         if (!intel_pt_next_psb(&buf_a, &len_a))
2698                 return buf_b; /* No overlap */
2699
2700         if (have_tsc) {
2701                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2702                                                   consecutive);
2703                 if (found)
2704                         return found;
2705         }
2706
2707         /*
2708          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2709          * we can ignore the first part of buffer 'a'.
2710          */
2711         while (len_b < len_a) {
2712                 if (!intel_pt_step_psb(&buf_a, &len_a))
2713                         return buf_b; /* No overlap */
2714         }
2715
2716         /* Now len_b >= len_a */
2717         while (1) {
2718                 /* Potential overlap so check the bytes */
2719                 found = memmem(buf_a, len_a, buf_b, len_a);
2720                 if (found) {
2721                         *consecutive = true;
2722                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
2723                 }
2724
2725                 /* Try again at next PSB in buffer 'a' */
2726                 if (!intel_pt_step_psb(&buf_a, &len_a))
2727                         return buf_b; /* No overlap */
2728         }
2729 }