tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <stdio.h>
  17 #include <stdbool.h>
  18 #include <errno.h>
  19 #include <linux/kernel.h>
  20 #include <linux/types.h>
  21
  22 #include "../perf.h"
  23 #include "session.h"
  24 #include "machine.h"
  25 #include "sort.h"
  26 #include "tool.h"
  27 #include "event.h"
  28 #include "evlist.h"
  29 #include "evsel.h"
  30 #include "map.h"
  31 #include "color.h"
  32 #include "util.h"
  33 #include "thread.h"
  34 #include "thread-stack.h"
  35 #include "symbol.h"
  36 #include "callchain.h"
  37 #include "dso.h"
  38 #include "debug.h"
  39 #include "auxtrace.h"
  40 #include "tsc.h"
  41 #include "intel-pt.h"
  42
  43 #include "intel-pt-decoder/intel-pt-log.h"
  44 #include "intel-pt-decoder/intel-pt-decoder.h"
  45 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  46 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  47
  48 #define MAX_TIMESTAMP (~0ULL)
  49
  50 struct intel_pt {
  51         struct auxtrace auxtrace;
  52         struct auxtrace_queues queues;
  53         struct auxtrace_heap heap;
  54         u32 auxtrace_type;
  55         struct perf_session *session;
  56         struct machine *machine;
  57         struct perf_evsel *switch_evsel;
  58         struct thread *unknown_thread;
  59         bool timeless_decoding;
  60         bool sampling_mode;
  61         bool snapshot_mode;
  62         bool per_cpu_mmaps;
  63         bool have_tsc;
  64         bool data_queued;
  65         bool est_tsc;
  66         bool sync_switch;
  67         bool mispred_all;
  68         int have_sched_switch;
  69         u32 pmu_type;
  70         u64 kernel_start;
  71         u64 switch_ip;
  72         u64 ptss_ip;
  73
  74         struct perf_tsc_conversion tc;
  75         bool cap_user_time_zero;
  76
  77         struct itrace_synth_opts synth_opts;
  78
  79         bool sample_instructions;
  80         u64 instructions_sample_type;
  81         u64 instructions_sample_period;
  82         u64 instructions_id;
  83
  84         bool sample_branches;
  85         u32 branches_filter;
  86         u64 branches_sample_type;
  87         u64 branches_id;
  88
  89         bool sample_transactions;
  90         u64 transactions_sample_type;
  91         u64 transactions_id;
  92
  93         bool synth_needs_swap;
  94
  95         u64 tsc_bit;
  96         u64 mtc_bit;
  97         u64 mtc_freq_bits;
  98         u32 tsc_ctc_ratio_n;
  99         u32 tsc_ctc_ratio_d;
 100         u64 cyc_bit;
 101         u64 noretcomp_bit;
 102         unsigned max_non_turbo_ratio;
 103 };
 104
 105 enum switch_state {
 106         INTEL_PT_SS_NOT_TRACING,
 107         INTEL_PT_SS_UNKNOWN,
 108         INTEL_PT_SS_TRACING,
 109         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 110         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 111 };
 112
 113 struct intel_pt_queue {
 114         struct intel_pt *pt;
 115         unsigned int queue_nr;
 116         struct auxtrace_buffer *buffer;
 117         void *decoder;
 118         const struct intel_pt_state *state;
 119         struct ip_callchain *chain;
 120         struct branch_stack *last_branch;
 121         struct branch_stack *last_branch_rb;
 122         size_t last_branch_pos;
 123         union perf_event *event_buf;
 124         bool on_heap;
 125         bool stop;
 126         bool step_through_buffers;
 127         bool use_buffer_pid_tid;
 128         bool sync_switch;
 129         pid_t pid, tid;
 130         int cpu;
 131         int switch_state;
 132         pid_t next_tid;
 133         struct thread *thread;
 134         bool exclude_kernel;
 135         bool have_sample;
 136         u64 time;
 137         u64 timestamp;
 138         u32 flags;
 139         u16 insn_len;
 140         u64 last_insn_cnt;
 141 };
 142
 143 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 144                           unsigned char *buf, size_t len)
 145 {
 146         struct intel_pt_pkt packet;
 147         size_t pos = 0;
 148         int ret, pkt_len, i;
 149         char desc[INTEL_PT_PKT_DESC_MAX];
 150         const char *color = PERF_COLOR_BLUE;
 151
 152         color_fprintf(stdout, color,
 153                       ". ... Intel Processor Trace data: size %zu bytes\n",
 154                       len);
 155
 156         while (len) {
 157                 ret = intel_pt_get_packet(buf, len, &packet);
 158                 if (ret > 0)
 159                         pkt_len = ret;
 160                 else
 161                         pkt_len = 1;
 162                 printf(".");
 163                 color_fprintf(stdout, color, "  %08x: ", pos);
 164                 for (i = 0; i < pkt_len; i++)
 165                         color_fprintf(stdout, color, " %02x", buf[i]);
 166                 for (; i < 16; i++)
 167                         color_fprintf(stdout, color, "   ");
 168                 if (ret > 0) {
 169                         ret = intel_pt_pkt_desc(&packet, desc,
 170                                                 INTEL_PT_PKT_DESC_MAX);
 171                         if (ret > 0)
 172                                 color_fprintf(stdout, color, " %s\n", desc);
 173                 } else {
 174                         color_fprintf(stdout, color, " Bad packet!\n");
 175                 }
 176                 pos += pkt_len;
 177                 buf += pkt_len;
 178                 len -= pkt_len;
 179         }
 180 }
 181
 182 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 183                                 size_t len)
 184 {
 185         printf(".\n");
 186         intel_pt_dump(pt, buf, len);
 187 }
 188
 189 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 190                                    struct auxtrace_buffer *b)
 191 {
 192         bool consecutive = false;
 193         void *start;
 194
 195         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 196                                       pt->have_tsc, &consecutive);
 197         if (!start)
 198                 return -EINVAL;
 199         b->use_size = b->data + b->size - start;
 200         b->use_data = start;
 201         if (b->use_size && consecutive)
 202                 b->consecutive = true;
 203         return 0;
 204 }
 205
 206 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 207                                         struct auxtrace_queue *queue,
 208                                         struct auxtrace_buffer *buffer)
 209 {
 210         if (queue->cpu == -1 && buffer->cpu != -1)
 211                 ptq->cpu = buffer->cpu;
 212
 213         ptq->pid = buffer->pid;
 214         ptq->tid = buffer->tid;
 215
 216         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 217                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 218
 219         thread__zput(ptq->thread);
 220
 221         if (ptq->tid != -1) {
 222                 if (ptq->pid != -1)
 223                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 224                                                               ptq->pid,
 225                                                               ptq->tid);
 226                 else
 227                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 228                                                            ptq->tid);
 229         }
 230 }
 231
 232 /* This function assumes data is processed sequentially only */
 233 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 234 {
 235         struct intel_pt_queue *ptq = data;
 236         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 237         struct auxtrace_queue *queue;
 238
 239         if (ptq->stop) {
 240                 b->len = 0;
 241                 return 0;
 242         }
 243
 244         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 245 next:
 246         buffer = auxtrace_buffer__next(queue, buffer);
 247         if (!buffer) {
 248                 if (old_buffer)
 249                         auxtrace_buffer__drop_data(old_buffer);
 250                 b->len = 0;
 251                 return 0;
 252         }
 253
 254         ptq->buffer = buffer;
 255
 256         if (!buffer->data) {
 257                 int fd = perf_data_file__fd(ptq->pt->session->file);
 258
 259                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 260                 if (!buffer->data)
 261                         return -ENOMEM;
 262         }
 263
 264         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 265             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 266                 return -ENOMEM;
 267
 268         if (buffer->use_data) {
 269                 b->len = buffer->use_size;
 270                 b->buf = buffer->use_data;
 271         } else {
 272                 b->len = buffer->size;
 273                 b->buf = buffer->data;
 274         }
 275         b->ref_timestamp = buffer->reference;
 276
 277         /*
 278          * If in snapshot mode and the buffer has no usable data, get next
 279          * buffer and again check overlap against old_buffer.
 280          */
 281         if (ptq->pt->snapshot_mode && !b->len)
 282                 goto next;
 283
 284         if (old_buffer)
 285                 auxtrace_buffer__drop_data(old_buffer);
 286
 287         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 288                                                       !buffer->consecutive)) {
 289                 b->consecutive = false;
 290                 b->trace_nr = buffer->buffer_nr + 1;
 291         } else {
 292                 b->consecutive = true;
 293         }
 294
 295         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 296                                         ptq->tid != buffer->tid))
 297                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 298
 299         if (ptq->step_through_buffers)
 300                 ptq->stop = true;
 301
 302         if (!b->len)
 303                 return intel_pt_get_trace(b, data);
 304
 305         return 0;
 306 }
 307
 308 struct intel_pt_cache_entry {
 309         struct auxtrace_cache_entry     entry;
 310         u64                             insn_cnt;
 311         u64                             byte_cnt;
 312         enum intel_pt_insn_op           op;
 313         enum intel_pt_insn_branch       branch;
 314         int                             length;
 315         int32_t                         rel;
 316 };
 317
 318 static int intel_pt_config_div(const char *var, const char *value, void *data)
 319 {
 320         int *d = data;
 321         long val;
 322
 323         if (!strcmp(var, "intel-pt.cache-divisor")) {
 324                 val = strtol(value, NULL, 0);
 325                 if (val > 0 && val <= INT_MAX)
 326                         *d = val;
 327         }
 328
 329         return 0;
 330 }
 331
 332 static int intel_pt_cache_divisor(void)
 333 {
 334         static int d;
 335
 336         if (d)
 337                 return d;
 338
 339         perf_config(intel_pt_config_div, &d);
 340
 341         if (!d)
 342                 d = 64;
 343
 344         return d;
 345 }
 346
 347 static unsigned int intel_pt_cache_size(struct dso *dso,
 348                                         struct machine *machine)
 349 {
 350         off_t size;
 351
 352         size = dso__data_size(dso, machine);
 353         size /= intel_pt_cache_divisor();
 354         if (size < 1000)
 355                 return 10;
 356         if (size > (1 << 21))
 357                 return 21;
 358         return 32 - __builtin_clz(size);
 359 }
 360
 361 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 362                                              struct machine *machine)
 363 {
 364         struct auxtrace_cache *c;
 365         unsigned int bits;
 366
 367         if (dso->auxtrace_cache)
 368                 return dso->auxtrace_cache;
 369
 370         bits = intel_pt_cache_size(dso, machine);
 371
 372         /* Ignoring cache creation failure */
 373         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 374
 375         dso->auxtrace_cache = c;
 376
 377         return c;
 378 }
 379
 380 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 381                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 382                               struct intel_pt_insn *intel_pt_insn)
 383 {
 384         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 385         struct intel_pt_cache_entry *e;
 386         int err;
 387
 388         if (!c)
 389                 return -ENOMEM;
 390
 391         e = auxtrace_cache__alloc_entry(c);
 392         if (!e)
 393                 return -ENOMEM;
 394
 395         e->insn_cnt = insn_cnt;
 396         e->byte_cnt = byte_cnt;
 397         e->op = intel_pt_insn->op;
 398         e->branch = intel_pt_insn->branch;
 399         e->length = intel_pt_insn->length;
 400         e->rel = intel_pt_insn->rel;
 401
 402         err = auxtrace_cache__add(c, offset, &e->entry);
 403         if (err)
 404                 auxtrace_cache__free_entry(c, e);
 405
 406         return err;
 407 }
 408
 409 static struct intel_pt_cache_entry *
 410 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 411 {
 412         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 413
 414         if (!c)
 415                 return NULL;
 416
 417         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 418 }
 419
 420 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 421                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 422                                    uint64_t to_ip, uint64_t max_insn_cnt,
 423                                    void *data)
 424 {
 425         struct intel_pt_queue *ptq = data;
 426         struct machine *machine = ptq->pt->machine;
 427         struct thread *thread;
 428         struct addr_location al;
 429         unsigned char buf[1024];
 430         size_t bufsz;
 431         ssize_t len;
 432         int x86_64;
 433         u8 cpumode;
 434         u64 offset, start_offset, start_ip;
 435         u64 insn_cnt = 0;
 436         bool one_map = true;
 437
 438         if (to_ip && *ip == to_ip)
 439                 goto out_no_cache;
 440
 441         bufsz = intel_pt_insn_max_size();
 442
 443         if (*ip >= ptq->pt->kernel_start)
 444                 cpumode = PERF_RECORD_MISC_KERNEL;
 445         else
 446                 cpumode = PERF_RECORD_MISC_USER;
 447
 448         thread = ptq->thread;
 449         if (!thread) {
 450                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 451                         return -EINVAL;
 452                 thread = ptq->pt->unknown_thread;
 453         }
 454
 455         while (1) {
 456                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 457                 if (!al.map || !al.map->dso)
 458                         return -EINVAL;
 459
 460                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 461                     dso__data_status_seen(al.map->dso,
 462                                           DSO_DATA_STATUS_SEEN_ITRACE))
 463                         return -ENOENT;
 464
 465                 offset = al.map->map_ip(al.map, *ip);
 466
 467                 if (!to_ip && one_map) {
 468                         struct intel_pt_cache_entry *e;
 469
 470                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 471                         if (e &&
 472                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 473                                 *insn_cnt_ptr = e->insn_cnt;
 474                                 *ip += e->byte_cnt;
 475                                 intel_pt_insn->op = e->op;
 476                                 intel_pt_insn->branch = e->branch;
 477                                 intel_pt_insn->length = e->length;
 478                                 intel_pt_insn->rel = e->rel;
 479                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 480                                 return 0;
 481                         }
 482                 }
 483
 484                 start_offset = offset;
 485                 start_ip = *ip;
 486
 487                 /* Load maps to ensure dso->is_64_bit has been updated */
 488                 map__load(al.map, machine->symbol_filter);
 489
 490                 x86_64 = al.map->dso->is_64_bit;
 491
 492                 while (1) {
 493                         len = dso__data_read_offset(al.map->dso, machine,
 494                                                     offset, buf, bufsz);
 495                         if (len <= 0)
 496                                 return -EINVAL;
 497
 498                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 499                                 return -EINVAL;
 500
 501                         intel_pt_log_insn(intel_pt_insn, *ip);
 502
 503                         insn_cnt += 1;
 504
 505                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 506                                 goto out;
 507
 508                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 509                                 goto out_no_cache;
 510
 511                         *ip += intel_pt_insn->length;
 512
 513                         if (to_ip && *ip == to_ip)
 514                                 goto out_no_cache;
 515
 516                         if (*ip >= al.map->end)
 517                                 break;
 518
 519                         offset += intel_pt_insn->length;
 520                 }
 521                 one_map = false;
 522         }
 523 out:
 524         *insn_cnt_ptr = insn_cnt;
 525
 526         if (!one_map)
 527                 goto out_no_cache;
 528
 529         /*
 530          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 531          * entries.
 532          */
 533         if (to_ip) {
 534                 struct intel_pt_cache_entry *e;
 535
 536                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 537                 if (e)
 538                         return 0;
 539         }
 540
 541         /* Ignore cache errors */
 542         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 543                            *ip - start_ip, intel_pt_insn);
 544
 545         return 0;
 546
 547 out_no_cache:
 548         *insn_cnt_ptr = insn_cnt;
 549         return 0;
 550 }
 551
 552 static bool intel_pt_get_config(struct intel_pt *pt,
 553                                 struct perf_event_attr *attr, u64 *config)
 554 {
 555         if (attr->type == pt->pmu_type) {
 556                 if (config)
 557                         *config = attr->config;
 558                 return true;
 559         }
 560
 561         return false;
 562 }
 563
 564 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 565 {
 566         struct perf_evsel *evsel;
 567
 568         evlist__for_each(pt->session->evlist, evsel) {
 569                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 570                     !evsel->attr.exclude_kernel)
 571                         return false;
 572         }
 573         return true;
 574 }
 575
 576 static bool intel_pt_return_compression(struct intel_pt *pt)
 577 {
 578         struct perf_evsel *evsel;
 579         u64 config;
 580
 581         if (!pt->noretcomp_bit)
 582                 return true;
 583
 584         evlist__for_each(pt->session->evlist, evsel) {
 585                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 586                     (config & pt->noretcomp_bit))
 587                         return false;
 588         }
 589         return true;
 590 }
 591
 592 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 593 {
 594         struct perf_evsel *evsel;
 595         unsigned int shift;
 596         u64 config;
 597
 598         if (!pt->mtc_freq_bits)
 599                 return 0;
 600
 601         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 602                 config >>= 1;
 603
 604         evlist__for_each(pt->session->evlist, evsel) {
 605                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 606                         return (config & pt->mtc_freq_bits) >> shift;
 607         }
 608         return 0;
 609 }
 610
 611 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 612 {
 613         struct perf_evsel *evsel;
 614         bool timeless_decoding = true;
 615         u64 config;
 616
 617         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 618                 return true;
 619
 620         evlist__for_each(pt->session->evlist, evsel) {
 621                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 622                         return true;
 623                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 624                         if (config & pt->tsc_bit)
 625                                 timeless_decoding = false;
 626                         else
 627                                 return true;
 628                 }
 629         }
 630         return timeless_decoding;
 631 }
 632
 633 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 634 {
 635         struct perf_evsel *evsel;
 636
 637         evlist__for_each(pt->session->evlist, evsel) {
 638                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 639                     !evsel->attr.exclude_kernel)
 640                         return true;
 641         }
 642         return false;
 643 }
 644
 645 static bool intel_pt_have_tsc(struct intel_pt *pt)
 646 {
 647         struct perf_evsel *evsel;
 648         bool have_tsc = false;
 649         u64 config;
 650
 651         if (!pt->tsc_bit)
 652                 return false;
 653
 654         evlist__for_each(pt->session->evlist, evsel) {
 655                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 656                         if (config & pt->tsc_bit)
 657                                 have_tsc = true;
 658                         else
 659                                 return false;
 660                 }
 661         }
 662         return have_tsc;
 663 }
 664
 665 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 666 {
 667         u64 quot, rem;
 668
 669         quot = ns / pt->tc.time_mult;
 670         rem  = ns % pt->tc.time_mult;
 671         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 672                 pt->tc.time_mult;
 673 }
 674
 675 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 676                                                    unsigned int queue_nr)
 677 {
 678         struct intel_pt_params params = { .get_trace = 0, };
 679         struct perf_env *env = pt->machine->env;
 680         struct intel_pt_queue *ptq;
 681
 682         ptq = zalloc(sizeof(struct intel_pt_queue));
 683         if (!ptq)
 684                 return NULL;
 685
 686         if (pt->synth_opts.callchain) {
 687                 size_t sz = sizeof(struct ip_callchain);
 688
 689                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 690                 ptq->chain = zalloc(sz);
 691                 if (!ptq->chain)
 692                         goto out_free;
 693         }
 694
 695         if (pt->synth_opts.last_branch) {
 696                 size_t sz = sizeof(struct branch_stack);
 697
 698                 sz += pt->synth_opts.last_branch_sz *
 699                       sizeof(struct branch_entry);
 700                 ptq->last_branch = zalloc(sz);
 701                 if (!ptq->last_branch)
 702                         goto out_free;
 703                 ptq->last_branch_rb = zalloc(sz);
 704                 if (!ptq->last_branch_rb)
 705                         goto out_free;
 706         }
 707
 708         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 709         if (!ptq->event_buf)
 710                 goto out_free;
 711
 712         ptq->pt = pt;
 713         ptq->queue_nr = queue_nr;
 714         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 715         ptq->pid = -1;
 716         ptq->tid = -1;
 717         ptq->cpu = -1;
 718         ptq->next_tid = -1;
 719
 720         params.get_trace = intel_pt_get_trace;
 721         params.walk_insn = intel_pt_walk_next_insn;
 722         params.data = ptq;
 723         params.return_compression = intel_pt_return_compression(pt);
 724         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 725         params.mtc_period = intel_pt_mtc_period(pt);
 726         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 727         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 728
 729         if (pt->synth_opts.instructions) {
 730                 if (pt->synth_opts.period) {
 731                         switch (pt->synth_opts.period_type) {
 732                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 733                                 params.period_type =
 734                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 735                                 params.period = pt->synth_opts.period;
 736                                 break;
 737                         case PERF_ITRACE_PERIOD_TICKS:
 738                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 739                                 params.period = pt->synth_opts.period;
 740                                 break;
 741                         case PERF_ITRACE_PERIOD_NANOSECS:
 742                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 743                                 params.period = intel_pt_ns_to_ticks(pt,
 744                                                         pt->synth_opts.period);
 745                                 break;
 746                         default:
 747                                 break;
 748                         }
 749                 }
 750
 751                 if (!params.period) {
 752                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 753                         params.period = 1;
 754                 }
 755         }
 756
 757         if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))
 758                 params.flags |= INTEL_PT_FUP_WITH_NLIP;
 759
 760         ptq->decoder = intel_pt_decoder_new(&params);
 761         if (!ptq->decoder)
 762                 goto out_free;
 763
 764         return ptq;
 765
 766 out_free:
 767         zfree(&ptq->event_buf);
 768         zfree(&ptq->last_branch);
 769         zfree(&ptq->last_branch_rb);
 770         zfree(&ptq->chain);
 771         free(ptq);
 772         return NULL;
 773 }
 774
 775 static void intel_pt_free_queue(void *priv)
 776 {
 777         struct intel_pt_queue *ptq = priv;
 778
 779         if (!ptq)
 780                 return;
 781         thread__zput(ptq->thread);
 782         intel_pt_decoder_free(ptq->decoder);
 783         zfree(&ptq->event_buf);
 784         zfree(&ptq->last_branch);
 785         zfree(&ptq->last_branch_rb);
 786         zfree(&ptq->chain);
 787         free(ptq);
 788 }
 789
 790 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 791                                      struct auxtrace_queue *queue)
 792 {
 793         struct intel_pt_queue *ptq = queue->priv;
 794
 795         if (queue->tid == -1 || pt->have_sched_switch) {
 796                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 797                 if (ptq->tid == -1)
 798                         ptq->pid = -1;
 799                 thread__zput(ptq->thread);
 800         }
 801
 802         if (!ptq->thread && ptq->tid != -1)
 803                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 804
 805         if (ptq->thread) {
 806                 ptq->pid = ptq->thread->pid_;
 807                 if (queue->cpu == -1)
 808                         ptq->cpu = ptq->thread->cpu;
 809         }
 810 }
 811
 812 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 813 {
 814         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 815                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 816         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 817                 if (ptq->state->to_ip)
 818                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 819                                      PERF_IP_FLAG_ASYNC |
 820                                      PERF_IP_FLAG_INTERRUPT;
 821                 else
 822                         ptq->flags = PERF_IP_FLAG_BRANCH |
 823                                      PERF_IP_FLAG_TRACE_END;
 824                 ptq->insn_len = 0;
 825         } else {
 826                 if (ptq->state->from_ip)
 827                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 828                 else
 829                         ptq->flags = PERF_IP_FLAG_BRANCH |
 830                                      PERF_IP_FLAG_TRACE_BEGIN;
 831                 if (ptq->state->flags & INTEL_PT_IN_TX)
 832                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 833                 ptq->insn_len = ptq->state->insn_len;
 834         }
 835 }
 836
 837 static int intel_pt_setup_queue(struct intel_pt *pt,
 838                                 struct auxtrace_queue *queue,
 839                                 unsigned int queue_nr)
 840 {
 841         struct intel_pt_queue *ptq = queue->priv;
 842
 843         if (list_empty(&queue->head))
 844                 return 0;
 845
 846         if (!ptq) {
 847                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 848                 if (!ptq)
 849                         return -ENOMEM;
 850                 queue->priv = ptq;
 851
 852                 if (queue->cpu != -1)
 853                         ptq->cpu = queue->cpu;
 854                 ptq->tid = queue->tid;
 855
 856                 if (pt->sampling_mode) {
 857                         if (pt->timeless_decoding)
 858                                 ptq->step_through_buffers = true;
 859                         if (pt->timeless_decoding || !pt->have_sched_switch)
 860                                 ptq->use_buffer_pid_tid = true;
 861                 }
 862
 863                 ptq->sync_switch = pt->sync_switch;
 864         }
 865
 866         if (!ptq->on_heap &&
 867             (!ptq->sync_switch ||
 868              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 869                 const struct intel_pt_state *state;
 870                 int ret;
 871
 872                 if (pt->timeless_decoding)
 873                         return 0;
 874
 875                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 876                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 877                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 878                 while (1) {
 879                         state = intel_pt_decode(ptq->decoder);
 880                         if (state->err) {
 881                                 if (state->err == INTEL_PT_ERR_NODATA) {
 882                                         intel_pt_log("queue %u has no timestamp\n",
 883                                                      queue_nr);
 884                                         return 0;
 885                                 }
 886                                 continue;
 887                         }
 888                         if (state->timestamp)
 889                                 break;
 890                 }
 891
 892                 ptq->timestamp = state->timestamp;
 893                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 894                              queue_nr, ptq->timestamp);
 895                 ptq->state = state;
 896                 ptq->have_sample = true;
 897                 intel_pt_sample_flags(ptq);
 898                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 899                 if (ret)
 900                         return ret;
 901                 ptq->on_heap = true;
 902         }
 903
 904         return 0;
 905 }
 906
 907 static int intel_pt_setup_queues(struct intel_pt *pt)
 908 {
 909         unsigned int i;
 910         int ret;
 911
 912         for (i = 0; i < pt->queues.nr_queues; i++) {
 913                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
 914                 if (ret)
 915                         return ret;
 916         }
 917         return 0;
 918 }
 919
 920 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
 921 {
 922         struct branch_stack *bs_src = ptq->last_branch_rb;
 923         struct branch_stack *bs_dst = ptq->last_branch;
 924         size_t nr = 0;
 925
 926         bs_dst->nr = bs_src->nr;
 927
 928         if (!bs_src->nr)
 929                 return;
 930
 931         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
 932         memcpy(&bs_dst->entries[0],
 933                &bs_src->entries[ptq->last_branch_pos],
 934                sizeof(struct branch_entry) * nr);
 935
 936         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
 937                 memcpy(&bs_dst->entries[nr],
 938                        &bs_src->entries[0],
 939                        sizeof(struct branch_entry) * ptq->last_branch_pos);
 940         }
 941 }
 942
 943 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
 944 {
 945         ptq->last_branch_pos = 0;
 946         ptq->last_branch_rb->nr = 0;
 947 }
 948
 949 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
 950 {
 951         const struct intel_pt_state *state = ptq->state;
 952         struct branch_stack *bs = ptq->last_branch_rb;
 953         struct branch_entry *be;
 954
 955         if (!ptq->last_branch_pos)
 956                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
 957
 958         ptq->last_branch_pos -= 1;
 959
 960         be              = &bs->entries[ptq->last_branch_pos];
 961         be->from        = state->from_ip;
 962         be->to          = state->to_ip;
 963         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
 964         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
 965         /* No support for mispredict */
 966         be->flags.mispred = ptq->pt->mispred_all;
 967
 968         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
 969                 bs->nr += 1;
 970 }
 971
 972 static int intel_pt_inject_event(union perf_event *event,
 973                                  struct perf_sample *sample, u64 type,
 974                                  bool swapped)
 975 {
 976         event->header.size = perf_event__sample_event_size(sample, type, 0);
 977         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
 978 }
 979
 980 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
 981 {
 982         int ret;
 983         struct intel_pt *pt = ptq->pt;
 984         union perf_event *event = ptq->event_buf;
 985         struct perf_sample sample = { .ip = 0, };
 986         struct dummy_branch_stack {
 987                 u64                     nr;
 988                 struct branch_entry     entries;
 989         } dummy_bs;
 990
 991         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
 992                 return 0;
 993
 994         event->sample.header.type = PERF_RECORD_SAMPLE;
 995         event->sample.header.misc = PERF_RECORD_MISC_USER;
 996         event->sample.header.size = sizeof(struct perf_event_header);
 997
 998         if (!pt->timeless_decoding)
 999                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1000
1001         sample.ip = ptq->state->from_ip;
1002         sample.pid = ptq->pid;
1003         sample.tid = ptq->tid;
1004         sample.addr = ptq->state->to_ip;
1005         sample.id = ptq->pt->branches_id;
1006         sample.stream_id = ptq->pt->branches_id;
1007         sample.period = 1;
1008         sample.cpu = ptq->cpu;
1009         sample.flags = ptq->flags;
1010         sample.insn_len = ptq->insn_len;
1011
1012         /*
1013          * perf report cannot handle events without a branch stack when using
1014          * SORT_MODE__BRANCH so make a dummy one.
1015          */
1016         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1017                 dummy_bs = (struct dummy_branch_stack){
1018                         .nr = 1,
1019                         .entries = {
1020                                 .from = sample.ip,
1021                                 .to = sample.addr,
1022                         },
1023                 };
1024                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1025         }
1026
1027         if (pt->synth_opts.inject) {
1028                 ret = intel_pt_inject_event(event, &sample,
1029                                             pt->branches_sample_type,
1030                                             pt->synth_needs_swap);
1031                 if (ret)
1032                         return ret;
1033         }
1034
1035         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1036         if (ret)
1037                 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1038                        ret);
1039
1040         return ret;
1041 }
1042
1043 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1044 {
1045         int ret;
1046         struct intel_pt *pt = ptq->pt;
1047         union perf_event *event = ptq->event_buf;
1048         struct perf_sample sample = { .ip = 0, };
1049
1050         event->sample.header.type = PERF_RECORD_SAMPLE;
1051         event->sample.header.misc = PERF_RECORD_MISC_USER;
1052         event->sample.header.size = sizeof(struct perf_event_header);
1053
1054         if (!pt->timeless_decoding)
1055                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1056
1057         sample.ip = ptq->state->from_ip;
1058         sample.pid = ptq->pid;
1059         sample.tid = ptq->tid;
1060         sample.addr = ptq->state->to_ip;
1061         sample.id = ptq->pt->instructions_id;
1062         sample.stream_id = ptq->pt->instructions_id;
1063         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1064         sample.cpu = ptq->cpu;
1065         sample.flags = ptq->flags;
1066         sample.insn_len = ptq->insn_len;
1067
1068         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1069
1070         if (pt->synth_opts.callchain) {
1071                 thread_stack__sample(ptq->thread, ptq->chain,
1072                                      pt->synth_opts.callchain_sz, sample.ip);
1073                 sample.callchain = ptq->chain;
1074         }
1075
1076         if (pt->synth_opts.last_branch) {
1077                 intel_pt_copy_last_branch_rb(ptq);
1078                 sample.branch_stack = ptq->last_branch;
1079         }
1080
1081         if (pt->synth_opts.inject) {
1082                 ret = intel_pt_inject_event(event, &sample,
1083                                             pt->instructions_sample_type,
1084                                             pt->synth_needs_swap);
1085                 if (ret)
1086                         return ret;
1087         }
1088
1089         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1090         if (ret)
1091                 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1092                        ret);
1093
1094         if (pt->synth_opts.last_branch)
1095                 intel_pt_reset_last_branch_rb(ptq);
1096
1097         return ret;
1098 }
1099
1100 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1101 {
1102         int ret;
1103         struct intel_pt *pt = ptq->pt;
1104         union perf_event *event = ptq->event_buf;
1105         struct perf_sample sample = { .ip = 0, };
1106
1107         event->sample.header.type = PERF_RECORD_SAMPLE;
1108         event->sample.header.misc = PERF_RECORD_MISC_USER;
1109         event->sample.header.size = sizeof(struct perf_event_header);
1110
1111         if (!pt->timeless_decoding)
1112                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1113
1114         sample.ip = ptq->state->from_ip;
1115         sample.pid = ptq->pid;
1116         sample.tid = ptq->tid;
1117         sample.addr = ptq->state->to_ip;
1118         sample.id = ptq->pt->transactions_id;
1119         sample.stream_id = ptq->pt->transactions_id;
1120         sample.period = 1;
1121         sample.cpu = ptq->cpu;
1122         sample.flags = ptq->flags;
1123         sample.insn_len = ptq->insn_len;
1124
1125         if (pt->synth_opts.callchain) {
1126                 thread_stack__sample(ptq->thread, ptq->chain,
1127                                      pt->synth_opts.callchain_sz, sample.ip);
1128                 sample.callchain = ptq->chain;
1129         }
1130
1131         if (pt->synth_opts.last_branch) {
1132                 intel_pt_copy_last_branch_rb(ptq);
1133                 sample.branch_stack = ptq->last_branch;
1134         }
1135
1136         if (pt->synth_opts.inject) {
1137                 ret = intel_pt_inject_event(event, &sample,
1138                                             pt->transactions_sample_type,
1139                                             pt->synth_needs_swap);
1140                 if (ret)
1141                         return ret;
1142         }
1143
1144         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1145         if (ret)
1146                 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1147                        ret);
1148
1149         if (pt->synth_opts.last_branch)
1150                 intel_pt_reset_last_branch_rb(ptq);
1151
1152         return ret;
1153 }
1154
1155 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1156                                 pid_t pid, pid_t tid, u64 ip)
1157 {
1158         union perf_event event;
1159         char msg[MAX_AUXTRACE_ERROR_MSG];
1160         int err;
1161
1162         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1163
1164         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1165                              code, cpu, pid, tid, ip, msg);
1166
1167         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1168         if (err)
1169                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1170                        err);
1171
1172         return err;
1173 }
1174
1175 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1176 {
1177         struct auxtrace_queue *queue;
1178         pid_t tid = ptq->next_tid;
1179         int err;
1180
1181         if (tid == -1)
1182                 return 0;
1183
1184         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1185
1186         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1187
1188         queue = &pt->queues.queue_array[ptq->queue_nr];
1189         intel_pt_set_pid_tid_cpu(pt, queue);
1190
1191         ptq->next_tid = -1;
1192
1193         return err;
1194 }
1195
1196 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1197 {
1198         struct intel_pt *pt = ptq->pt;
1199
1200         return ip == pt->switch_ip &&
1201                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1202                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1203                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1204 }
1205
1206 static int intel_pt_sample(struct intel_pt_queue *ptq)
1207 {
1208         const struct intel_pt_state *state = ptq->state;
1209         struct intel_pt *pt = ptq->pt;
1210         int err;
1211
1212         if (!ptq->have_sample)
1213                 return 0;
1214
1215         ptq->have_sample = false;
1216
1217         if (pt->sample_instructions &&
1218             (state->type & INTEL_PT_INSTRUCTION)) {
1219                 err = intel_pt_synth_instruction_sample(ptq);
1220                 if (err)
1221                         return err;
1222         }
1223
1224         if (pt->sample_transactions &&
1225             (state->type & INTEL_PT_TRANSACTION)) {
1226                 err = intel_pt_synth_transaction_sample(ptq);
1227                 if (err)
1228                         return err;
1229         }
1230
1231         if (!(state->type & INTEL_PT_BRANCH))
1232                 return 0;
1233
1234         if (pt->synth_opts.callchain)
1235                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1236                                     state->to_ip, ptq->insn_len,
1237                                     state->trace_nr);
1238         else
1239                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1240
1241         if (pt->sample_branches) {
1242                 err = intel_pt_synth_branch_sample(ptq);
1243                 if (err)
1244                         return err;
1245         }
1246
1247         if (pt->synth_opts.last_branch)
1248                 intel_pt_update_last_branch_rb(ptq);
1249
1250         if (!ptq->sync_switch)
1251                 return 0;
1252
1253         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1254                 switch (ptq->switch_state) {
1255                 case INTEL_PT_SS_NOT_TRACING:
1256                 case INTEL_PT_SS_UNKNOWN:
1257                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1258                         err = intel_pt_next_tid(pt, ptq);
1259                         if (err)
1260                                 return err;
1261                         ptq->switch_state = INTEL_PT_SS_TRACING;
1262                         break;
1263                 default:
1264                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1265                         return 1;
1266                 }
1267         } else if (!state->to_ip) {
1268                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1269         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1270                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1271         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1272                    state->to_ip == pt->ptss_ip &&
1273                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1274                 ptq->switch_state = INTEL_PT_SS_TRACING;
1275         }
1276
1277         return 0;
1278 }
1279
1280 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1281 {
1282         struct machine *machine = pt->machine;
1283         struct map *map;
1284         struct symbol *sym, *start;
1285         u64 ip, switch_ip = 0;
1286         const char *ptss;
1287
1288         if (ptss_ip)
1289                 *ptss_ip = 0;
1290
1291         map = machine__kernel_map(machine);
1292         if (!map)
1293                 return 0;
1294
1295         if (map__load(map, machine->symbol_filter))
1296                 return 0;
1297
1298         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1299
1300         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1301                 if (sym->binding == STB_GLOBAL &&
1302                     !strcmp(sym->name, "__switch_to")) {
1303                         ip = map->unmap_ip(map, sym->start);
1304                         if (ip >= map->start && ip < map->end) {
1305                                 switch_ip = ip;
1306                                 break;
1307                         }
1308                 }
1309         }
1310
1311         if (!switch_ip || !ptss_ip)
1312                 return 0;
1313
1314         if (pt->have_sched_switch == 1)
1315                 ptss = "perf_trace_sched_switch";
1316         else
1317                 ptss = "__perf_event_task_sched_out";
1318
1319         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1320                 if (!strcmp(sym->name, ptss)) {
1321                         ip = map->unmap_ip(map, sym->start);
1322                         if (ip >= map->start && ip < map->end) {
1323                                 *ptss_ip = ip;
1324                                 break;
1325                         }
1326                 }
1327         }
1328
1329         return switch_ip;
1330 }
1331
1332 static void intel_pt_enable_sync_switch(struct intel_pt *pt)
1333 {
1334         unsigned int i;
1335
1336         pt->sync_switch = true;
1337
1338         for (i = 0; i < pt->queues.nr_queues; i++) {
1339                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1340                 struct intel_pt_queue *ptq = queue->priv;
1341
1342                 if (ptq)
1343                         ptq->sync_switch = true;
1344         }
1345 }
1346
1347 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1348 {
1349         const struct intel_pt_state *state = ptq->state;
1350         struct intel_pt *pt = ptq->pt;
1351         int err;
1352
1353         if (!pt->kernel_start) {
1354                 pt->kernel_start = machine__kernel_start(pt->machine);
1355                 if (pt->per_cpu_mmaps &&
1356                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1357                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1358                     !pt->sampling_mode) {
1359                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1360                         if (pt->switch_ip) {
1361                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1362                                              pt->switch_ip, pt->ptss_ip);
1363                                 intel_pt_enable_sync_switch(pt);
1364                         }
1365                 }
1366         }
1367
1368         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1369                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1370         while (1) {
1371                 err = intel_pt_sample(ptq);
1372                 if (err)
1373                         return err;
1374
1375                 state = intel_pt_decode(ptq->decoder);
1376                 if (state->err) {
1377                         if (state->err == INTEL_PT_ERR_NODATA)
1378                                 return 1;
1379                         if (ptq->sync_switch &&
1380                             state->from_ip >= pt->kernel_start) {
1381                                 ptq->sync_switch = false;
1382                                 intel_pt_next_tid(pt, ptq);
1383                         }
1384                         if (pt->synth_opts.errors) {
1385                                 err = intel_pt_synth_error(pt, state->err,
1386                                                            ptq->cpu, ptq->pid,
1387                                                            ptq->tid,
1388                                                            state->from_ip);
1389                                 if (err)
1390                                         return err;
1391                         }
1392                         continue;
1393                 }
1394
1395                 ptq->state = state;
1396                 ptq->have_sample = true;
1397                 intel_pt_sample_flags(ptq);
1398
1399                 /* Use estimated TSC upon return to user space */
1400                 if (pt->est_tsc &&
1401                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1402                     state->to_ip && state->to_ip < pt->kernel_start) {
1403                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1404                                      state->timestamp, state->est_timestamp);
1405                         ptq->timestamp = state->est_timestamp;
1406                 /* Use estimated TSC in unknown switch state */
1407                 } else if (ptq->sync_switch &&
1408                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1409                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1410                            ptq->next_tid == -1) {
1411                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1412                                      state->timestamp, state->est_timestamp);
1413                         ptq->timestamp = state->est_timestamp;
1414                 } else if (state->timestamp > ptq->timestamp) {
1415                         ptq->timestamp = state->timestamp;
1416                 }
1417
1418                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1419                         *timestamp = ptq->timestamp;
1420                         return 0;
1421                 }
1422         }
1423         return 0;
1424 }
1425
1426 static inline int intel_pt_update_queues(struct intel_pt *pt)
1427 {
1428         if (pt->queues.new_data) {
1429                 pt->queues.new_data = false;
1430                 return intel_pt_setup_queues(pt);
1431         }
1432         return 0;
1433 }
1434
1435 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1436 {
1437         unsigned int queue_nr;
1438         u64 ts;
1439         int ret;
1440
1441         while (1) {
1442                 struct auxtrace_queue *queue;
1443                 struct intel_pt_queue *ptq;
1444
1445                 if (!pt->heap.heap_cnt)
1446                         return 0;
1447
1448                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1449                         return 0;
1450
1451                 queue_nr = pt->heap.heap_array[0].queue_nr;
1452                 queue = &pt->queues.queue_array[queue_nr];
1453                 ptq = queue->priv;
1454
1455                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1456                              queue_nr, pt->heap.heap_array[0].ordinal,
1457                              timestamp);
1458
1459                 auxtrace_heap__pop(&pt->heap);
1460
1461                 if (pt->heap.heap_cnt) {
1462                         ts = pt->heap.heap_array[0].ordinal + 1;
1463                         if (ts > timestamp)
1464                                 ts = timestamp;
1465                 } else {
1466                         ts = timestamp;
1467                 }
1468
1469                 intel_pt_set_pid_tid_cpu(pt, queue);
1470
1471                 ret = intel_pt_run_decoder(ptq, &ts);
1472
1473                 if (ret < 0) {
1474                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1475                         return ret;
1476                 }
1477
1478                 if (!ret) {
1479                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1480                         if (ret < 0)
1481                                 return ret;
1482                 } else {
1483                         ptq->on_heap = false;
1484                 }
1485         }
1486
1487         return 0;
1488 }
1489
1490 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1491                                             u64 time_)
1492 {
1493         struct auxtrace_queues *queues = &pt->queues;
1494         unsigned int i;
1495         u64 ts = 0;
1496
1497         for (i = 0; i < queues->nr_queues; i++) {
1498                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1499                 struct intel_pt_queue *ptq = queue->priv;
1500
1501                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1502                         ptq->time = time_;
1503                         intel_pt_set_pid_tid_cpu(pt, queue);
1504                         intel_pt_run_decoder(ptq, &ts);
1505                 }
1506         }
1507         return 0;
1508 }
1509
1510 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1511 {
1512         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1513                                     sample->pid, sample->tid, 0);
1514 }
1515
1516 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1517 {
1518         unsigned i, j;
1519
1520         if (cpu < 0 || !pt->queues.nr_queues)
1521                 return NULL;
1522
1523         if ((unsigned)cpu >= pt->queues.nr_queues)
1524                 i = pt->queues.nr_queues - 1;
1525         else
1526                 i = cpu;
1527
1528         if (pt->queues.queue_array[i].cpu == cpu)
1529                 return pt->queues.queue_array[i].priv;
1530
1531         for (j = 0; i > 0; j++) {
1532                 if (pt->queues.queue_array[--i].cpu == cpu)
1533                         return pt->queues.queue_array[i].priv;
1534         }
1535
1536         for (; j < pt->queues.nr_queues; j++) {
1537                 if (pt->queues.queue_array[j].cpu == cpu)
1538                         return pt->queues.queue_array[j].priv;
1539         }
1540
1541         return NULL;
1542 }
1543
1544 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1545                                 u64 timestamp)
1546 {
1547         struct intel_pt_queue *ptq;
1548         int err;
1549
1550         if (!pt->sync_switch)
1551                 return 1;
1552
1553         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1554         if (!ptq || !ptq->sync_switch)
1555                 return 1;
1556
1557         switch (ptq->switch_state) {
1558         case INTEL_PT_SS_NOT_TRACING:
1559                 ptq->next_tid = -1;
1560                 break;
1561         case INTEL_PT_SS_UNKNOWN:
1562         case INTEL_PT_SS_TRACING:
1563                 ptq->next_tid = tid;
1564                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1565                 return 0;
1566         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1567                 if (!ptq->on_heap) {
1568                         ptq->timestamp = perf_time_to_tsc(timestamp,
1569                                                           &pt->tc);
1570                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1571                                                  ptq->timestamp);
1572                         if (err)
1573                                 return err;
1574                         ptq->on_heap = true;
1575                 }
1576                 ptq->switch_state = INTEL_PT_SS_TRACING;
1577                 break;
1578         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1579                 ptq->next_tid = tid;
1580                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1581                 break;
1582         default:
1583                 break;
1584         }
1585
1586         return 1;
1587 }
1588
1589 static int intel_pt_process_switch(struct intel_pt *pt,
1590                                    struct perf_sample *sample)
1591 {
1592         struct perf_evsel *evsel;
1593         pid_t tid;
1594         int cpu, ret;
1595
1596         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1597         if (evsel != pt->switch_evsel)
1598                 return 0;
1599
1600         tid = perf_evsel__intval(evsel, sample, "next_pid");
1601         cpu = sample->cpu;
1602
1603         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1604                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1605                      &pt->tc));
1606
1607         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1608         if (ret <= 0)
1609                 return ret;
1610
1611         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1612 }
1613
1614 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1615                                    struct perf_sample *sample)
1616 {
1617         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1618         pid_t pid, tid;
1619         int cpu, ret;
1620
1621         cpu = sample->cpu;
1622
1623         if (pt->have_sched_switch == 3) {
1624                 if (!out)
1625                         return 0;
1626                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1627                         pr_err("Expecting CPU-wide context switch event\n");
1628                         return -EINVAL;
1629                 }
1630                 pid = event->context_switch.next_prev_pid;
1631                 tid = event->context_switch.next_prev_tid;
1632         } else {
1633                 if (out)
1634                         return 0;
1635                 pid = sample->pid;
1636                 tid = sample->tid;
1637         }
1638
1639         if (tid == -1)
1640                 intel_pt_log("context_switch event has no tid\n");
1641
1642         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1643                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1644                      &pt->tc));
1645
1646         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1647         if (ret <= 0)
1648                 return ret;
1649
1650         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1651 }
1652
1653 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1654                                          union perf_event *event,
1655                                          struct perf_sample *sample)
1656 {
1657         if (!pt->per_cpu_mmaps)
1658                 return 0;
1659
1660         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1661                      sample->cpu, event->itrace_start.pid,
1662                      event->itrace_start.tid, sample->time,
1663                      perf_time_to_tsc(sample->time, &pt->tc));
1664
1665         return machine__set_current_tid(pt->machine, sample->cpu,
1666                                         event->itrace_start.pid,
1667                                         event->itrace_start.tid);
1668 }
1669
1670 static int intel_pt_process_event(struct perf_session *session,
1671                                   union perf_event *event,
1672                                   struct perf_sample *sample,
1673                                   struct perf_tool *tool)
1674 {
1675         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1676                                            auxtrace);
1677         u64 timestamp;
1678         int err = 0;
1679
1680         if (dump_trace)
1681                 return 0;
1682
1683         if (!tool->ordered_events) {
1684                 pr_err("Intel Processor Trace requires ordered events\n");
1685                 return -EINVAL;
1686         }
1687
1688         if (sample->time && sample->time != (u64)-1)
1689                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1690         else
1691                 timestamp = 0;
1692
1693         if (timestamp || pt->timeless_decoding) {
1694                 err = intel_pt_update_queues(pt);
1695                 if (err)
1696                         return err;
1697         }
1698
1699         if (pt->timeless_decoding) {
1700                 if (event->header.type == PERF_RECORD_EXIT) {
1701                         err = intel_pt_process_timeless_queues(pt,
1702                                                                event->fork.tid,
1703                                                                sample->time);
1704                 }
1705         } else if (timestamp) {
1706                 err = intel_pt_process_queues(pt, timestamp);
1707         }
1708         if (err)
1709                 return err;
1710
1711         if (event->header.type == PERF_RECORD_AUX &&
1712             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1713             pt->synth_opts.errors) {
1714                 err = intel_pt_lost(pt, sample);
1715                 if (err)
1716                         return err;
1717         }
1718
1719         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1720                 err = intel_pt_process_switch(pt, sample);
1721         else if (event->header.type == PERF_RECORD_ITRACE_START)
1722                 err = intel_pt_process_itrace_start(pt, event, sample);
1723         else if (event->header.type == PERF_RECORD_SWITCH ||
1724                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1725                 err = intel_pt_context_switch(pt, event, sample);
1726
1727         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1728                      perf_event__name(event->header.type), event->header.type,
1729                      sample->cpu, sample->time, timestamp);
1730
1731         return err;
1732 }
1733
1734 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1735 {
1736         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1737                                            auxtrace);
1738         int ret;
1739
1740         if (dump_trace)
1741                 return 0;
1742
1743         if (!tool->ordered_events)
1744                 return -EINVAL;
1745
1746         ret = intel_pt_update_queues(pt);
1747         if (ret < 0)
1748                 return ret;
1749
1750         if (pt->timeless_decoding)
1751                 return intel_pt_process_timeless_queues(pt, -1,
1752                                                         MAX_TIMESTAMP - 1);
1753
1754         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1755 }
1756
1757 static void intel_pt_free_events(struct perf_session *session)
1758 {
1759         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1760                                            auxtrace);
1761         struct auxtrace_queues *queues = &pt->queues;
1762         unsigned int i;
1763
1764         for (i = 0; i < queues->nr_queues; i++) {
1765                 intel_pt_free_queue(queues->queue_array[i].priv);
1766                 queues->queue_array[i].priv = NULL;
1767         }
1768         intel_pt_log_disable();
1769         auxtrace_queues__free(queues);
1770 }
1771
1772 static void intel_pt_free(struct perf_session *session)
1773 {
1774         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1775                                            auxtrace);
1776
1777         auxtrace_heap__free(&pt->heap);
1778         intel_pt_free_events(session);
1779         session->auxtrace = NULL;
1780         thread__delete(pt->unknown_thread);
1781         free(pt);
1782 }
1783
1784 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1785                                            union perf_event *event,
1786                                            struct perf_tool *tool __maybe_unused)
1787 {
1788         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1789                                            auxtrace);
1790
1791         if (pt->sampling_mode)
1792                 return 0;
1793
1794         if (!pt->data_queued) {
1795                 struct auxtrace_buffer *buffer;
1796                 off_t data_offset;
1797                 int fd = perf_data_file__fd(session->file);
1798                 int err;
1799
1800                 if (perf_data_file__is_pipe(session->file)) {
1801                         data_offset = 0;
1802                 } else {
1803                         data_offset = lseek(fd, 0, SEEK_CUR);
1804                         if (data_offset == -1)
1805                                 return -errno;
1806                 }
1807
1808                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1809                                                  data_offset, &buffer);
1810                 if (err)
1811                         return err;
1812
1813                 /* Dump here now we have copied a piped trace out of the pipe */
1814                 if (dump_trace) {
1815                         if (auxtrace_buffer__get_data(buffer, fd)) {
1816                                 intel_pt_dump_event(pt, buffer->data,
1817                                                     buffer->size);
1818                                 auxtrace_buffer__put_data(buffer);
1819                         }
1820                 }
1821         }
1822
1823         return 0;
1824 }
1825
1826 struct intel_pt_synth {
1827         struct perf_tool dummy_tool;
1828         struct perf_session *session;
1829 };
1830
1831 static int intel_pt_event_synth(struct perf_tool *tool,
1832                                 union perf_event *event,
1833                                 struct perf_sample *sample __maybe_unused,
1834                                 struct machine *machine __maybe_unused)
1835 {
1836         struct intel_pt_synth *intel_pt_synth =
1837                         container_of(tool, struct intel_pt_synth, dummy_tool);
1838
1839         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1840                                                  NULL);
1841 }
1842
1843 static int intel_pt_synth_event(struct perf_session *session,
1844                                 struct perf_event_attr *attr, u64 id)
1845 {
1846         struct intel_pt_synth intel_pt_synth;
1847
1848         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1849         intel_pt_synth.session = session;
1850
1851         return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1852                                            &id, intel_pt_event_synth);
1853 }
1854
1855 static int intel_pt_synth_events(struct intel_pt *pt,
1856                                  struct perf_session *session)
1857 {
1858         struct perf_evlist *evlist = session->evlist;
1859         struct perf_evsel *evsel;
1860         struct perf_event_attr attr;
1861         bool found = false;
1862         u64 id;
1863         int err;
1864
1865         evlist__for_each(evlist, evsel) {
1866                 if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1867                         found = true;
1868                         break;
1869                 }
1870         }
1871
1872         if (!found) {
1873                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1874                 return 0;
1875         }
1876
1877         memset(&attr, 0, sizeof(struct perf_event_attr));
1878         attr.size = sizeof(struct perf_event_attr);
1879         attr.type = PERF_TYPE_HARDWARE;
1880         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1881         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1882                             PERF_SAMPLE_PERIOD;
1883         if (pt->timeless_decoding)
1884                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1885         else
1886                 attr.sample_type |= PERF_SAMPLE_TIME;
1887         if (!pt->per_cpu_mmaps)
1888                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1889         attr.exclude_user = evsel->attr.exclude_user;
1890         attr.exclude_kernel = evsel->attr.exclude_kernel;
1891         attr.exclude_hv = evsel->attr.exclude_hv;
1892         attr.exclude_host = evsel->attr.exclude_host;
1893         attr.exclude_guest = evsel->attr.exclude_guest;
1894         attr.sample_id_all = evsel->attr.sample_id_all;
1895         attr.read_format = evsel->attr.read_format;
1896
1897         id = evsel->id[0] + 1000000000;
1898         if (!id)
1899                 id = 1;
1900
1901         if (pt->synth_opts.instructions) {
1902                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1903                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
1904                         attr.sample_period =
1905                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
1906                 else
1907                         attr.sample_period = pt->synth_opts.period;
1908                 pt->instructions_sample_period = attr.sample_period;
1909                 if (pt->synth_opts.callchain)
1910                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1911                 if (pt->synth_opts.last_branch)
1912                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1913                 pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1914                          id, (u64)attr.sample_type);
1915                 err = intel_pt_synth_event(session, &attr, id);
1916                 if (err) {
1917                         pr_err("%s: failed to synthesize 'instructions' event type\n",
1918                                __func__);
1919                         return err;
1920                 }
1921                 pt->sample_instructions = true;
1922                 pt->instructions_sample_type = attr.sample_type;
1923                 pt->instructions_id = id;
1924                 id += 1;
1925         }
1926
1927         if (pt->synth_opts.transactions) {
1928                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1929                 attr.sample_period = 1;
1930                 if (pt->synth_opts.callchain)
1931                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1932                 if (pt->synth_opts.last_branch)
1933                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1934                 pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1935                          id, (u64)attr.sample_type);
1936                 err = intel_pt_synth_event(session, &attr, id);
1937                 if (err) {
1938                         pr_err("%s: failed to synthesize 'transactions' event type\n",
1939                                __func__);
1940                         return err;
1941                 }
1942                 pt->sample_transactions = true;
1943                 pt->transactions_id = id;
1944                 id += 1;
1945                 evlist__for_each(evlist, evsel) {
1946                         if (evsel->id && evsel->id[0] == pt->transactions_id) {
1947                                 if (evsel->name)
1948                                         zfree(&evsel->name);
1949                                 evsel->name = strdup("transactions");
1950                                 break;
1951                         }
1952                 }
1953         }
1954
1955         if (pt->synth_opts.branches) {
1956                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1957                 attr.sample_period = 1;
1958                 attr.sample_type |= PERF_SAMPLE_ADDR;
1959                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
1960                 attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
1961                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1962                          id, (u64)attr.sample_type);
1963                 err = intel_pt_synth_event(session, &attr, id);
1964                 if (err) {
1965                         pr_err("%s: failed to synthesize 'branches' event type\n",
1966                                __func__);
1967                         return err;
1968                 }
1969                 pt->sample_branches = true;
1970                 pt->branches_sample_type = attr.sample_type;
1971                 pt->branches_id = id;
1972         }
1973
1974         pt->synth_needs_swap = evsel->needs_swap;
1975
1976         return 0;
1977 }
1978
1979 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
1980 {
1981         struct perf_evsel *evsel;
1982
1983         evlist__for_each_reverse(evlist, evsel) {
1984                 const char *name = perf_evsel__name(evsel);
1985
1986                 if (!strcmp(name, "sched:sched_switch"))
1987                         return evsel;
1988         }
1989
1990         return NULL;
1991 }
1992
1993 static bool intel_pt_find_switch(struct perf_evlist *evlist)
1994 {
1995         struct perf_evsel *evsel;
1996
1997         evlist__for_each(evlist, evsel) {
1998                 if (evsel->attr.context_switch)
1999                         return true;
2000         }
2001
2002         return false;
2003 }
2004
2005 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2006 {
2007         struct intel_pt *pt = data;
2008
2009         if (!strcmp(var, "intel-pt.mispred-all"))
2010                 pt->mispred_all = perf_config_bool(var, value);
2011
2012         return 0;
2013 }
2014
2015 static const char * const intel_pt_info_fmts[] = {
2016         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
2017         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
2018         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
2019         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
2020         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
2021         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
2022         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
2023         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
2024         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
2025         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
2026         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
2027         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
2028         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
2029         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2030 };
2031
2032 static void intel_pt_print_info(u64 *arr, int start, int finish)
2033 {
2034         int i;
2035
2036         if (!dump_trace)
2037                 return;
2038
2039         for (i = start; i <= finish; i++)
2040                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2041 }
2042
2043 int intel_pt_process_auxtrace_info(union perf_event *event,
2044                                    struct perf_session *session)
2045 {
2046         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2047         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2048         struct intel_pt *pt;
2049         int err;
2050
2051         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2052                                         min_sz)
2053                 return -EINVAL;
2054
2055         pt = zalloc(sizeof(struct intel_pt));
2056         if (!pt)
2057                 return -ENOMEM;
2058
2059         perf_config(intel_pt_perf_config, pt);
2060
2061         err = auxtrace_queues__init(&pt->queues);
2062         if (err)
2063                 goto err_free;
2064
2065         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2066
2067         pt->session = session;
2068         pt->machine = &session->machines.host; /* No kvm support */
2069         pt->auxtrace_type = auxtrace_info->type;
2070         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2071         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2072         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2073         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2074         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2075         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2076         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2077         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2078         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2079         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2080         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2081                             INTEL_PT_PER_CPU_MMAPS);
2082
2083         if (auxtrace_info->header.size >= sizeof(struct auxtrace_info_event) +
2084                                         (sizeof(u64) * INTEL_PT_CYC_BIT)) {
2085                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2086                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2087                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2088                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2089                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2090                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2091                                     INTEL_PT_CYC_BIT);
2092         }
2093
2094         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2095         pt->have_tsc = intel_pt_have_tsc(pt);
2096         pt->sampling_mode = false;
2097         pt->est_tsc = !pt->timeless_decoding;
2098
2099         pt->unknown_thread = thread__new(999999999, 999999999);
2100         if (!pt->unknown_thread) {
2101                 err = -ENOMEM;
2102                 goto err_free_queues;
2103         }
2104         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2105         if (err)
2106                 goto err_delete_thread;
2107         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2108                 err = -ENOMEM;
2109                 goto err_delete_thread;
2110         }
2111
2112         pt->auxtrace.process_event = intel_pt_process_event;
2113         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2114         pt->auxtrace.flush_events = intel_pt_flush;
2115         pt->auxtrace.free_events = intel_pt_free_events;
2116         pt->auxtrace.free = intel_pt_free;
2117         session->auxtrace = &pt->auxtrace;
2118
2119         if (dump_trace)
2120                 return 0;
2121
2122         if (pt->have_sched_switch == 1) {
2123                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2124                 if (!pt->switch_evsel) {
2125                         pr_err("%s: missing sched_switch event\n", __func__);
2126                         goto err_delete_thread;
2127                 }
2128         } else if (pt->have_sched_switch == 2 &&
2129                    !intel_pt_find_switch(session->evlist)) {
2130                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2131                 goto err_delete_thread;
2132         }
2133
2134         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2135                 pt->synth_opts = *session->itrace_synth_opts;
2136         } else {
2137                 itrace_synth_opts__set_default(&pt->synth_opts);
2138                 if (use_browser != -1) {
2139                         pt->synth_opts.branches = false;
2140                         pt->synth_opts.callchain = true;
2141                 }
2142         }
2143
2144         if (pt->synth_opts.log)
2145                 intel_pt_log_enable();
2146
2147         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2148         if (pt->tc.time_mult) {
2149                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2150
2151                 pt->max_non_turbo_ratio = (tsc_freq + 50000000) / 100000000;
2152                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2153                 intel_pt_log("Maximum non-turbo ratio %u\n",
2154                              pt->max_non_turbo_ratio);
2155         }
2156
2157         if (pt->synth_opts.calls)
2158                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2159                                        PERF_IP_FLAG_TRACE_END;
2160         if (pt->synth_opts.returns)
2161                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2162                                        PERF_IP_FLAG_TRACE_BEGIN;
2163
2164         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2165                 symbol_conf.use_callchain = true;
2166                 if (callchain_register_param(&callchain_param) < 0) {
2167                         symbol_conf.use_callchain = false;
2168                         pt->synth_opts.callchain = false;
2169                 }
2170         }
2171
2172         err = intel_pt_synth_events(pt, session);
2173         if (err)
2174                 goto err_delete_thread;
2175
2176         err = auxtrace_queues__process_index(&pt->queues, session);
2177         if (err)
2178                 goto err_delete_thread;
2179
2180         if (pt->queues.populated)
2181                 pt->data_queued = true;
2182
2183         if (pt->timeless_decoding)
2184                 pr_debug2("Intel PT decoding without timestamps\n");
2185
2186         return 0;
2187
2188 err_delete_thread:
2189         thread__delete(pt->unknown_thread);
2190 err_free_queues:
2191         intel_pt_log_disable();
2192         auxtrace_queues__free(&pt->queues);
2193         session->auxtrace = NULL;
2194 err_free:
2195         free(pt);
2196         return err;
2197 }