drivers/hwtracing/coresight/coresight-tmc-etf.c

   1 /*
   2  * Copyright(C) 2016 Linaro Limited. All rights reserved.
   3  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/circ_buf.h>
  19 #include <linux/coresight.h>
  20 #include <linux/perf_event.h>
  21 #include <linux/slab.h>
  22 #include "coresight-priv.h"
  23 #include "coresight-tmc.h"
  24
  25 static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
  26 {
  27         CS_UNLOCK(drvdata->base);
  28
  29         /* Wait for TMCSReady bit to be set */
  30         tmc_wait_for_tmcready(drvdata);
  31
  32         writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
  33         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
  34                        TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
  35                        TMC_FFCR_TRIGON_TRIGIN,
  36                        drvdata->base + TMC_FFCR);
  37
  38         writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
  39         tmc_enable_hw(drvdata);
  40
  41         CS_LOCK(drvdata->base);
  42 }
  43
  44 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
  45 {
  46         char *bufp;
  47         u32 read_data;
  48         int i;
  49
  50         bufp = drvdata->buf;
  51         drvdata->len = 0;
  52         while (1) {
  53                 for (i = 0; i < drvdata->memwidth; i++) {
  54                         read_data = readl_relaxed(drvdata->base + TMC_RRD);
  55                         if (read_data == 0xFFFFFFFF)
  56                                 return;
  57                         memcpy(bufp, &read_data, 4);
  58                         bufp += 4;
  59                         drvdata->len += 4;
  60                 }
  61         }
  62 }
  63
  64 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
  65 {
  66         CS_UNLOCK(drvdata->base);
  67
  68         tmc_flush_and_stop(drvdata);
  69         /*
  70          * When operating in sysFS mode the content of the buffer needs to be
  71          * read before the TMC is disabled.
  72          */
  73         if (local_read(&drvdata->mode) == CS_MODE_SYSFS)
  74                 tmc_etb_dump_hw(drvdata);
  75         tmc_disable_hw(drvdata);
  76
  77         CS_LOCK(drvdata->base);
  78 }
  79
  80 static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
  81 {
  82         CS_UNLOCK(drvdata->base);
  83
  84         /* Wait for TMCSReady bit to be set */
  85         tmc_wait_for_tmcready(drvdata);
  86
  87         writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
  88         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
  89                        drvdata->base + TMC_FFCR);
  90         writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
  91         tmc_enable_hw(drvdata);
  92
  93         CS_LOCK(drvdata->base);
  94 }
  95
  96 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
  97 {
  98         CS_UNLOCK(drvdata->base);
  99
 100         tmc_flush_and_stop(drvdata);
 101         tmc_disable_hw(drvdata);
 102
 103         CS_LOCK(drvdata->base);
 104 }
 105
 106 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev, u32 mode)
 107 {
 108         int ret = 0;
 109         bool used = false;
 110         char *buf = NULL;
 111         long val;
 112         unsigned long flags;
 113         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 114
 115          /* This shouldn't be happening */
 116         if (WARN_ON(mode != CS_MODE_SYSFS))
 117                 return -EINVAL;
 118
 119         /*
 120          * If we don't have a buffer release the lock and allocate memory.
 121          * Otherwise keep the lock and move along.
 122          */
 123         spin_lock_irqsave(&drvdata->spinlock, flags);
 124         if (!drvdata->buf) {
 125                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 126
 127                 /* Allocating the memory here while outside of the spinlock */
 128                 buf = kzalloc(drvdata->size, GFP_KERNEL);
 129                 if (!buf)
 130                         return -ENOMEM;
 131
 132                 /* Let's try again */
 133                 spin_lock_irqsave(&drvdata->spinlock, flags);
 134         }
 135
 136         if (drvdata->reading) {
 137                 ret = -EBUSY;
 138                 goto out;
 139         }
 140
 141         val = local_xchg(&drvdata->mode, mode);
 142         /*
 143          * In sysFS mode we can have multiple writers per sink.  Since this
 144          * sink is already enabled no memory is needed and the HW need not be
 145          * touched.
 146          */
 147         if (val == CS_MODE_SYSFS)
 148                 goto out;
 149
 150         /*
 151          * If drvdata::buf isn't NULL, memory was allocated for a previous
 152          * trace run but wasn't read.  If so simply zero-out the memory.
 153          * Otherwise use the memory allocated above.
 154          *
 155          * The memory is freed when users read the buffer using the
 156          * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
 157          * details.
 158          */
 159         if (drvdata->buf) {
 160                 memset(drvdata->buf, 0, drvdata->size);
 161         } else {
 162                 used = true;
 163                 drvdata->buf = buf;
 164         }
 165
 166         tmc_etb_enable_hw(drvdata);
 167 out:
 168         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 169
 170         /* Free memory outside the spinlock if need be */
 171         if (!used)
 172                 kfree(buf);
 173
 174         if (!ret)
 175                 dev_info(drvdata->dev, "TMC-ETB/ETF enabled\n");
 176
 177         return ret;
 178 }
 179
 180 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, u32 mode)
 181 {
 182         int ret = 0;
 183         long val;
 184         unsigned long flags;
 185         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 186
 187          /* This shouldn't be happening */
 188         if (WARN_ON(mode != CS_MODE_PERF))
 189                 return -EINVAL;
 190
 191         spin_lock_irqsave(&drvdata->spinlock, flags);
 192         if (drvdata->reading) {
 193                 ret = -EINVAL;
 194                 goto out;
 195         }
 196
 197         val = local_xchg(&drvdata->mode, mode);
 198         /*
 199          * In Perf mode there can be only one writer per sink.  There
 200          * is also no need to continue if the ETB/ETR is already operated
 201          * from sysFS.
 202          */
 203         if (val != CS_MODE_DISABLED) {
 204                 ret = -EINVAL;
 205                 goto out;
 206         }
 207
 208         tmc_etb_enable_hw(drvdata);
 209 out:
 210         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 211
 212         return ret;
 213 }
 214
 215 static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode)
 216 {
 217         switch (mode) {
 218         case CS_MODE_SYSFS:
 219                 return tmc_enable_etf_sink_sysfs(csdev, mode);
 220         case CS_MODE_PERF:
 221                 return tmc_enable_etf_sink_perf(csdev, mode);
 222         }
 223
 224         /* We shouldn't be here */
 225         return -EINVAL;
 226 }
 227
 228 static void tmc_disable_etf_sink(struct coresight_device *csdev)
 229 {
 230         long val;
 231         unsigned long flags;
 232         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 233
 234         spin_lock_irqsave(&drvdata->spinlock, flags);
 235         if (drvdata->reading) {
 236                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 237                 return;
 238         }
 239
 240         val = local_xchg(&drvdata->mode, CS_MODE_DISABLED);
 241         /* Disable the TMC only if it needs to */
 242         if (val != CS_MODE_DISABLED)
 243                 tmc_etb_disable_hw(drvdata);
 244
 245         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 246
 247         dev_info(drvdata->dev, "TMC-ETB/ETF disabled\n");
 248 }
 249
 250 static int tmc_enable_etf_link(struct coresight_device *csdev,
 251                                int inport, int outport)
 252 {
 253         unsigned long flags;
 254         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 255
 256         spin_lock_irqsave(&drvdata->spinlock, flags);
 257         if (drvdata->reading) {
 258                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 259                 return -EBUSY;
 260         }
 261
 262         tmc_etf_enable_hw(drvdata);
 263         local_set(&drvdata->mode, CS_MODE_SYSFS);
 264         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 265
 266         dev_info(drvdata->dev, "TMC-ETF enabled\n");
 267         return 0;
 268 }
 269
 270 static void tmc_disable_etf_link(struct coresight_device *csdev,
 271                                  int inport, int outport)
 272 {
 273         unsigned long flags;
 274         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 275
 276         spin_lock_irqsave(&drvdata->spinlock, flags);
 277         if (drvdata->reading) {
 278                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 279                 return;
 280         }
 281
 282         tmc_etf_disable_hw(drvdata);
 283         local_set(&drvdata->mode, CS_MODE_DISABLED);
 284         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 285
 286         dev_info(drvdata->dev, "TMC disabled\n");
 287 }
 288
 289 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, int cpu,
 290                                   void **pages, int nr_pages, bool overwrite)
 291 {
 292         int node;
 293         struct cs_buffers *buf;
 294
 295         node = (cpu == -1) ? NUMA_NO_NODE : cpu_to_node(cpu);
 296
 297         /* Allocate memory structure for interaction with Perf */
 298         buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
 299         if (!buf)
 300                 return NULL;
 301
 302         buf->snapshot = overwrite;
 303         buf->nr_pages = nr_pages;
 304         buf->data_pages = pages;
 305
 306         return buf;
 307 }
 308
 309 static void tmc_free_etf_buffer(void *config)
 310 {
 311         struct cs_buffers *buf = config;
 312
 313         kfree(buf);
 314 }
 315
 316 static int tmc_set_etf_buffer(struct coresight_device *csdev,
 317                               struct perf_output_handle *handle,
 318                               void *sink_config)
 319 {
 320         int ret = 0;
 321         unsigned long head;
 322         struct cs_buffers *buf = sink_config;
 323
 324         /* wrap head around to the amount of space we have */
 325         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 326
 327         /* find the page to write to */
 328         buf->cur = head / PAGE_SIZE;
 329
 330         /* and offset within that page */
 331         buf->offset = head % PAGE_SIZE;
 332
 333         local_set(&buf->data_size, 0);
 334
 335         return ret;
 336 }
 337
 338 static unsigned long tmc_reset_etf_buffer(struct coresight_device *csdev,
 339                                           struct perf_output_handle *handle,
 340                                           void *sink_config, bool *lost)
 341 {
 342         long size = 0;
 343         struct cs_buffers *buf = sink_config;
 344
 345         if (buf) {
 346                 /*
 347                  * In snapshot mode ->data_size holds the new address of the
 348                  * ring buffer's head.  The size itself is the whole address
 349                  * range since we want the latest information.
 350                  */
 351                 if (buf->snapshot)
 352                         handle->head = local_xchg(&buf->data_size,
 353                                                   buf->nr_pages << PAGE_SHIFT);
 354                 /*
 355                  * Tell the tracer PMU how much we got in this run and if
 356                  * something went wrong along the way.  Nobody else can use
 357                  * this cs_buffers instance until we are done.  As such
 358                  * resetting parameters here and squaring off with the ring
 359                  * buffer API in the tracer PMU is fine.
 360                  */
 361                 *lost = !!local_xchg(&buf->lost, 0);
 362                 size = local_xchg(&buf->data_size, 0);
 363         }
 364
 365         return size;
 366 }
 367
 368 static void tmc_update_etf_buffer(struct coresight_device *csdev,
 369                                   struct perf_output_handle *handle,
 370                                   void *sink_config)
 371 {
 372         int i, cur;
 373         u32 *buf_ptr;
 374         u32 read_ptr, write_ptr;
 375         u32 status, to_read;
 376         unsigned long offset;
 377         struct cs_buffers *buf = sink_config;
 378         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 379
 380         if (!buf)
 381                 return;
 382
 383         /* This shouldn't happen */
 384         if (WARN_ON_ONCE(local_read(&drvdata->mode) != CS_MODE_PERF))
 385                 return;
 386
 387         CS_UNLOCK(drvdata->base);
 388
 389         tmc_flush_and_stop(drvdata);
 390
 391         read_ptr = readl_relaxed(drvdata->base + TMC_RRP);
 392         write_ptr = readl_relaxed(drvdata->base + TMC_RWP);
 393
 394         /*
 395          * Get a hold of the status register and see if a wrap around
 396          * has occurred.  If so adjust things accordingly.
 397          */
 398         status = readl_relaxed(drvdata->base + TMC_STS);
 399         if (status & TMC_STS_FULL) {
 400                 local_inc(&buf->lost);
 401                 to_read = drvdata->size;
 402         } else {
 403                 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
 404         }
 405
 406         /*
 407          * The TMC RAM buffer may be bigger than the space available in the
 408          * perf ring buffer (handle->size).  If so advance the RRP so that we
 409          * get the latest trace data.
 410          */
 411         if (to_read > handle->size) {
 412                 u32 mask = 0;
 413
 414                 /*
 415                  * The value written to RRP must be byte-address aligned to
 416                  * the width of the trace memory databus _and_ to a frame
 417                  * boundary (16 byte), whichever is the biggest. For example,
 418                  * for 32-bit, 64-bit and 128-bit wide trace memory, the four
 419                  * LSBs must be 0s. For 256-bit wide trace memory, the five
 420                  * LSBs must be 0s.
 421                  */
 422                 switch (drvdata->memwidth) {
 423                 case TMC_MEM_INTF_WIDTH_32BITS:
 424                 case TMC_MEM_INTF_WIDTH_64BITS:
 425                 case TMC_MEM_INTF_WIDTH_128BITS:
 426                         mask = GENMASK(31, 4);
 427                         break;
 428                 case TMC_MEM_INTF_WIDTH_256BITS:
 429                         mask = GENMASK(31, 5);
 430                         break;
 431                 }
 432
 433                 /*
 434                  * Make sure the new size is aligned in accordance with the
 435                  * requirement explained above.
 436                  */
 437                 to_read = handle->size & mask;
 438                 /* Move the RAM read pointer up */
 439                 read_ptr = (write_ptr + drvdata->size) - to_read;
 440                 /* Make sure we are still within our limits */
 441                 if (read_ptr > (drvdata->size - 1))
 442                         read_ptr -= drvdata->size;
 443                 /* Tell the HW */
 444                 writel_relaxed(read_ptr, drvdata->base + TMC_RRP);
 445                 local_inc(&buf->lost);
 446         }
 447
 448         cur = buf->cur;
 449         offset = buf->offset;
 450
 451         /* for every byte to read */
 452         for (i = 0; i < to_read; i += 4) {
 453                 buf_ptr = buf->data_pages[cur] + offset;
 454                 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
 455
 456                 offset += 4;
 457                 if (offset >= PAGE_SIZE) {
 458                         offset = 0;
 459                         cur++;
 460                         /* wrap around at the end of the buffer */
 461                         cur &= buf->nr_pages - 1;
 462                 }
 463         }
 464
 465         /*
 466          * In snapshot mode all we have to do is communicate to
 467          * perf_aux_output_end() the address of the current head.  In full
 468          * trace mode the same function expects a size to move rb->aux_head
 469          * forward.
 470          */
 471         if (buf->snapshot)
 472                 local_set(&buf->data_size, (cur * PAGE_SIZE) + offset);
 473         else
 474                 local_add(to_read, &buf->data_size);
 475
 476         CS_LOCK(drvdata->base);
 477 }
 478
 479 static const struct coresight_ops_sink tmc_etf_sink_ops = {
 480         .enable         = tmc_enable_etf_sink,
 481         .disable        = tmc_disable_etf_sink,
 482         .alloc_buffer   = tmc_alloc_etf_buffer,
 483         .free_buffer    = tmc_free_etf_buffer,
 484         .set_buffer     = tmc_set_etf_buffer,
 485         .reset_buffer   = tmc_reset_etf_buffer,
 486         .update_buffer  = tmc_update_etf_buffer,
 487 };
 488
 489 static const struct coresight_ops_link tmc_etf_link_ops = {
 490         .enable         = tmc_enable_etf_link,
 491         .disable        = tmc_disable_etf_link,
 492 };
 493
 494 const struct coresight_ops tmc_etb_cs_ops = {
 495         .sink_ops       = &tmc_etf_sink_ops,
 496 };
 497
 498 const struct coresight_ops tmc_etf_cs_ops = {
 499         .sink_ops       = &tmc_etf_sink_ops,
 500         .link_ops       = &tmc_etf_link_ops,
 501 };
 502
 503 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 504 {
 505         long val;
 506         enum tmc_mode mode;
 507         int ret = 0;
 508         unsigned long flags;
 509
 510         /* config types are set a boot time and never change */
 511         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 512                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 513                 return -EINVAL;
 514
 515         spin_lock_irqsave(&drvdata->spinlock, flags);
 516
 517         if (drvdata->reading) {
 518                 ret = -EBUSY;
 519                 goto out;
 520         }
 521
 522         /* There is no point in reading a TMC in HW FIFO mode */
 523         mode = readl_relaxed(drvdata->base + TMC_MODE);
 524         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 525                 ret = -EINVAL;
 526                 goto out;
 527         }
 528
 529         val = local_read(&drvdata->mode);
 530         /* Don't interfere if operated from Perf */
 531         if (val == CS_MODE_PERF) {
 532                 ret = -EINVAL;
 533                 goto out;
 534         }
 535
 536         /* If drvdata::buf is NULL the trace data has been read already */
 537         if (drvdata->buf == NULL) {
 538                 ret = -EINVAL;
 539                 goto out;
 540         }
 541
 542         /* Disable the TMC if need be */
 543         if (val == CS_MODE_SYSFS)
 544                 tmc_etb_disable_hw(drvdata);
 545
 546         drvdata->reading = true;
 547 out:
 548         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 549
 550         return ret;
 551 }
 552
 553 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
 554 {
 555         char *buf = NULL;
 556         enum tmc_mode mode;
 557         unsigned long flags;
 558
 559         /* config types are set a boot time and never change */
 560         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 561                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 562                 return -EINVAL;
 563
 564         spin_lock_irqsave(&drvdata->spinlock, flags);
 565
 566         /* There is no point in reading a TMC in HW FIFO mode */
 567         mode = readl_relaxed(drvdata->base + TMC_MODE);
 568         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 569                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 570                 return -EINVAL;
 571         }
 572
 573         /* Re-enable the TMC if need be */
 574         if (local_read(&drvdata->mode) == CS_MODE_SYSFS) {
 575                 /*
 576                  * The trace run will continue with the same allocated trace
 577                  * buffer. As such zero-out the buffer so that we don't end
 578                  * up with stale data.
 579                  *
 580                  * Since the tracer is still enabled drvdata::buf
 581                  * can't be NULL.
 582                  */
 583                 memset(drvdata->buf, 0, drvdata->size);
 584                 tmc_etb_enable_hw(drvdata);
 585         } else {
 586                 /*
 587                  * The ETB/ETF is not tracing and the buffer was just read.
 588                  * As such prepare to free the trace buffer.
 589                  */
 590                 buf = drvdata->buf;
 591                 drvdata->buf = NULL;
 592         }
 593
 594         drvdata->reading = false;
 595         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 596
 597         /*
 598          * Free allocated memory outside of the spinlock.  There is no need
 599          * to assert the validity of 'buf' since calling kfree(NULL) is safe.
 600          */
 601         kfree(buf);
 602
 603         return 0;
 604 }