drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c

   1 /*
   2  * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #define CREATE_TRACE_POINTS
  33 #include "fw_tracer.h"
  34 #include "fw_tracer_tracepoint.h"
  35
  36 static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer)
  37 {
  38         u32 *string_db_base_address_out = tracer->str_db.base_address_out;
  39         u32 *string_db_size_out = tracer->str_db.size_out;
  40         struct mlx5_core_dev *dev = tracer->dev;
  41         u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
  42         u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
  43         void *mtrc_cap_sp;
  44         int err, i;
  45
  46         err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
  47                                    MLX5_REG_MTRC_CAP, 0, 0);
  48         if (err) {
  49                 mlx5_core_warn(dev, "FWTracer: Error reading tracer caps %d\n",
  50                                err);
  51                 return err;
  52         }
  53
  54         if (!MLX5_GET(mtrc_cap, out, trace_to_memory)) {
  55                 mlx5_core_dbg(dev, "FWTracer: Device does not support logging traces to memory\n");
  56                 return -ENOTSUPP;
  57         }
  58
  59         tracer->trc_ver = MLX5_GET(mtrc_cap, out, trc_ver);
  60         tracer->str_db.first_string_trace =
  61                         MLX5_GET(mtrc_cap, out, first_string_trace);
  62         tracer->str_db.num_string_trace =
  63                         MLX5_GET(mtrc_cap, out, num_string_trace);
  64         tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db);
  65         tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
  66
  67         for (i = 0; i < tracer->str_db.num_string_db; i++) {
  68                 mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]);
  69                 string_db_base_address_out[i] = MLX5_GET(mtrc_string_db_param,
  70                                                          mtrc_cap_sp,
  71                                                          string_db_base_address);
  72                 string_db_size_out[i] = MLX5_GET(mtrc_string_db_param,
  73                                                  mtrc_cap_sp, string_db_size);
  74         }
  75
  76         return err;
  77 }
  78
  79 static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer *tracer,
  80                                           u32 *out, u32 out_size,
  81                                           u8 trace_owner)
  82 {
  83         struct mlx5_core_dev *dev = tracer->dev;
  84         u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
  85
  86         MLX5_SET(mtrc_cap, in, trace_owner, trace_owner);
  87
  88         return mlx5_core_access_reg(dev, in, sizeof(in), out, out_size,
  89                                     MLX5_REG_MTRC_CAP, 0, 1);
  90 }
  91
  92 static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer *tracer)
  93 {
  94         struct mlx5_core_dev *dev = tracer->dev;
  95         u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
  96         int err;
  97
  98         err = mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
  99                                              MLX5_FW_TRACER_ACQUIRE_OWNERSHIP);
 100         if (err) {
 101                 mlx5_core_warn(dev, "FWTracer: Acquire tracer ownership failed %d\n",
 102                                err);
 103                 return err;
 104         }
 105
 106         tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
 107
 108         if (!tracer->owner)
 109                 return -EBUSY;
 110
 111         return 0;
 112 }
 113
 114 static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer)
 115 {
 116         u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
 117
 118         mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
 119                                        MLX5_FW_TRACER_RELEASE_OWNERSHIP);
 120         tracer->owner = false;
 121 }
 122
 123 static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer)
 124 {
 125         struct mlx5_core_dev *dev = tracer->dev;
 126         struct device *ddev = &dev->pdev->dev;
 127         dma_addr_t dma;
 128         void *buff;
 129         gfp_t gfp;
 130         int err;
 131
 132         tracer->buff.size = TRACE_BUFFER_SIZE_BYTE;
 133
 134         gfp = GFP_KERNEL | __GFP_ZERO;
 135         buff = (void *)__get_free_pages(gfp,
 136                                         get_order(tracer->buff.size));
 137         if (!buff) {
 138                 err = -ENOMEM;
 139                 mlx5_core_warn(dev, "FWTracer: Failed to allocate pages, %d\n", err);
 140                 return err;
 141         }
 142         tracer->buff.log_buf = buff;
 143
 144         dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE);
 145         if (dma_mapping_error(ddev, dma)) {
 146                 mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n",
 147                                dma_mapping_error(ddev, dma));
 148                 err = -ENOMEM;
 149                 goto free_pages;
 150         }
 151         tracer->buff.dma = dma;
 152
 153         return 0;
 154
 155 free_pages:
 156         free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
 157
 158         return err;
 159 }
 160
 161 static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer)
 162 {
 163         struct mlx5_core_dev *dev = tracer->dev;
 164         struct device *ddev = &dev->pdev->dev;
 165
 166         if (!tracer->buff.log_buf)
 167                 return;
 168
 169         dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE);
 170         free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
 171 }
 172
 173 static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer *tracer)
 174 {
 175         struct mlx5_core_dev *dev = tracer->dev;
 176         int err, inlen, i;
 177         __be64 *mtt;
 178         void *mkc;
 179         u32 *in;
 180
 181         inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
 182                         sizeof(*mtt) * round_up(TRACER_BUFFER_PAGE_NUM, 2);
 183
 184         in = kvzalloc(inlen, GFP_KERNEL);
 185         if (!in)
 186                 return -ENOMEM;
 187
 188         MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
 189                  DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
 190         mtt = (u64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 191         for (i = 0 ; i < TRACER_BUFFER_PAGE_NUM ; i++)
 192                 mtt[i] = cpu_to_be64(tracer->buff.dma + i * PAGE_SIZE);
 193
 194         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 195         MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
 196         MLX5_SET(mkc, mkc, lr, 1);
 197         MLX5_SET(mkc, mkc, lw, 1);
 198         MLX5_SET(mkc, mkc, pd, tracer->buff.pdn);
 199         MLX5_SET(mkc, mkc, bsf_octword_size, 0);
 200         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 201         MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
 202         MLX5_SET(mkc, mkc, translations_octword_size,
 203                  DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
 204         MLX5_SET64(mkc, mkc, start_addr, tracer->buff.dma);
 205         MLX5_SET64(mkc, mkc, len, tracer->buff.size);
 206         err = mlx5_core_create_mkey(dev, &tracer->buff.mkey, in, inlen);
 207         if (err)
 208                 mlx5_core_warn(dev, "FWTracer: Failed to create mkey, %d\n", err);
 209
 210         kvfree(in);
 211
 212         return err;
 213 }
 214
 215 static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer *tracer)
 216 {
 217         u32 num_string_db = tracer->str_db.num_string_db;
 218         int i;
 219
 220         for (i = 0; i < num_string_db; i++) {
 221                 kfree(tracer->str_db.buffer[i]);
 222                 tracer->str_db.buffer[i] = NULL;
 223         }
 224 }
 225
 226 static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer)
 227 {
 228         u32 *string_db_size_out = tracer->str_db.size_out;
 229         u32 num_string_db = tracer->str_db.num_string_db;
 230         int i;
 231
 232         for (i = 0; i < num_string_db; i++) {
 233                 tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL);
 234                 if (!tracer->str_db.buffer[i])
 235                         goto free_strings_db;
 236         }
 237
 238         return 0;
 239
 240 free_strings_db:
 241         mlx5_fw_tracer_free_strings_db(tracer);
 242         return -ENOMEM;
 243 }
 244
 245 static void mlx5_tracer_read_strings_db(struct work_struct *work)
 246 {
 247         struct mlx5_fw_tracer *tracer = container_of(work, struct mlx5_fw_tracer,
 248                                                      read_fw_strings_work);
 249         u32 num_of_reads, num_string_db = tracer->str_db.num_string_db;
 250         struct mlx5_core_dev *dev = tracer->dev;
 251         u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
 252         u32 leftovers, offset;
 253         int err = 0, i, j;
 254         u32 *out, outlen;
 255         void *out_value;
 256
 257         outlen = MLX5_ST_SZ_BYTES(mtrc_stdb) + STRINGS_DB_READ_SIZE_BYTES;
 258         out = kzalloc(outlen, GFP_KERNEL);
 259         if (!out) {
 260                 err = -ENOMEM;
 261                 goto out;
 262         }
 263
 264         for (i = 0; i < num_string_db; i++) {
 265                 offset = 0;
 266                 MLX5_SET(mtrc_stdb, in, string_db_index, i);
 267                 num_of_reads = tracer->str_db.size_out[i] /
 268                                 STRINGS_DB_READ_SIZE_BYTES;
 269                 leftovers = (tracer->str_db.size_out[i] %
 270                                 STRINGS_DB_READ_SIZE_BYTES) /
 271                                         STRINGS_DB_LEFTOVER_SIZE_BYTES;
 272
 273                 MLX5_SET(mtrc_stdb, in, read_size, STRINGS_DB_READ_SIZE_BYTES);
 274                 for (j = 0; j < num_of_reads; j++) {
 275                         MLX5_SET(mtrc_stdb, in, start_offset, offset);
 276
 277                         err = mlx5_core_access_reg(dev, in, sizeof(in), out,
 278                                                    outlen, MLX5_REG_MTRC_STDB,
 279                                                    0, 1);
 280                         if (err) {
 281                                 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
 282                                               err);
 283                                 goto out_free;
 284                         }
 285
 286                         out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
 287                         memcpy(tracer->str_db.buffer[i] + offset, out_value,
 288                                STRINGS_DB_READ_SIZE_BYTES);
 289                         offset += STRINGS_DB_READ_SIZE_BYTES;
 290                 }
 291
 292                 /* Strings database is aligned to 64, need to read leftovers*/
 293                 MLX5_SET(mtrc_stdb, in, read_size,
 294                          STRINGS_DB_LEFTOVER_SIZE_BYTES);
 295                 for (j = 0; j < leftovers; j++) {
 296                         MLX5_SET(mtrc_stdb, in, start_offset, offset);
 297
 298                         err = mlx5_core_access_reg(dev, in, sizeof(in), out,
 299                                                    outlen, MLX5_REG_MTRC_STDB,
 300                                                    0, 1);
 301                         if (err) {
 302                                 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
 303                                               err);
 304                                 goto out_free;
 305                         }
 306
 307                         out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
 308                         memcpy(tracer->str_db.buffer[i] + offset, out_value,
 309                                STRINGS_DB_LEFTOVER_SIZE_BYTES);
 310                         offset += STRINGS_DB_LEFTOVER_SIZE_BYTES;
 311                 }
 312         }
 313
 314         tracer->str_db.loaded = true;
 315
 316 out_free:
 317         kfree(out);
 318 out:
 319         return;
 320 }
 321
 322 static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev)
 323 {
 324         u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
 325         u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
 326         int err;
 327
 328         MLX5_SET(mtrc_ctrl, in, arm_event, 1);
 329
 330         err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
 331                                    MLX5_REG_MTRC_CTRL, 0, 1);
 332         if (err)
 333                 mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err);
 334 }
 335
 336 static const char *VAL_PARM             = "%llx";
 337 static const char *REPLACE_64_VAL_PARM  = "%x%x";
 338 static const char *PARAM_CHAR           = "%";
 339
 340 static int mlx5_tracer_message_hash(u32 message_id)
 341 {
 342         return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1);
 343 }
 344
 345 static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer,
 346                                                                struct tracer_event *tracer_event)
 347 {
 348         struct hlist_head *head =
 349                 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
 350         struct tracer_string_format *cur_string;
 351
 352         cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL);
 353         if (!cur_string)
 354                 return NULL;
 355
 356         hlist_add_head(&cur_string->hlist, head);
 357
 358         return cur_string;
 359 }
 360
 361 static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer,
 362                                                            struct tracer_event *tracer_event)
 363 {
 364         struct tracer_string_format *cur_string;
 365         u32 str_ptr, offset;
 366         int i;
 367
 368         str_ptr = tracer_event->string_event.string_param;
 369
 370         for (i = 0; i < tracer->str_db.num_string_db; i++) {
 371                 if (str_ptr > tracer->str_db.base_address_out[i] &&
 372                     str_ptr < tracer->str_db.base_address_out[i] +
 373                     tracer->str_db.size_out[i]) {
 374                         offset = str_ptr - tracer->str_db.base_address_out[i];
 375                         /* add it to the hash */
 376                         cur_string = mlx5_tracer_message_insert(tracer, tracer_event);
 377                         if (!cur_string)
 378                                 return NULL;
 379                         cur_string->string = (char *)(tracer->str_db.buffer[i] +
 380                                                         offset);
 381                         return cur_string;
 382                 }
 383         }
 384
 385         return NULL;
 386 }
 387
 388 static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt)
 389 {
 390         hlist_del(&str_frmt->hlist);
 391         kfree(str_frmt);
 392 }
 393
 394 static int mlx5_tracer_get_num_of_params(char *str)
 395 {
 396         char *substr, *pstr = str;
 397         int num_of_params = 0;
 398
 399         /* replace %llx with %x%x */
 400         substr = strstr(pstr, VAL_PARM);
 401         while (substr) {
 402                 memcpy(substr, REPLACE_64_VAL_PARM, 4);
 403                 pstr = substr;
 404                 substr = strstr(pstr, VAL_PARM);
 405         }
 406
 407         /* count all the % characters */
 408         substr = strstr(str, PARAM_CHAR);
 409         while (substr) {
 410                 num_of_params += 1;
 411                 str = substr + 1;
 412                 substr = strstr(str, PARAM_CHAR);
 413         }
 414
 415         return num_of_params;
 416 }
 417
 418 static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head,
 419                                                              u8 event_id, u32 tmsn)
 420 {
 421         struct tracer_string_format *message;
 422
 423         hlist_for_each_entry(message, head, hlist)
 424                 if (message->event_id == event_id && message->tmsn == tmsn)
 425                         return message;
 426
 427         return NULL;
 428 }
 429
 430 static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer,
 431                                                             struct tracer_event *tracer_event)
 432 {
 433         struct hlist_head *head =
 434                 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
 435
 436         return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn);
 437 }
 438
 439 static void poll_trace(struct mlx5_fw_tracer *tracer,
 440                        struct tracer_event *tracer_event, u64 *trace)
 441 {
 442         u32 timestamp_low, timestamp_mid, timestamp_high, urts;
 443
 444         tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id);
 445         tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost);
 446
 447         switch (tracer_event->event_id) {
 448         case TRACER_EVENT_TYPE_TIMESTAMP:
 449                 tracer_event->type = TRACER_EVENT_TYPE_TIMESTAMP;
 450                 urts = MLX5_GET(tracer_timestamp_event, trace, urts);
 451                 if (tracer->trc_ver == 0)
 452                         tracer_event->timestamp_event.unreliable = !!(urts >> 2);
 453                 else
 454                         tracer_event->timestamp_event.unreliable = !!(urts & 1);
 455
 456                 timestamp_low = MLX5_GET(tracer_timestamp_event,
 457                                          trace, timestamp7_0);
 458                 timestamp_mid = MLX5_GET(tracer_timestamp_event,
 459                                          trace, timestamp39_8);
 460                 timestamp_high = MLX5_GET(tracer_timestamp_event,
 461                                           trace, timestamp52_40);
 462
 463                 tracer_event->timestamp_event.timestamp =
 464                                 ((u64)timestamp_high << 40) |
 465                                 ((u64)timestamp_mid << 8) |
 466                                 (u64)timestamp_low;
 467                 break;
 468         default:
 469                 if (tracer_event->event_id >= tracer->str_db.first_string_trace ||
 470                     tracer_event->event_id <= tracer->str_db.first_string_trace +
 471                                               tracer->str_db.num_string_trace) {
 472                         tracer_event->type = TRACER_EVENT_TYPE_STRING;
 473                         tracer_event->string_event.timestamp =
 474                                 MLX5_GET(tracer_string_event, trace, timestamp);
 475                         tracer_event->string_event.string_param =
 476                                 MLX5_GET(tracer_string_event, trace, string_param);
 477                         tracer_event->string_event.tmsn =
 478                                 MLX5_GET(tracer_string_event, trace, tmsn);
 479                         tracer_event->string_event.tdsn =
 480                                 MLX5_GET(tracer_string_event, trace, tdsn);
 481                 } else {
 482                         tracer_event->type = TRACER_EVENT_TYPE_UNRECOGNIZED;
 483                 }
 484                 break;
 485         }
 486 }
 487
 488 static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event)
 489 {
 490         struct tracer_event tracer_event;
 491         u8 event_id;
 492
 493         event_id = MLX5_GET(tracer_event, ts_event, event_id);
 494
 495         if (event_id == TRACER_EVENT_TYPE_TIMESTAMP)
 496                 poll_trace(tracer, &tracer_event, ts_event);
 497         else
 498                 tracer_event.timestamp_event.timestamp = 0;
 499
 500         return tracer_event.timestamp_event.timestamp;
 501 }
 502
 503 static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer)
 504 {
 505         struct tracer_string_format *str_frmt;
 506         struct hlist_node *n;
 507         int i;
 508
 509         for (i = 0; i < MESSAGE_HASH_SIZE; i++) {
 510                 hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist)
 511                         mlx5_tracer_clean_message(str_frmt);
 512         }
 513 }
 514
 515 static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer)
 516 {
 517         struct tracer_string_format *str_frmt, *tmp_str;
 518
 519         list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list,
 520                                  list)
 521                 list_del(&str_frmt->list);
 522 }
 523
 524 static void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt,
 525                                     struct mlx5_core_dev *dev,
 526                                     u64 trace_timestamp)
 527 {
 528         char    tmp[512];
 529
 530         snprintf(tmp, sizeof(tmp), str_frmt->string,
 531                  str_frmt->params[0],
 532                  str_frmt->params[1],
 533                  str_frmt->params[2],
 534                  str_frmt->params[3],
 535                  str_frmt->params[4],
 536                  str_frmt->params[5],
 537                  str_frmt->params[6]);
 538
 539         trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost,
 540                       str_frmt->event_id, tmp);
 541
 542         /* remove it from hash */
 543         mlx5_tracer_clean_message(str_frmt);
 544 }
 545
 546 static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
 547                                            struct tracer_event *tracer_event)
 548 {
 549         struct tracer_string_format *cur_string;
 550
 551         if (tracer_event->string_event.tdsn == 0) {
 552                 cur_string = mlx5_tracer_get_string(tracer, tracer_event);
 553                 if (!cur_string)
 554                         return -1;
 555
 556                 cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string);
 557                 cur_string->last_param_num = 0;
 558                 cur_string->event_id = tracer_event->event_id;
 559                 cur_string->tmsn = tracer_event->string_event.tmsn;
 560                 cur_string->timestamp = tracer_event->string_event.timestamp;
 561                 cur_string->lost = tracer_event->lost_event;
 562                 if (cur_string->num_of_params == 0) /* trace with no params */
 563                         list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 564         } else {
 565                 cur_string = mlx5_tracer_message_get(tracer, tracer_event);
 566                 if (!cur_string) {
 567                         pr_debug("%s Got string event for unknown string tdsm: %d\n",
 568                                  __func__, tracer_event->string_event.tmsn);
 569                         return -1;
 570                 }
 571                 cur_string->last_param_num += 1;
 572                 if (cur_string->last_param_num > TRACER_MAX_PARAMS) {
 573                         pr_debug("%s Number of params exceeds the max (%d)\n",
 574                                  __func__, TRACER_MAX_PARAMS);
 575                         list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 576                         return 0;
 577                 }
 578                 /* keep the new parameter */
 579                 cur_string->params[cur_string->last_param_num - 1] =
 580                         tracer_event->string_event.string_param;
 581                 if (cur_string->last_param_num == cur_string->num_of_params)
 582                         list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 583         }
 584
 585         return 0;
 586 }
 587
 588 static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer,
 589                                                struct tracer_event *tracer_event)
 590 {
 591         struct tracer_timestamp_event timestamp_event =
 592                                                 tracer_event->timestamp_event;
 593         struct tracer_string_format *str_frmt, *tmp_str;
 594         struct mlx5_core_dev *dev = tracer->dev;
 595         u64 trace_timestamp;
 596
 597         list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) {
 598                 list_del(&str_frmt->list);
 599                 if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0))
 600                         trace_timestamp = (timestamp_event.timestamp & MASK_52_7) |
 601                                           (str_frmt->timestamp & MASK_6_0);
 602                 else
 603                         trace_timestamp = ((timestamp_event.timestamp & MASK_52_7) - 1) |
 604                                           (str_frmt->timestamp & MASK_6_0);
 605
 606                 mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp);
 607         }
 608 }
 609
 610 static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer,
 611                                     struct tracer_event *tracer_event)
 612 {
 613         if (tracer_event->type == TRACER_EVENT_TYPE_STRING) {
 614                 mlx5_tracer_handle_string_trace(tracer, tracer_event);
 615         } else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) {
 616                 if (!tracer_event->timestamp_event.unreliable)
 617                         mlx5_tracer_handle_timestamp_trace(tracer, tracer_event);
 618         } else {
 619                 pr_debug("%s Got unrecognised type %d for parsing, exiting..\n",
 620                          __func__, tracer_event->type);
 621         }
 622         return 0;
 623 }
 624
 625 static void mlx5_fw_tracer_handle_traces(struct work_struct *work)
 626 {
 627         struct mlx5_fw_tracer *tracer =
 628                         container_of(work, struct mlx5_fw_tracer, handle_traces_work);
 629         u64 block_timestamp, last_block_timestamp, tmp_trace_block[TRACES_PER_BLOCK];
 630         u32 block_count, start_offset, prev_start_offset, prev_consumer_index;
 631         u32 trace_event_size = MLX5_ST_SZ_BYTES(tracer_event);
 632         struct mlx5_core_dev *dev = tracer->dev;
 633         struct tracer_event tracer_event;
 634         int i;
 635
 636         mlx5_core_dbg(dev, "FWTracer: Handle Trace event, owner=(%d)\n", tracer->owner);
 637         if (!tracer->owner)
 638                 return;
 639
 640         block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE;
 641         start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
 642
 643         /* Copy the block to local buffer to avoid HW override while being processed*/
 644         memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
 645                TRACER_BLOCK_SIZE_BYTE);
 646
 647         block_timestamp =
 648                 get_block_timestamp(tracer, &tmp_trace_block[TRACES_PER_BLOCK - 1]);
 649
 650         while (block_timestamp > tracer->last_timestamp) {
 651                 /* Check block override if its not the first block */
 652                 if (!tracer->last_timestamp) {
 653                         u64 *ts_event;
 654                         /* To avoid block override be the HW in case of buffer
 655                          * wraparound, the time stamp of the previous block
 656                          * should be compared to the last timestamp handled
 657                          * by the driver.
 658                          */
 659                         prev_consumer_index =
 660                                 (tracer->buff.consumer_index - 1) & (block_count - 1);
 661                         prev_start_offset = prev_consumer_index * TRACER_BLOCK_SIZE_BYTE;
 662
 663                         ts_event = tracer->buff.log_buf + prev_start_offset +
 664                                    (TRACES_PER_BLOCK - 1) * trace_event_size;
 665                         last_block_timestamp = get_block_timestamp(tracer, ts_event);
 666                         /* If previous timestamp different from last stored
 667                          * timestamp then there is a good chance that the
 668                          * current buffer is overwritten and therefore should
 669                          * not be parsed.
 670                          */
 671                         if (tracer->last_timestamp != last_block_timestamp) {
 672                                 mlx5_core_warn(dev, "FWTracer: Events were lost\n");
 673                                 tracer->last_timestamp = block_timestamp;
 674                                 tracer->buff.consumer_index =
 675                                         (tracer->buff.consumer_index + 1) & (block_count - 1);
 676                                 break;
 677                         }
 678                 }
 679
 680                 /* Parse events */
 681                 for (i = 0; i < TRACES_PER_BLOCK ; i++) {
 682                         poll_trace(tracer, &tracer_event, &tmp_trace_block[i]);
 683                         mlx5_tracer_handle_trace(tracer, &tracer_event);
 684                 }
 685
 686                 tracer->buff.consumer_index =
 687                         (tracer->buff.consumer_index + 1) & (block_count - 1);
 688
 689                 tracer->last_timestamp = block_timestamp;
 690                 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
 691                 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
 692                        TRACER_BLOCK_SIZE_BYTE);
 693                 block_timestamp = get_block_timestamp(tracer,
 694                                                       &tmp_trace_block[TRACES_PER_BLOCK - 1]);
 695         }
 696
 697         mlx5_fw_tracer_arm(dev);
 698 }
 699
 700 static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer)
 701 {
 702         struct mlx5_core_dev *dev = tracer->dev;
 703         u32 out[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
 704         u32 in[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
 705         int err;
 706
 707         MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY);
 708         MLX5_SET(mtrc_conf, in, log_trace_buffer_size,
 709                  ilog2(TRACER_BUFFER_PAGE_NUM));
 710         MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey.key);
 711
 712         err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
 713                                    MLX5_REG_MTRC_CONF, 0, 1);
 714         if (err)
 715                 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err);
 716
 717         return err;
 718 }
 719
 720 static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer *tracer, u8 status, u8 arm)
 721 {
 722         struct mlx5_core_dev *dev = tracer->dev;
 723         u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
 724         u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
 725         int err;
 726
 727         MLX5_SET(mtrc_ctrl, in, modify_field_select, TRACE_STATUS);
 728         MLX5_SET(mtrc_ctrl, in, trace_status, status);
 729         MLX5_SET(mtrc_ctrl, in, arm_event, arm);
 730
 731         err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
 732                                    MLX5_REG_MTRC_CTRL, 0, 1);
 733
 734         if (!err && status)
 735                 tracer->last_timestamp = 0;
 736
 737         return err;
 738 }
 739
 740 static int mlx5_fw_tracer_start(struct mlx5_fw_tracer *tracer)
 741 {
 742         struct mlx5_core_dev *dev = tracer->dev;
 743         int err;
 744
 745         err = mlx5_fw_tracer_ownership_acquire(tracer);
 746         if (err) {
 747                 mlx5_core_dbg(dev, "FWTracer: Ownership was not granted %d\n", err);
 748                 /* Don't fail since ownership can be acquired on a later FW event */
 749                 return 0;
 750         }
 751
 752         err = mlx5_fw_tracer_set_mtrc_conf(tracer);
 753         if (err) {
 754                 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configuration %d\n", err);
 755                 goto release_ownership;
 756         }
 757
 758         /* enable tracer & trace events */
 759         err = mlx5_fw_tracer_set_mtrc_ctrl(tracer, 1, 1);
 760         if (err) {
 761                 mlx5_core_warn(dev, "FWTracer: Failed to enable tracer %d\n", err);
 762                 goto release_ownership;
 763         }
 764
 765         mlx5_core_dbg(dev, "FWTracer: Ownership granted and active\n");
 766         return 0;
 767
 768 release_ownership:
 769         mlx5_fw_tracer_ownership_release(tracer);
 770         return err;
 771 }
 772
 773 static void mlx5_fw_tracer_ownership_change(struct work_struct *work)
 774 {
 775         struct mlx5_fw_tracer *tracer =
 776                 container_of(work, struct mlx5_fw_tracer, ownership_change_work);
 777
 778         mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner);
 779         if (tracer->owner) {
 780                 tracer->owner = false;
 781                 tracer->buff.consumer_index = 0;
 782                 return;
 783         }
 784
 785         mlx5_fw_tracer_start(tracer);
 786 }
 787
 788 /* Create software resources (Buffers, etc ..) */
 789 struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev)
 790 {
 791         struct mlx5_fw_tracer *tracer = NULL;
 792         int err;
 793
 794         if (!MLX5_CAP_MCAM_REG(dev, tracer_registers)) {
 795                 mlx5_core_dbg(dev, "FWTracer: Tracer capability not present\n");
 796                 return NULL;
 797         }
 798
 799         tracer = kvzalloc(sizeof(*tracer), GFP_KERNEL);
 800         if (!tracer)
 801                 return ERR_PTR(-ENOMEM);
 802
 803         tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer");
 804         if (!tracer->work_queue) {
 805                 err = -ENOMEM;
 806                 goto free_tracer;
 807         }
 808
 809         tracer->dev = dev;
 810
 811         INIT_LIST_HEAD(&tracer->ready_strings_list);
 812         INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change);
 813         INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db);
 814         INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces);
 815
 816
 817         err = mlx5_query_mtrc_caps(tracer);
 818         if (err) {
 819                 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err);
 820                 goto destroy_workqueue;
 821         }
 822
 823         err = mlx5_fw_tracer_create_log_buf(tracer);
 824         if (err) {
 825                 mlx5_core_warn(dev, "FWTracer: Create log buffer failed %d\n", err);
 826                 goto destroy_workqueue;
 827         }
 828
 829         err = mlx5_fw_tracer_allocate_strings_db(tracer);
 830         if (err) {
 831                 mlx5_core_warn(dev, "FWTracer: Allocate strings database failed %d\n", err);
 832                 goto free_log_buf;
 833         }
 834
 835         mlx5_core_dbg(dev, "FWTracer: Tracer created\n");
 836
 837         return tracer;
 838
 839 free_log_buf:
 840         mlx5_fw_tracer_destroy_log_buf(tracer);
 841 destroy_workqueue:
 842         tracer->dev = NULL;
 843         destroy_workqueue(tracer->work_queue);
 844 free_tracer:
 845         kvfree(tracer);
 846         return ERR_PTR(err);
 847 }
 848
 849 /* Create HW resources + start tracer
 850  * must be called before Async EQ is created
 851  */
 852 int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer)
 853 {
 854         struct mlx5_core_dev *dev;
 855         int err;
 856
 857         if (IS_ERR_OR_NULL(tracer))
 858                 return 0;
 859
 860         dev = tracer->dev;
 861
 862         if (!tracer->str_db.loaded)
 863                 queue_work(tracer->work_queue, &tracer->read_fw_strings_work);
 864
 865         err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn);
 866         if (err) {
 867                 mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err);
 868                 return err;
 869         }
 870
 871         err = mlx5_fw_tracer_create_mkey(tracer);
 872         if (err) {
 873                 mlx5_core_warn(dev, "FWTracer: Failed to create mkey %d\n", err);
 874                 goto err_dealloc_pd;
 875         }
 876
 877         mlx5_fw_tracer_start(tracer);
 878
 879         return 0;
 880
 881 err_dealloc_pd:
 882         mlx5_core_dealloc_pd(dev, tracer->buff.pdn);
 883         return err;
 884 }
 885
 886 /* Stop tracer + Cleanup HW resources
 887  * must be called after Async EQ is destroyed
 888  */
 889 void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer)
 890 {
 891         if (IS_ERR_OR_NULL(tracer))
 892                 return;
 893
 894         mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n",
 895                       tracer->owner);
 896
 897         cancel_work_sync(&tracer->ownership_change_work);
 898         cancel_work_sync(&tracer->handle_traces_work);
 899
 900         if (tracer->owner)
 901                 mlx5_fw_tracer_ownership_release(tracer);
 902
 903         mlx5_core_destroy_mkey(tracer->dev, &tracer->buff.mkey);
 904         mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn);
 905 }
 906
 907 /* Free software resources (Buffers, etc ..) */
 908 void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer)
 909 {
 910         if (IS_ERR_OR_NULL(tracer))
 911                 return;
 912
 913         mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n");
 914
 915         cancel_work_sync(&tracer->read_fw_strings_work);
 916         mlx5_fw_tracer_clean_ready_list(tracer);
 917         mlx5_fw_tracer_clean_print_hash(tracer);
 918         mlx5_fw_tracer_free_strings_db(tracer);
 919         mlx5_fw_tracer_destroy_log_buf(tracer);
 920         flush_workqueue(tracer->work_queue);
 921         destroy_workqueue(tracer->work_queue);
 922         kvfree(tracer);
 923 }
 924
 925 void mlx5_fw_tracer_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe)
 926 {
 927         struct mlx5_fw_tracer *tracer = dev->tracer;
 928
 929         if (!tracer)
 930                 return;
 931
 932         switch (eqe->sub_type) {
 933         case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE:
 934                 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state))
 935                         queue_work(tracer->work_queue, &tracer->ownership_change_work);
 936                 break;
 937         case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE:
 938                 if (likely(tracer->str_db.loaded))
 939                         queue_work(tracer->work_queue, &tracer->handle_traces_work);
 940                 break;
 941         default:
 942                 mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n",
 943                               eqe->sub_type);
 944         }
 945 }
 946
 947 EXPORT_TRACEPOINT_SYMBOL(mlx5_fw);