2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #define CREATE_TRACE_POINTS
33 #include "fw_tracer.h"
34 #include "fw_tracer_tracepoint.h"
36 static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer)
38 u32 *string_db_base_address_out = tracer->str_db.base_address_out;
39 u32 *string_db_size_out = tracer->str_db.size_out;
40 struct mlx5_core_dev *dev = tracer->dev;
41 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
42 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
46 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
47 MLX5_REG_MTRC_CAP, 0, 0);
49 mlx5_core_warn(dev, "FWTracer: Error reading tracer caps %d\n",
54 if (!MLX5_GET(mtrc_cap, out, trace_to_memory)) {
55 mlx5_core_dbg(dev, "FWTracer: Device does not support logging traces to memory\n");
59 tracer->trc_ver = MLX5_GET(mtrc_cap, out, trc_ver);
60 tracer->str_db.first_string_trace =
61 MLX5_GET(mtrc_cap, out, first_string_trace);
62 tracer->str_db.num_string_trace =
63 MLX5_GET(mtrc_cap, out, num_string_trace);
64 tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db);
65 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
67 for (i = 0; i < tracer->str_db.num_string_db; i++) {
68 mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]);
69 string_db_base_address_out[i] = MLX5_GET(mtrc_string_db_param,
71 string_db_base_address);
72 string_db_size_out[i] = MLX5_GET(mtrc_string_db_param,
73 mtrc_cap_sp, string_db_size);
79 static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer *tracer,
80 u32 *out, u32 out_size,
83 struct mlx5_core_dev *dev = tracer->dev;
84 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
86 MLX5_SET(mtrc_cap, in, trace_owner, trace_owner);
88 return mlx5_core_access_reg(dev, in, sizeof(in), out, out_size,
89 MLX5_REG_MTRC_CAP, 0, 1);
92 static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer *tracer)
94 struct mlx5_core_dev *dev = tracer->dev;
95 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
98 err = mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
99 MLX5_FW_TRACER_ACQUIRE_OWNERSHIP);
101 mlx5_core_warn(dev, "FWTracer: Acquire tracer ownership failed %d\n",
106 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
114 static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer)
116 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
118 mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
119 MLX5_FW_TRACER_RELEASE_OWNERSHIP);
120 tracer->owner = false;
123 static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer)
125 struct mlx5_core_dev *dev = tracer->dev;
126 struct device *ddev = &dev->pdev->dev;
132 tracer->buff.size = TRACE_BUFFER_SIZE_BYTE;
134 gfp = GFP_KERNEL | __GFP_ZERO;
135 buff = (void *)__get_free_pages(gfp,
136 get_order(tracer->buff.size));
139 mlx5_core_warn(dev, "FWTracer: Failed to allocate pages, %d\n", err);
142 tracer->buff.log_buf = buff;
144 dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE);
145 if (dma_mapping_error(ddev, dma)) {
146 mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n",
147 dma_mapping_error(ddev, dma));
151 tracer->buff.dma = dma;
156 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
161 static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer)
163 struct mlx5_core_dev *dev = tracer->dev;
164 struct device *ddev = &dev->pdev->dev;
166 if (!tracer->buff.log_buf)
169 dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE);
170 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
173 static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer *tracer)
175 struct mlx5_core_dev *dev = tracer->dev;
181 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
182 sizeof(*mtt) * round_up(TRACER_BUFFER_PAGE_NUM, 2);
184 in = kvzalloc(inlen, GFP_KERNEL);
188 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
189 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
190 mtt = (u64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
191 for (i = 0 ; i < TRACER_BUFFER_PAGE_NUM ; i++)
192 mtt[i] = cpu_to_be64(tracer->buff.dma + i * PAGE_SIZE);
194 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
195 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
196 MLX5_SET(mkc, mkc, lr, 1);
197 MLX5_SET(mkc, mkc, lw, 1);
198 MLX5_SET(mkc, mkc, pd, tracer->buff.pdn);
199 MLX5_SET(mkc, mkc, bsf_octword_size, 0);
200 MLX5_SET(mkc, mkc, qpn, 0xffffff);
201 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
202 MLX5_SET(mkc, mkc, translations_octword_size,
203 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
204 MLX5_SET64(mkc, mkc, start_addr, tracer->buff.dma);
205 MLX5_SET64(mkc, mkc, len, tracer->buff.size);
206 err = mlx5_core_create_mkey(dev, &tracer->buff.mkey, in, inlen);
208 mlx5_core_warn(dev, "FWTracer: Failed to create mkey, %d\n", err);
215 static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer *tracer)
217 u32 num_string_db = tracer->str_db.num_string_db;
220 for (i = 0; i < num_string_db; i++) {
221 kfree(tracer->str_db.buffer[i]);
222 tracer->str_db.buffer[i] = NULL;
226 static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer)
228 u32 *string_db_size_out = tracer->str_db.size_out;
229 u32 num_string_db = tracer->str_db.num_string_db;
232 for (i = 0; i < num_string_db; i++) {
233 tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL);
234 if (!tracer->str_db.buffer[i])
235 goto free_strings_db;
241 mlx5_fw_tracer_free_strings_db(tracer);
245 static void mlx5_tracer_read_strings_db(struct work_struct *work)
247 struct mlx5_fw_tracer *tracer = container_of(work, struct mlx5_fw_tracer,
248 read_fw_strings_work);
249 u32 num_of_reads, num_string_db = tracer->str_db.num_string_db;
250 struct mlx5_core_dev *dev = tracer->dev;
251 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
252 u32 leftovers, offset;
257 outlen = MLX5_ST_SZ_BYTES(mtrc_stdb) + STRINGS_DB_READ_SIZE_BYTES;
258 out = kzalloc(outlen, GFP_KERNEL);
264 for (i = 0; i < num_string_db; i++) {
266 MLX5_SET(mtrc_stdb, in, string_db_index, i);
267 num_of_reads = tracer->str_db.size_out[i] /
268 STRINGS_DB_READ_SIZE_BYTES;
269 leftovers = (tracer->str_db.size_out[i] %
270 STRINGS_DB_READ_SIZE_BYTES) /
271 STRINGS_DB_LEFTOVER_SIZE_BYTES;
273 MLX5_SET(mtrc_stdb, in, read_size, STRINGS_DB_READ_SIZE_BYTES);
274 for (j = 0; j < num_of_reads; j++) {
275 MLX5_SET(mtrc_stdb, in, start_offset, offset);
277 err = mlx5_core_access_reg(dev, in, sizeof(in), out,
278 outlen, MLX5_REG_MTRC_STDB,
281 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
286 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
287 memcpy(tracer->str_db.buffer[i] + offset, out_value,
288 STRINGS_DB_READ_SIZE_BYTES);
289 offset += STRINGS_DB_READ_SIZE_BYTES;
292 /* Strings database is aligned to 64, need to read leftovers*/
293 MLX5_SET(mtrc_stdb, in, read_size,
294 STRINGS_DB_LEFTOVER_SIZE_BYTES);
295 for (j = 0; j < leftovers; j++) {
296 MLX5_SET(mtrc_stdb, in, start_offset, offset);
298 err = mlx5_core_access_reg(dev, in, sizeof(in), out,
299 outlen, MLX5_REG_MTRC_STDB,
302 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
307 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
308 memcpy(tracer->str_db.buffer[i] + offset, out_value,
309 STRINGS_DB_LEFTOVER_SIZE_BYTES);
310 offset += STRINGS_DB_LEFTOVER_SIZE_BYTES;
314 tracer->str_db.loaded = true;
322 static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev)
324 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
325 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
328 MLX5_SET(mtrc_ctrl, in, arm_event, 1);
330 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
331 MLX5_REG_MTRC_CTRL, 0, 1);
333 mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err);
336 static const char *VAL_PARM = "%llx";
337 static const char *REPLACE_64_VAL_PARM = "%x%x";
338 static const char *PARAM_CHAR = "%";
340 static int mlx5_tracer_message_hash(u32 message_id)
342 return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1);
345 static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer,
346 struct tracer_event *tracer_event)
348 struct hlist_head *head =
349 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
350 struct tracer_string_format *cur_string;
352 cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL);
356 hlist_add_head(&cur_string->hlist, head);
361 static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer,
362 struct tracer_event *tracer_event)
364 struct tracer_string_format *cur_string;
368 str_ptr = tracer_event->string_event.string_param;
370 for (i = 0; i < tracer->str_db.num_string_db; i++) {
371 if (str_ptr > tracer->str_db.base_address_out[i] &&
372 str_ptr < tracer->str_db.base_address_out[i] +
373 tracer->str_db.size_out[i]) {
374 offset = str_ptr - tracer->str_db.base_address_out[i];
375 /* add it to the hash */
376 cur_string = mlx5_tracer_message_insert(tracer, tracer_event);
379 cur_string->string = (char *)(tracer->str_db.buffer[i] +
388 static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt)
390 hlist_del(&str_frmt->hlist);
394 static int mlx5_tracer_get_num_of_params(char *str)
396 char *substr, *pstr = str;
397 int num_of_params = 0;
399 /* replace %llx with %x%x */
400 substr = strstr(pstr, VAL_PARM);
402 memcpy(substr, REPLACE_64_VAL_PARM, 4);
404 substr = strstr(pstr, VAL_PARM);
407 /* count all the % characters */
408 substr = strstr(str, PARAM_CHAR);
412 substr = strstr(str, PARAM_CHAR);
415 return num_of_params;
418 static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head,
419 u8 event_id, u32 tmsn)
421 struct tracer_string_format *message;
423 hlist_for_each_entry(message, head, hlist)
424 if (message->event_id == event_id && message->tmsn == tmsn)
430 static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer,
431 struct tracer_event *tracer_event)
433 struct hlist_head *head =
434 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
436 return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn);
439 static void poll_trace(struct mlx5_fw_tracer *tracer,
440 struct tracer_event *tracer_event, u64 *trace)
442 u32 timestamp_low, timestamp_mid, timestamp_high, urts;
444 tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id);
445 tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost);
447 switch (tracer_event->event_id) {
448 case TRACER_EVENT_TYPE_TIMESTAMP:
449 tracer_event->type = TRACER_EVENT_TYPE_TIMESTAMP;
450 urts = MLX5_GET(tracer_timestamp_event, trace, urts);
451 if (tracer->trc_ver == 0)
452 tracer_event->timestamp_event.unreliable = !!(urts >> 2);
454 tracer_event->timestamp_event.unreliable = !!(urts & 1);
456 timestamp_low = MLX5_GET(tracer_timestamp_event,
457 trace, timestamp7_0);
458 timestamp_mid = MLX5_GET(tracer_timestamp_event,
459 trace, timestamp39_8);
460 timestamp_high = MLX5_GET(tracer_timestamp_event,
461 trace, timestamp52_40);
463 tracer_event->timestamp_event.timestamp =
464 ((u64)timestamp_high << 40) |
465 ((u64)timestamp_mid << 8) |
469 if (tracer_event->event_id >= tracer->str_db.first_string_trace &&
470 tracer_event->event_id <= tracer->str_db.first_string_trace +
471 tracer->str_db.num_string_trace) {
472 tracer_event->type = TRACER_EVENT_TYPE_STRING;
473 tracer_event->string_event.timestamp =
474 MLX5_GET(tracer_string_event, trace, timestamp);
475 tracer_event->string_event.string_param =
476 MLX5_GET(tracer_string_event, trace, string_param);
477 tracer_event->string_event.tmsn =
478 MLX5_GET(tracer_string_event, trace, tmsn);
479 tracer_event->string_event.tdsn =
480 MLX5_GET(tracer_string_event, trace, tdsn);
482 tracer_event->type = TRACER_EVENT_TYPE_UNRECOGNIZED;
488 static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event)
490 struct tracer_event tracer_event;
493 event_id = MLX5_GET(tracer_event, ts_event, event_id);
495 if (event_id == TRACER_EVENT_TYPE_TIMESTAMP)
496 poll_trace(tracer, &tracer_event, ts_event);
498 tracer_event.timestamp_event.timestamp = 0;
500 return tracer_event.timestamp_event.timestamp;
503 static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer)
505 struct tracer_string_format *str_frmt;
506 struct hlist_node *n;
509 for (i = 0; i < MESSAGE_HASH_SIZE; i++) {
510 hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist)
511 mlx5_tracer_clean_message(str_frmt);
515 static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer)
517 struct tracer_string_format *str_frmt, *tmp_str;
519 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list,
521 list_del(&str_frmt->list);
524 static void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt,
525 struct mlx5_core_dev *dev,
530 snprintf(tmp, sizeof(tmp), str_frmt->string,
537 str_frmt->params[6]);
539 trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost,
540 str_frmt->event_id, tmp);
542 /* remove it from hash */
543 mlx5_tracer_clean_message(str_frmt);
546 static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
547 struct tracer_event *tracer_event)
549 struct tracer_string_format *cur_string;
551 if (tracer_event->string_event.tdsn == 0) {
552 cur_string = mlx5_tracer_get_string(tracer, tracer_event);
556 cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string);
557 cur_string->last_param_num = 0;
558 cur_string->event_id = tracer_event->event_id;
559 cur_string->tmsn = tracer_event->string_event.tmsn;
560 cur_string->timestamp = tracer_event->string_event.timestamp;
561 cur_string->lost = tracer_event->lost_event;
562 if (cur_string->num_of_params == 0) /* trace with no params */
563 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
565 cur_string = mlx5_tracer_message_get(tracer, tracer_event);
567 pr_debug("%s Got string event for unknown string tmsn: %d\n",
568 __func__, tracer_event->string_event.tmsn);
571 cur_string->last_param_num += 1;
572 if (cur_string->last_param_num > TRACER_MAX_PARAMS) {
573 pr_debug("%s Number of params exceeds the max (%d)\n",
574 __func__, TRACER_MAX_PARAMS);
575 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
578 /* keep the new parameter */
579 cur_string->params[cur_string->last_param_num - 1] =
580 tracer_event->string_event.string_param;
581 if (cur_string->last_param_num == cur_string->num_of_params)
582 list_add_tail(&cur_string->list, &tracer->ready_strings_list);
588 static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer,
589 struct tracer_event *tracer_event)
591 struct tracer_timestamp_event timestamp_event =
592 tracer_event->timestamp_event;
593 struct tracer_string_format *str_frmt, *tmp_str;
594 struct mlx5_core_dev *dev = tracer->dev;
597 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) {
598 list_del(&str_frmt->list);
599 if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0))
600 trace_timestamp = (timestamp_event.timestamp & MASK_52_7) |
601 (str_frmt->timestamp & MASK_6_0);
603 trace_timestamp = ((timestamp_event.timestamp - 1) & MASK_52_7) |
604 (str_frmt->timestamp & MASK_6_0);
606 mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp);
610 static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer,
611 struct tracer_event *tracer_event)
613 if (tracer_event->type == TRACER_EVENT_TYPE_STRING) {
614 mlx5_tracer_handle_string_trace(tracer, tracer_event);
615 } else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) {
616 if (!tracer_event->timestamp_event.unreliable)
617 mlx5_tracer_handle_timestamp_trace(tracer, tracer_event);
619 pr_debug("%s Got unrecognised type %d for parsing, exiting..\n",
620 __func__, tracer_event->type);
625 static void mlx5_fw_tracer_handle_traces(struct work_struct *work)
627 struct mlx5_fw_tracer *tracer =
628 container_of(work, struct mlx5_fw_tracer, handle_traces_work);
629 u64 block_timestamp, last_block_timestamp, tmp_trace_block[TRACES_PER_BLOCK];
630 u32 block_count, start_offset, prev_start_offset, prev_consumer_index;
631 u32 trace_event_size = MLX5_ST_SZ_BYTES(tracer_event);
632 struct mlx5_core_dev *dev = tracer->dev;
633 struct tracer_event tracer_event;
636 mlx5_core_dbg(dev, "FWTracer: Handle Trace event, owner=(%d)\n", tracer->owner);
640 if (unlikely(!tracer->str_db.loaded))
643 block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE;
644 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
646 /* Copy the block to local buffer to avoid HW override while being processed*/
647 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
648 TRACER_BLOCK_SIZE_BYTE);
651 get_block_timestamp(tracer, &tmp_trace_block[TRACES_PER_BLOCK - 1]);
653 while (block_timestamp > tracer->last_timestamp) {
654 /* Check block override if it's not the first block */
655 if (tracer->last_timestamp) {
657 /* To avoid block override be the HW in case of buffer
658 * wraparound, the time stamp of the previous block
659 * should be compared to the last timestamp handled
662 prev_consumer_index =
663 (tracer->buff.consumer_index - 1) & (block_count - 1);
664 prev_start_offset = prev_consumer_index * TRACER_BLOCK_SIZE_BYTE;
666 ts_event = tracer->buff.log_buf + prev_start_offset +
667 (TRACES_PER_BLOCK - 1) * trace_event_size;
668 last_block_timestamp = get_block_timestamp(tracer, ts_event);
669 /* If previous timestamp different from last stored
670 * timestamp then there is a good chance that the
671 * current buffer is overwritten and therefore should
674 if (tracer->last_timestamp != last_block_timestamp) {
675 mlx5_core_warn(dev, "FWTracer: Events were lost\n");
676 tracer->last_timestamp = block_timestamp;
677 tracer->buff.consumer_index =
678 (tracer->buff.consumer_index + 1) & (block_count - 1);
684 for (i = 0; i < TRACES_PER_BLOCK ; i++) {
685 poll_trace(tracer, &tracer_event, &tmp_trace_block[i]);
686 mlx5_tracer_handle_trace(tracer, &tracer_event);
689 tracer->buff.consumer_index =
690 (tracer->buff.consumer_index + 1) & (block_count - 1);
692 tracer->last_timestamp = block_timestamp;
693 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
694 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
695 TRACER_BLOCK_SIZE_BYTE);
696 block_timestamp = get_block_timestamp(tracer,
697 &tmp_trace_block[TRACES_PER_BLOCK - 1]);
701 mlx5_fw_tracer_arm(dev);
704 static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer)
706 struct mlx5_core_dev *dev = tracer->dev;
707 u32 out[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
708 u32 in[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
711 MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY);
712 MLX5_SET(mtrc_conf, in, log_trace_buffer_size,
713 ilog2(TRACER_BUFFER_PAGE_NUM));
714 MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey.key);
716 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
717 MLX5_REG_MTRC_CONF, 0, 1);
719 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err);
724 static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer *tracer, u8 status, u8 arm)
726 struct mlx5_core_dev *dev = tracer->dev;
727 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
728 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
731 MLX5_SET(mtrc_ctrl, in, modify_field_select, TRACE_STATUS);
732 MLX5_SET(mtrc_ctrl, in, trace_status, status);
733 MLX5_SET(mtrc_ctrl, in, arm_event, arm);
735 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
736 MLX5_REG_MTRC_CTRL, 0, 1);
739 tracer->last_timestamp = 0;
744 static int mlx5_fw_tracer_start(struct mlx5_fw_tracer *tracer)
746 struct mlx5_core_dev *dev = tracer->dev;
749 err = mlx5_fw_tracer_ownership_acquire(tracer);
751 mlx5_core_dbg(dev, "FWTracer: Ownership was not granted %d\n", err);
752 /* Don't fail since ownership can be acquired on a later FW event */
756 err = mlx5_fw_tracer_set_mtrc_conf(tracer);
758 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configuration %d\n", err);
759 goto release_ownership;
762 /* enable tracer & trace events */
763 err = mlx5_fw_tracer_set_mtrc_ctrl(tracer, 1, 1);
765 mlx5_core_warn(dev, "FWTracer: Failed to enable tracer %d\n", err);
766 goto release_ownership;
769 mlx5_core_dbg(dev, "FWTracer: Ownership granted and active\n");
773 mlx5_fw_tracer_ownership_release(tracer);
777 static void mlx5_fw_tracer_ownership_change(struct work_struct *work)
779 struct mlx5_fw_tracer *tracer =
780 container_of(work, struct mlx5_fw_tracer, ownership_change_work);
782 mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner);
784 tracer->owner = false;
785 tracer->buff.consumer_index = 0;
789 mlx5_fw_tracer_start(tracer);
792 /* Create software resources (Buffers, etc ..) */
793 struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev)
795 struct mlx5_fw_tracer *tracer = NULL;
798 if (!MLX5_CAP_MCAM_REG(dev, tracer_registers)) {
799 mlx5_core_dbg(dev, "FWTracer: Tracer capability not present\n");
803 tracer = kvzalloc(sizeof(*tracer), GFP_KERNEL);
805 return ERR_PTR(-ENOMEM);
807 tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer");
808 if (!tracer->work_queue) {
815 INIT_LIST_HEAD(&tracer->ready_strings_list);
816 INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change);
817 INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db);
818 INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces);
821 err = mlx5_query_mtrc_caps(tracer);
823 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err);
824 goto destroy_workqueue;
827 err = mlx5_fw_tracer_create_log_buf(tracer);
829 mlx5_core_warn(dev, "FWTracer: Create log buffer failed %d\n", err);
830 goto destroy_workqueue;
833 err = mlx5_fw_tracer_allocate_strings_db(tracer);
835 mlx5_core_warn(dev, "FWTracer: Allocate strings database failed %d\n", err);
839 mlx5_core_dbg(dev, "FWTracer: Tracer created\n");
844 mlx5_fw_tracer_destroy_log_buf(tracer);
847 destroy_workqueue(tracer->work_queue);
853 /* Create HW resources + start tracer
854 * must be called before Async EQ is created
856 int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer)
858 struct mlx5_core_dev *dev;
861 if (IS_ERR_OR_NULL(tracer))
866 if (!tracer->str_db.loaded)
867 queue_work(tracer->work_queue, &tracer->read_fw_strings_work);
869 err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn);
871 mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err);
875 err = mlx5_fw_tracer_create_mkey(tracer);
877 mlx5_core_warn(dev, "FWTracer: Failed to create mkey %d\n", err);
881 mlx5_fw_tracer_start(tracer);
886 mlx5_core_dealloc_pd(dev, tracer->buff.pdn);
890 /* Stop tracer + Cleanup HW resources
891 * must be called after Async EQ is destroyed
893 void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer)
895 if (IS_ERR_OR_NULL(tracer))
898 mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n",
901 cancel_work_sync(&tracer->ownership_change_work);
902 cancel_work_sync(&tracer->handle_traces_work);
905 mlx5_fw_tracer_ownership_release(tracer);
907 mlx5_core_destroy_mkey(tracer->dev, &tracer->buff.mkey);
908 mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn);
911 /* Free software resources (Buffers, etc ..) */
912 void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer)
914 if (IS_ERR_OR_NULL(tracer))
917 mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n");
919 cancel_work_sync(&tracer->read_fw_strings_work);
920 mlx5_fw_tracer_clean_ready_list(tracer);
921 mlx5_fw_tracer_clean_print_hash(tracer);
922 mlx5_fw_tracer_free_strings_db(tracer);
923 mlx5_fw_tracer_destroy_log_buf(tracer);
924 flush_workqueue(tracer->work_queue);
925 destroy_workqueue(tracer->work_queue);
929 void mlx5_fw_tracer_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe)
931 struct mlx5_fw_tracer *tracer = dev->tracer;
936 switch (eqe->sub_type) {
937 case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE:
938 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state))
939 queue_work(tracer->work_queue, &tracer->ownership_change_work);
941 case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE:
942 queue_work(tracer->work_queue, &tracer->handle_traces_work);
945 mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n",
950 EXPORT_TRACEPOINT_SYMBOL(mlx5_fw);