1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2019 Mellanox Technologies. All rights reserved.
5 #include <rdma/ib_verbs.h>
6 #include <rdma/rdma_counter.h>
11 #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID)
13 static int __counter_set_mode(struct rdma_port_counter *port_counter,
14 enum rdma_nl_counter_mode new_mode,
15 enum rdma_nl_counter_mask new_mask)
17 if (new_mode == RDMA_COUNTER_MODE_AUTO) {
18 if (new_mask & (~ALL_AUTO_MODE_MASKS))
20 if (port_counter->num_counters)
24 port_counter->mode.mode = new_mode;
25 port_counter->mode.mask = new_mask;
30 * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode
32 * @dev: Device to operate
34 * @mask: Mask to configure
35 * @extack: Message to the user
37 * Return 0 on success. If counter mode wasn't changed then it is considered
39 * Return -EBUSY when changing to auto mode while there are bounded counters.
42 int rdma_counter_set_auto_mode(struct ib_device *dev, u32 port,
43 enum rdma_nl_counter_mask mask,
44 struct netlink_ext_ack *extack)
46 struct rdma_port_counter *port_counter;
47 enum rdma_nl_counter_mode mode;
50 port_counter = &dev->port_data[port].port_counter;
51 if (!port_counter->hstats)
54 mutex_lock(&port_counter->lock);
56 mode = RDMA_COUNTER_MODE_AUTO;
58 mode = (port_counter->num_counters) ? RDMA_COUNTER_MODE_MANUAL :
59 RDMA_COUNTER_MODE_NONE;
61 if (port_counter->mode.mode == mode &&
62 port_counter->mode.mask == mask) {
67 ret = __counter_set_mode(port_counter, mode, mask);
70 mutex_unlock(&port_counter->lock);
74 "Modifying auto mode is not allowed when there is a bound QP");
78 static void auto_mode_init_counter(struct rdma_counter *counter,
79 const struct ib_qp *qp,
80 enum rdma_nl_counter_mask new_mask)
82 struct auto_mode_param *param = &counter->mode.param;
84 counter->mode.mode = RDMA_COUNTER_MODE_AUTO;
85 counter->mode.mask = new_mask;
87 if (new_mask & RDMA_COUNTER_MASK_QP_TYPE)
88 param->qp_type = qp->qp_type;
91 static int __rdma_counter_bind_qp(struct rdma_counter *counter,
99 if (!qp->device->ops.counter_bind_qp)
102 mutex_lock(&counter->lock);
103 ret = qp->device->ops.counter_bind_qp(counter, qp);
104 mutex_unlock(&counter->lock);
109 int rdma_counter_modify(struct ib_device *dev, u32 port,
110 unsigned int index, bool enable)
112 struct rdma_hw_stats *stats;
115 if (!dev->ops.modify_hw_stat)
118 stats = ib_get_hw_stats_port(dev, port);
119 if (!stats || index >= stats->num_counters ||
120 !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL))
123 mutex_lock(&stats->lock);
125 if (enable != test_bit(index, stats->is_disabled))
128 ret = dev->ops.modify_hw_stat(dev, port, index, enable);
133 clear_bit(index, stats->is_disabled);
135 set_bit(index, stats->is_disabled);
137 mutex_unlock(&stats->lock);
141 static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port,
143 enum rdma_nl_counter_mode mode)
145 struct rdma_port_counter *port_counter;
146 struct rdma_counter *counter;
149 if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats)
152 counter = kzalloc(sizeof(*counter), GFP_KERNEL);
156 counter->device = dev;
157 counter->port = port;
159 rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER);
160 counter->stats = dev->ops.counter_alloc_stats(counter);
164 port_counter = &dev->port_data[port].port_counter;
165 mutex_lock(&port_counter->lock);
167 case RDMA_COUNTER_MODE_MANUAL:
168 ret = __counter_set_mode(port_counter, RDMA_COUNTER_MODE_MANUAL,
171 mutex_unlock(&port_counter->lock);
175 case RDMA_COUNTER_MODE_AUTO:
176 auto_mode_init_counter(counter, qp, port_counter->mode.mask);
180 mutex_unlock(&port_counter->lock);
184 port_counter->num_counters++;
185 mutex_unlock(&port_counter->lock);
187 counter->mode.mode = mode;
188 kref_init(&counter->kref);
189 mutex_init(&counter->lock);
191 ret = __rdma_counter_bind_qp(counter, qp);
195 rdma_restrack_parent_name(&counter->res, &qp->res);
196 rdma_restrack_add(&counter->res);
200 rdma_free_hw_stats_struct(counter->stats);
202 rdma_restrack_put(&counter->res);
207 static void rdma_counter_free(struct rdma_counter *counter)
209 struct rdma_port_counter *port_counter;
211 port_counter = &counter->device->port_data[counter->port].port_counter;
212 mutex_lock(&port_counter->lock);
213 port_counter->num_counters--;
214 if (!port_counter->num_counters &&
215 (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL))
216 __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0);
218 mutex_unlock(&port_counter->lock);
220 rdma_restrack_del(&counter->res);
221 rdma_free_hw_stats_struct(counter->stats);
225 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,
226 enum rdma_nl_counter_mask auto_mask)
228 struct auto_mode_param *param = &counter->mode.param;
231 if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE)
232 match &= (param->qp_type == qp->qp_type);
234 if (auto_mask & RDMA_COUNTER_MASK_PID)
235 match &= (task_pid_nr(counter->res.task) ==
236 task_pid_nr(qp->res.task));
241 static int __rdma_counter_unbind_qp(struct ib_qp *qp)
243 struct rdma_counter *counter = qp->counter;
246 if (!qp->device->ops.counter_unbind_qp)
249 mutex_lock(&counter->lock);
250 ret = qp->device->ops.counter_unbind_qp(qp);
251 mutex_unlock(&counter->lock);
256 static void counter_history_stat_update(struct rdma_counter *counter)
258 struct ib_device *dev = counter->device;
259 struct rdma_port_counter *port_counter;
262 port_counter = &dev->port_data[counter->port].port_counter;
263 if (!port_counter->hstats)
266 rdma_counter_query_stats(counter);
268 for (i = 0; i < counter->stats->num_counters; i++)
269 port_counter->hstats->value[i] += counter->stats->value[i];
273 * rdma_get_counter_auto_mode - Find the counter that @qp should be bound
276 * Return: The counter (with ref-count increased) if found
278 static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp,
281 struct rdma_port_counter *port_counter;
282 struct rdma_counter *counter = NULL;
283 struct ib_device *dev = qp->device;
284 struct rdma_restrack_entry *res;
285 struct rdma_restrack_root *rt;
286 unsigned long id = 0;
288 port_counter = &dev->port_data[port].port_counter;
289 rt = &dev->res[RDMA_RESTRACK_COUNTER];
291 xa_for_each(&rt->xa, id, res) {
292 counter = container_of(res, struct rdma_counter, res);
293 if ((counter->device != qp->device) || (counter->port != port))
296 if (auto_mode_match(qp, counter, port_counter->mode.mask))
302 if (counter && !kref_get_unless_zero(&counter->kref))
309 static void counter_release(struct kref *kref)
311 struct rdma_counter *counter;
313 counter = container_of(kref, struct rdma_counter, kref);
314 counter_history_stat_update(counter);
315 counter->device->ops.counter_dealloc(counter);
316 rdma_counter_free(counter);
320 * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on
323 int rdma_counter_bind_qp_auto(struct ib_qp *qp, u32 port)
325 struct rdma_port_counter *port_counter;
326 struct ib_device *dev = qp->device;
327 struct rdma_counter *counter;
330 if (!rdma_restrack_is_tracked(&qp->res) || rdma_is_kernel_res(&qp->res))
333 if (!rdma_is_port_valid(dev, port))
336 port_counter = &dev->port_data[port].port_counter;
337 if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO)
340 counter = rdma_get_counter_auto_mode(qp, port);
342 ret = __rdma_counter_bind_qp(counter, qp);
344 kref_put(&counter->kref, counter_release);
348 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO);
357 * rdma_counter_unbind_qp - Unbind a qp from a counter
359 * true - Decrease the counter ref-count anyway (e.g., qp destroy)
361 int rdma_counter_unbind_qp(struct ib_qp *qp, bool force)
363 struct rdma_counter *counter = qp->counter;
369 ret = __rdma_counter_unbind_qp(qp);
373 kref_put(&counter->kref, counter_release);
377 int rdma_counter_query_stats(struct rdma_counter *counter)
379 struct ib_device *dev = counter->device;
382 if (!dev->ops.counter_update_stats)
385 mutex_lock(&counter->lock);
386 ret = dev->ops.counter_update_stats(counter);
387 mutex_unlock(&counter->lock);
392 static u64 get_running_counters_hwstat_sum(struct ib_device *dev,
395 struct rdma_restrack_entry *res;
396 struct rdma_restrack_root *rt;
397 struct rdma_counter *counter;
398 unsigned long id = 0;
401 rt = &dev->res[RDMA_RESTRACK_COUNTER];
403 xa_for_each(&rt->xa, id, res) {
404 if (!rdma_restrack_get(res))
409 counter = container_of(res, struct rdma_counter, res);
410 if ((counter->device != dev) || (counter->port != port) ||
411 rdma_counter_query_stats(counter))
414 sum += counter->stats->value[index];
418 rdma_restrack_put(res);
426 * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a
427 * specific port, including the running ones and history data
429 u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u32 port, u32 index)
431 struct rdma_port_counter *port_counter;
434 port_counter = &dev->port_data[port].port_counter;
435 if (!port_counter->hstats)
438 sum = get_running_counters_hwstat_sum(dev, port, index);
439 sum += port_counter->hstats->value[index];
444 static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num)
446 struct rdma_restrack_entry *res = NULL;
447 struct ib_qp *qp = NULL;
449 res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num);
453 qp = container_of(res, struct ib_qp, res);
454 if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
460 rdma_restrack_put(res);
464 static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev,
467 struct rdma_restrack_entry *res;
468 struct rdma_counter *counter;
470 res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id);
474 counter = container_of(res, struct rdma_counter, res);
475 kref_get(&counter->kref);
476 rdma_restrack_put(res);
482 * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id
484 int rdma_counter_bind_qpn(struct ib_device *dev, u32 port,
485 u32 qp_num, u32 counter_id)
487 struct rdma_port_counter *port_counter;
488 struct rdma_counter *counter;
492 port_counter = &dev->port_data[port].port_counter;
493 if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)
496 qp = rdma_counter_get_qp(dev, qp_num);
500 counter = rdma_get_counter_by_id(dev, counter_id);
506 if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) {
511 if ((counter->device != qp->device) || (counter->port != qp->port)) {
516 ret = __rdma_counter_bind_qp(counter, qp);
520 rdma_restrack_put(&qp->res);
524 kref_put(&counter->kref, counter_release);
526 rdma_restrack_put(&qp->res);
531 * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it
532 * The id of new counter is returned in @counter_id
534 int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u32 port,
535 u32 qp_num, u32 *counter_id)
537 struct rdma_port_counter *port_counter;
538 struct rdma_counter *counter;
542 if (!rdma_is_port_valid(dev, port))
545 port_counter = &dev->port_data[port].port_counter;
546 if (!port_counter->hstats)
549 if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)
552 qp = rdma_counter_get_qp(dev, qp_num);
556 if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {
561 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL);
568 *counter_id = counter->id;
570 rdma_restrack_put(&qp->res);
574 rdma_restrack_put(&qp->res);
579 * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter
581 int rdma_counter_unbind_qpn(struct ib_device *dev, u32 port,
582 u32 qp_num, u32 counter_id)
584 struct rdma_port_counter *port_counter;
588 if (!rdma_is_port_valid(dev, port))
591 qp = rdma_counter_get_qp(dev, qp_num);
595 if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {
600 port_counter = &dev->port_data[port].port_counter;
601 if (!qp->counter || qp->counter->id != counter_id ||
602 port_counter->mode.mode != RDMA_COUNTER_MODE_MANUAL) {
607 ret = rdma_counter_unbind_qp(qp, false);
610 rdma_restrack_put(&qp->res);
614 int rdma_counter_get_mode(struct ib_device *dev, u32 port,
615 enum rdma_nl_counter_mode *mode,
616 enum rdma_nl_counter_mask *mask)
618 struct rdma_port_counter *port_counter;
620 port_counter = &dev->port_data[port].port_counter;
621 *mode = port_counter->mode.mode;
622 *mask = port_counter->mode.mask;
627 void rdma_counter_init(struct ib_device *dev)
629 struct rdma_port_counter *port_counter;
635 rdma_for_each_port(dev, port) {
636 port_counter = &dev->port_data[port].port_counter;
637 port_counter->mode.mode = RDMA_COUNTER_MODE_NONE;
638 mutex_init(&port_counter->lock);
640 if (!dev->ops.alloc_hw_port_stats)
643 port_counter->hstats = dev->ops.alloc_hw_port_stats(dev, port);
644 if (!port_counter->hstats)
651 for (i = port; i >= rdma_start_port(dev); i--) {
652 port_counter = &dev->port_data[port].port_counter;
653 rdma_free_hw_stats_struct(port_counter->hstats);
654 port_counter->hstats = NULL;
655 mutex_destroy(&port_counter->lock);
659 void rdma_counter_release(struct ib_device *dev)
661 struct rdma_port_counter *port_counter;
664 rdma_for_each_port(dev, port) {
665 port_counter = &dev->port_data[port].port_counter;
666 rdma_free_hw_stats_struct(port_counter->hstats);
667 mutex_destroy(&port_counter->lock);