core/nldev.c

   1 /*
   2  * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are met:
   6  *
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  * 3. Neither the names of the copyright holders nor the names of its
  13  *    contributors may be used to endorse or promote products derived from
  14  *    this software without specific prior written permission.
  15  *
  16  * Alternatively, this software may be distributed under the terms of the
  17  * GNU General Public License ("GPL") version 2 as published by the Free
  18  * Software Foundation.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <linux/module.h>
  34 #include <linux/pid.h>
  35 #include <linux/pid_namespace.h>
  36 #include <linux/mutex.h>
  37 #include <net/netlink.h>
  38 #include <rdma/rdma_cm.h>
  39 #include <rdma/rdma_netlink.h>
  40
  41 #include "core_priv.h"
  42 #include "cma_priv.h"
  43 #include "restrack.h"
  44 #include "uverbs.h"
  45
  46 /*
  47  * This determines whether a non-privileged user is allowed to specify a
  48  * controlled QKEY or not, when true non-privileged user is allowed to specify
  49  * a controlled QKEY.
  50  */
  51 static bool privileged_qkey;
  52
  53 typedef int (*res_fill_func_t)(struct sk_buff*, bool,
  54                                struct rdma_restrack_entry*, uint32_t);
  55
  56 /*
  57  * Sort array elements by the netlink attribute name
  58  */
  59 static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
  60         [RDMA_NLDEV_ATTR_CHARDEV]               = { .type = NLA_U64 },
  61         [RDMA_NLDEV_ATTR_CHARDEV_ABI]           = { .type = NLA_U64 },
  62         [RDMA_NLDEV_ATTR_CHARDEV_NAME]          = { .type = NLA_NUL_STRING,
  63                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  64         [RDMA_NLDEV_ATTR_CHARDEV_TYPE]          = { .type = NLA_NUL_STRING,
  65                                         .len = RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE },
  66         [RDMA_NLDEV_ATTR_DEV_DIM]               = { .type = NLA_U8 },
  67         [RDMA_NLDEV_ATTR_DEV_INDEX]             = { .type = NLA_U32 },
  68         [RDMA_NLDEV_ATTR_DEV_NAME]              = { .type = NLA_NUL_STRING,
  69                                         .len = IB_DEVICE_NAME_MAX },
  70         [RDMA_NLDEV_ATTR_DEV_NODE_TYPE]         = { .type = NLA_U8 },
  71         [RDMA_NLDEV_ATTR_DEV_PROTOCOL]          = { .type = NLA_NUL_STRING,
  72                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  73         [RDMA_NLDEV_ATTR_DRIVER]                = { .type = NLA_NESTED },
  74         [RDMA_NLDEV_ATTR_DRIVER_ENTRY]          = { .type = NLA_NESTED },
  75         [RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE]     = { .type = NLA_U8 },
  76         [RDMA_NLDEV_ATTR_DRIVER_STRING]         = { .type = NLA_NUL_STRING,
  77                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  78         [RDMA_NLDEV_ATTR_DRIVER_S32]            = { .type = NLA_S32 },
  79         [RDMA_NLDEV_ATTR_DRIVER_S64]            = { .type = NLA_S64 },
  80         [RDMA_NLDEV_ATTR_DRIVER_U32]            = { .type = NLA_U32 },
  81         [RDMA_NLDEV_ATTR_DRIVER_U64]            = { .type = NLA_U64 },
  82         [RDMA_NLDEV_ATTR_FW_VERSION]            = { .type = NLA_NUL_STRING,
  83                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
  84         [RDMA_NLDEV_ATTR_LID]                   = { .type = NLA_U32 },
  85         [RDMA_NLDEV_ATTR_LINK_TYPE]             = { .type = NLA_NUL_STRING,
  86                                         .len = IFNAMSIZ },
  87         [RDMA_NLDEV_ATTR_LMC]                   = { .type = NLA_U8 },
  88         [RDMA_NLDEV_ATTR_NDEV_INDEX]            = { .type = NLA_U32 },
  89         [RDMA_NLDEV_ATTR_NDEV_NAME]             = { .type = NLA_NUL_STRING,
  90                                         .len = IFNAMSIZ },
  91         [RDMA_NLDEV_ATTR_NODE_GUID]             = { .type = NLA_U64 },
  92         [RDMA_NLDEV_ATTR_PORT_INDEX]            = { .type = NLA_U32 },
  93         [RDMA_NLDEV_ATTR_PORT_PHYS_STATE]       = { .type = NLA_U8 },
  94         [RDMA_NLDEV_ATTR_PORT_STATE]            = { .type = NLA_U8 },
  95         [RDMA_NLDEV_ATTR_RES_CM_ID]             = { .type = NLA_NESTED },
  96         [RDMA_NLDEV_ATTR_RES_CM_IDN]            = { .type = NLA_U32 },
  97         [RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY]       = { .type = NLA_NESTED },
  98         [RDMA_NLDEV_ATTR_RES_CQ]                = { .type = NLA_NESTED },
  99         [RDMA_NLDEV_ATTR_RES_CQE]               = { .type = NLA_U32 },
 100         [RDMA_NLDEV_ATTR_RES_CQN]               = { .type = NLA_U32 },
 101         [RDMA_NLDEV_ATTR_RES_CQ_ENTRY]          = { .type = NLA_NESTED },
 102         [RDMA_NLDEV_ATTR_RES_CTX]               = { .type = NLA_NESTED },
 103         [RDMA_NLDEV_ATTR_RES_CTXN]              = { .type = NLA_U32 },
 104         [RDMA_NLDEV_ATTR_RES_CTX_ENTRY]         = { .type = NLA_NESTED },
 105         [RDMA_NLDEV_ATTR_RES_DST_ADDR]          = {
 106                         .len = sizeof(struct __kernel_sockaddr_storage) },
 107         [RDMA_NLDEV_ATTR_RES_IOVA]              = { .type = NLA_U64 },
 108         [RDMA_NLDEV_ATTR_RES_KERN_NAME]         = { .type = NLA_NUL_STRING,
 109                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
 110         [RDMA_NLDEV_ATTR_RES_LKEY]              = { .type = NLA_U32 },
 111         [RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY]    = { .type = NLA_U32 },
 112         [RDMA_NLDEV_ATTR_RES_LQPN]              = { .type = NLA_U32 },
 113         [RDMA_NLDEV_ATTR_RES_MR]                = { .type = NLA_NESTED },
 114         [RDMA_NLDEV_ATTR_RES_MRLEN]             = { .type = NLA_U64 },
 115         [RDMA_NLDEV_ATTR_RES_MRN]               = { .type = NLA_U32 },
 116         [RDMA_NLDEV_ATTR_RES_MR_ENTRY]          = { .type = NLA_NESTED },
 117         [RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE]    = { .type = NLA_U8 },
 118         [RDMA_NLDEV_ATTR_RES_PD]                = { .type = NLA_NESTED },
 119         [RDMA_NLDEV_ATTR_RES_PDN]               = { .type = NLA_U32 },
 120         [RDMA_NLDEV_ATTR_RES_PD_ENTRY]          = { .type = NLA_NESTED },
 121         [RDMA_NLDEV_ATTR_RES_PID]               = { .type = NLA_U32 },
 122         [RDMA_NLDEV_ATTR_RES_POLL_CTX]          = { .type = NLA_U8 },
 123         [RDMA_NLDEV_ATTR_RES_PS]                = { .type = NLA_U32 },
 124         [RDMA_NLDEV_ATTR_RES_QP]                = { .type = NLA_NESTED },
 125         [RDMA_NLDEV_ATTR_RES_QP_ENTRY]          = { .type = NLA_NESTED },
 126         [RDMA_NLDEV_ATTR_RES_RAW]               = { .type = NLA_BINARY },
 127         [RDMA_NLDEV_ATTR_RES_RKEY]              = { .type = NLA_U32 },
 128         [RDMA_NLDEV_ATTR_RES_RQPN]              = { .type = NLA_U32 },
 129         [RDMA_NLDEV_ATTR_RES_RQ_PSN]            = { .type = NLA_U32 },
 130         [RDMA_NLDEV_ATTR_RES_SQ_PSN]            = { .type = NLA_U32 },
 131         [RDMA_NLDEV_ATTR_RES_SRC_ADDR]          = {
 132                         .len = sizeof(struct __kernel_sockaddr_storage) },
 133         [RDMA_NLDEV_ATTR_RES_STATE]             = { .type = NLA_U8 },
 134         [RDMA_NLDEV_ATTR_RES_SUMMARY]           = { .type = NLA_NESTED },
 135         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]     = { .type = NLA_NESTED },
 136         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR]= { .type = NLA_U64 },
 137         [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING,
 138                                         .len = RDMA_NLDEV_ATTR_EMPTY_STRING },
 139         [RDMA_NLDEV_ATTR_RES_TYPE]              = { .type = NLA_U8 },
 140         [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 },
 141         [RDMA_NLDEV_ATTR_RES_USECNT]            = { .type = NLA_U64 },
 142         [RDMA_NLDEV_ATTR_RES_SRQ]               = { .type = NLA_NESTED },
 143         [RDMA_NLDEV_ATTR_RES_SRQN]              = { .type = NLA_U32 },
 144         [RDMA_NLDEV_ATTR_RES_SRQ_ENTRY]         = { .type = NLA_NESTED },
 145         [RDMA_NLDEV_ATTR_MIN_RANGE]             = { .type = NLA_U32 },
 146         [RDMA_NLDEV_ATTR_MAX_RANGE]             = { .type = NLA_U32 },
 147         [RDMA_NLDEV_ATTR_SM_LID]                = { .type = NLA_U32 },
 148         [RDMA_NLDEV_ATTR_SUBNET_PREFIX]         = { .type = NLA_U64 },
 149         [RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]   = { .type = NLA_U32 },
 150         [RDMA_NLDEV_ATTR_STAT_MODE]             = { .type = NLA_U32 },
 151         [RDMA_NLDEV_ATTR_STAT_RES]              = { .type = NLA_U32 },
 152         [RDMA_NLDEV_ATTR_STAT_COUNTER]          = { .type = NLA_NESTED },
 153         [RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY]    = { .type = NLA_NESTED },
 154         [RDMA_NLDEV_ATTR_STAT_COUNTER_ID]       = { .type = NLA_U32 },
 155         [RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]       = { .type = NLA_NESTED },
 156         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY]  = { .type = NLA_NESTED },
 157         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME] = { .type = NLA_NUL_STRING },
 158         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE] = { .type = NLA_U64 },
 159         [RDMA_NLDEV_ATTR_SYS_IMAGE_GUID]        = { .type = NLA_U64 },
 160         [RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID]      = { .type = NLA_U32 },
 161         [RDMA_NLDEV_NET_NS_FD]                  = { .type = NLA_U32 },
 162         [RDMA_NLDEV_SYS_ATTR_NETNS_MODE]        = { .type = NLA_U8 },
 163         [RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK]      = { .type = NLA_U8 },
 164         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX]  = { .type = NLA_U32 },
 165         [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC] = { .type = NLA_U8 },
 166         [RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE] = { .type = NLA_U8 },
 167 };
 168
 169 static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
 170                                       enum rdma_nldev_print_type print_type)
 171 {
 172         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, name))
 173                 return -EMSGSIZE;
 174         if (print_type != RDMA_NLDEV_PRINT_TYPE_UNSPEC &&
 175             nla_put_u8(msg, RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE, print_type))
 176                 return -EMSGSIZE;
 177
 178         return 0;
 179 }
 180
 181 static int _rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name,
 182                                    enum rdma_nldev_print_type print_type,
 183                                    u32 value)
 184 {
 185         if (put_driver_name_print_type(msg, name, print_type))
 186                 return -EMSGSIZE;
 187         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DRIVER_U32, value))
 188                 return -EMSGSIZE;
 189
 190         return 0;
 191 }
 192
 193 static int _rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name,
 194                                    enum rdma_nldev_print_type print_type,
 195                                    u64 value)
 196 {
 197         if (put_driver_name_print_type(msg, name, print_type))
 198                 return -EMSGSIZE;
 199         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_DRIVER_U64, value,
 200                               RDMA_NLDEV_ATTR_PAD))
 201                 return -EMSGSIZE;
 202
 203         return 0;
 204 }
 205
 206 int rdma_nl_put_driver_string(struct sk_buff *msg, const char *name,
 207                               const char *str)
 208 {
 209         if (put_driver_name_print_type(msg, name,
 210                                        RDMA_NLDEV_PRINT_TYPE_UNSPEC))
 211                 return -EMSGSIZE;
 212         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, str))
 213                 return -EMSGSIZE;
 214
 215         return 0;
 216 }
 217 EXPORT_SYMBOL(rdma_nl_put_driver_string);
 218
 219 int rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name, u32 value)
 220 {
 221         return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,
 222                                        value);
 223 }
 224 EXPORT_SYMBOL(rdma_nl_put_driver_u32);
 225
 226 int rdma_nl_put_driver_u32_hex(struct sk_buff *msg, const char *name,
 227                                u32 value)
 228 {
 229         return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX,
 230                                        value);
 231 }
 232 EXPORT_SYMBOL(rdma_nl_put_driver_u32_hex);
 233
 234 int rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name, u64 value)
 235 {
 236         return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC,
 237                                        value);
 238 }
 239 EXPORT_SYMBOL(rdma_nl_put_driver_u64);
 240
 241 int rdma_nl_put_driver_u64_hex(struct sk_buff *msg, const char *name, u64 value)
 242 {
 243         return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX,
 244                                        value);
 245 }
 246 EXPORT_SYMBOL(rdma_nl_put_driver_u64_hex);
 247
 248 bool rdma_nl_get_privileged_qkey(void)
 249 {
 250         return privileged_qkey || capable(CAP_NET_RAW);
 251 }
 252 EXPORT_SYMBOL(rdma_nl_get_privileged_qkey);
 253
 254 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
 255 {
 256         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index))
 257                 return -EMSGSIZE;
 258         if (nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME,
 259                            dev_name(&device->dev)))
 260                 return -EMSGSIZE;
 261
 262         return 0;
 263 }
 264
 265 static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
 266 {
 267         char fw[IB_FW_VERSION_NAME_MAX];
 268         int ret = 0;
 269         u32 port;
 270
 271         if (fill_nldev_handle(msg, device))
 272                 return -EMSGSIZE;
 273
 274         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, rdma_end_port(device)))
 275                 return -EMSGSIZE;
 276
 277         BUILD_BUG_ON(sizeof(device->attrs.device_cap_flags) != sizeof(u64));
 278         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
 279                               device->attrs.device_cap_flags,
 280                               RDMA_NLDEV_ATTR_PAD))
 281                 return -EMSGSIZE;
 282
 283         ib_get_device_fw_str(device, fw);
 284         /* Device without FW has strlen(fw) = 0 */
 285         if (strlen(fw) && nla_put_string(msg, RDMA_NLDEV_ATTR_FW_VERSION, fw))
 286                 return -EMSGSIZE;
 287
 288         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_NODE_GUID,
 289                               be64_to_cpu(device->node_guid),
 290                               RDMA_NLDEV_ATTR_PAD))
 291                 return -EMSGSIZE;
 292         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SYS_IMAGE_GUID,
 293                               be64_to_cpu(device->attrs.sys_image_guid),
 294                               RDMA_NLDEV_ATTR_PAD))
 295                 return -EMSGSIZE;
 296         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_NODE_TYPE, device->node_type))
 297                 return -EMSGSIZE;
 298         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, device->use_cq_dim))
 299                 return -EMSGSIZE;
 300
 301         /*
 302          * Link type is determined on first port and mlx4 device
 303          * which can potentially have two different link type for the same
 304          * IB device is considered as better to be avoided in the future,
 305          */
 306         port = rdma_start_port(device);
 307         if (rdma_cap_opa_mad(device, port))
 308                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "opa");
 309         else if (rdma_protocol_ib(device, port))
 310                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "ib");
 311         else if (rdma_protocol_iwarp(device, port))
 312                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "iw");
 313         else if (rdma_protocol_roce(device, port))
 314                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "roce");
 315         else if (rdma_protocol_usnic(device, port))
 316                 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL,
 317                                      "usnic");
 318         return ret;
 319 }
 320
 321 static int fill_port_info(struct sk_buff *msg,
 322                           struct ib_device *device, u32 port,
 323                           const struct net *net)
 324 {
 325         struct net_device *netdev = NULL;
 326         struct ib_port_attr attr;
 327         int ret;
 328         u64 cap_flags = 0;
 329
 330         if (fill_nldev_handle(msg, device))
 331                 return -EMSGSIZE;
 332
 333         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
 334                 return -EMSGSIZE;
 335
 336         ret = ib_query_port(device, port, &attr);
 337         if (ret)
 338                 return ret;
 339
 340         if (rdma_protocol_ib(device, port)) {
 341                 BUILD_BUG_ON((sizeof(attr.port_cap_flags) +
 342                                 sizeof(attr.port_cap_flags2)) > sizeof(u64));
 343                 cap_flags = attr.port_cap_flags |
 344                         ((u64)attr.port_cap_flags2 << 32);
 345                 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
 346                                       cap_flags, RDMA_NLDEV_ATTR_PAD))
 347                         return -EMSGSIZE;
 348                 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SUBNET_PREFIX,
 349                                       attr.subnet_prefix, RDMA_NLDEV_ATTR_PAD))
 350                         return -EMSGSIZE;
 351                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_LID, attr.lid))
 352                         return -EMSGSIZE;
 353                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_SM_LID, attr.sm_lid))
 354                         return -EMSGSIZE;
 355                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_LMC, attr.lmc))
 356                         return -EMSGSIZE;
 357         }
 358         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_STATE, attr.state))
 359                 return -EMSGSIZE;
 360         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_PHYS_STATE, attr.phys_state))
 361                 return -EMSGSIZE;
 362
 363         netdev = ib_device_get_netdev(device, port);
 364         if (netdev && net_eq(dev_net(netdev), net)) {
 365                 ret = nla_put_u32(msg,
 366                                   RDMA_NLDEV_ATTR_NDEV_INDEX, netdev->ifindex);
 367                 if (ret)
 368                         goto out;
 369                 ret = nla_put_string(msg,
 370                                      RDMA_NLDEV_ATTR_NDEV_NAME, netdev->name);
 371         }
 372
 373 out:
 374         dev_put(netdev);
 375         return ret;
 376 }
 377
 378 static int fill_res_info_entry(struct sk_buff *msg,
 379                                const char *name, u64 curr)
 380 {
 381         struct nlattr *entry_attr;
 382
 383         entry_attr = nla_nest_start_noflag(msg,
 384                                            RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY);
 385         if (!entry_attr)
 386                 return -EMSGSIZE;
 387
 388         if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name))
 389                 goto err;
 390         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr,
 391                               RDMA_NLDEV_ATTR_PAD))
 392                 goto err;
 393
 394         nla_nest_end(msg, entry_attr);
 395         return 0;
 396
 397 err:
 398         nla_nest_cancel(msg, entry_attr);
 399         return -EMSGSIZE;
 400 }
 401
 402 static int fill_res_info(struct sk_buff *msg, struct ib_device *device)
 403 {
 404         static const char * const names[RDMA_RESTRACK_MAX] = {
 405                 [RDMA_RESTRACK_PD] = "pd",
 406                 [RDMA_RESTRACK_CQ] = "cq",
 407                 [RDMA_RESTRACK_QP] = "qp",
 408                 [RDMA_RESTRACK_CM_ID] = "cm_id",
 409                 [RDMA_RESTRACK_MR] = "mr",
 410                 [RDMA_RESTRACK_CTX] = "ctx",
 411                 [RDMA_RESTRACK_SRQ] = "srq",
 412         };
 413
 414         struct nlattr *table_attr;
 415         int ret, i, curr;
 416
 417         if (fill_nldev_handle(msg, device))
 418                 return -EMSGSIZE;
 419
 420         table_attr = nla_nest_start_noflag(msg, RDMA_NLDEV_ATTR_RES_SUMMARY);
 421         if (!table_attr)
 422                 return -EMSGSIZE;
 423
 424         for (i = 0; i < RDMA_RESTRACK_MAX; i++) {
 425                 if (!names[i])
 426                         continue;
 427                 curr = rdma_restrack_count(device, i);
 428                 ret = fill_res_info_entry(msg, names[i], curr);
 429                 if (ret)
 430                         goto err;
 431         }
 432
 433         nla_nest_end(msg, table_attr);
 434         return 0;
 435
 436 err:
 437         nla_nest_cancel(msg, table_attr);
 438         return ret;
 439 }
 440
 441 static int fill_res_name_pid(struct sk_buff *msg,
 442                              struct rdma_restrack_entry *res)
 443 {
 444         int err = 0;
 445
 446         /*
 447          * For user resources, user is should read /proc/PID/comm to get the
 448          * name of the task file.
 449          */
 450         if (rdma_is_kernel_res(res)) {
 451                 err = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME,
 452                                      res->kern_name);
 453         } else {
 454                 pid_t pid;
 455
 456                 pid = task_pid_vnr(res->task);
 457                 /*
 458                  * Task is dead and in zombie state.
 459                  * There is no need to print PID anymore.
 460                  */
 461                 if (pid)
 462                         /*
 463                          * This part is racy, task can be killed and PID will
 464                          * be zero right here but it is ok, next query won't
 465                          * return PID. We don't promise real-time reflection
 466                          * of SW objects.
 467                          */
 468                         err = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID, pid);
 469         }
 470
 471         return err ? -EMSGSIZE : 0;
 472 }
 473
 474 static int fill_res_qp_entry_query(struct sk_buff *msg,
 475                                    struct rdma_restrack_entry *res,
 476                                    struct ib_device *dev,
 477                                    struct ib_qp *qp)
 478 {
 479         struct ib_qp_init_attr qp_init_attr;
 480         struct ib_qp_attr qp_attr;
 481         int ret;
 482
 483         ret = ib_query_qp(qp, &qp_attr, 0, &qp_init_attr);
 484         if (ret)
 485                 return ret;
 486
 487         if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) {
 488                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN,
 489                                 qp_attr.dest_qp_num))
 490                         goto err;
 491                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQ_PSN,
 492                                 qp_attr.rq_psn))
 493                         goto err;
 494         }
 495
 496         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SQ_PSN, qp_attr.sq_psn))
 497                 goto err;
 498
 499         if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC ||
 500             qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) {
 501                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE,
 502                                qp_attr.path_mig_state))
 503                         goto err;
 504         }
 505         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, qp->qp_type))
 506                 goto err;
 507         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state))
 508                 goto err;
 509
 510         if (dev->ops.fill_res_qp_entry)
 511                 return dev->ops.fill_res_qp_entry(msg, qp);
 512         return 0;
 513
 514 err:    return -EMSGSIZE;
 515 }
 516
 517 static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin,
 518                              struct rdma_restrack_entry *res, uint32_t port)
 519 {
 520         struct ib_qp *qp = container_of(res, struct ib_qp, res);
 521         struct ib_device *dev = qp->device;
 522         int ret;
 523
 524         if (port && port != qp->port)
 525                 return -EAGAIN;
 526
 527         /* In create_qp() port is not set yet */
 528         if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port))
 529                 return -EMSGSIZE;
 530
 531         ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num);
 532         if (ret)
 533                 return -EMSGSIZE;
 534
 535         if (!rdma_is_kernel_res(res) &&
 536             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id))
 537                 return -EMSGSIZE;
 538
 539         ret = fill_res_name_pid(msg, res);
 540         if (ret)
 541                 return -EMSGSIZE;
 542
 543         return fill_res_qp_entry_query(msg, res, dev, qp);
 544 }
 545
 546 static int fill_res_qp_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 547                                  struct rdma_restrack_entry *res, uint32_t port)
 548 {
 549         struct ib_qp *qp = container_of(res, struct ib_qp, res);
 550         struct ib_device *dev = qp->device;
 551
 552         if (port && port != qp->port)
 553                 return -EAGAIN;
 554         if (!dev->ops.fill_res_qp_entry_raw)
 555                 return -EINVAL;
 556         return dev->ops.fill_res_qp_entry_raw(msg, qp);
 557 }
 558
 559 static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin,
 560                                 struct rdma_restrack_entry *res, uint32_t port)
 561 {
 562         struct rdma_id_private *id_priv =
 563                                 container_of(res, struct rdma_id_private, res);
 564         struct ib_device *dev = id_priv->id.device;
 565         struct rdma_cm_id *cm_id = &id_priv->id;
 566
 567         if (port && port != cm_id->port_num)
 568                 return -EAGAIN;
 569
 570         if (cm_id->port_num &&
 571             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id->port_num))
 572                 goto err;
 573
 574         if (id_priv->qp_num) {
 575                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, id_priv->qp_num))
 576                         goto err;
 577                 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, cm_id->qp_type))
 578                         goto err;
 579         }
 580
 581         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PS, cm_id->ps))
 582                 goto err;
 583
 584         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, id_priv->state))
 585                 goto err;
 586
 587         if (cm_id->route.addr.src_addr.ss_family &&
 588             nla_put(msg, RDMA_NLDEV_ATTR_RES_SRC_ADDR,
 589                     sizeof(cm_id->route.addr.src_addr),
 590                     &cm_id->route.addr.src_addr))
 591                 goto err;
 592         if (cm_id->route.addr.dst_addr.ss_family &&
 593             nla_put(msg, RDMA_NLDEV_ATTR_RES_DST_ADDR,
 594                     sizeof(cm_id->route.addr.dst_addr),
 595                     &cm_id->route.addr.dst_addr))
 596                 goto err;
 597
 598         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CM_IDN, res->id))
 599                 goto err;
 600
 601         if (fill_res_name_pid(msg, res))
 602                 goto err;
 603
 604         if (dev->ops.fill_res_cm_id_entry)
 605                 return dev->ops.fill_res_cm_id_entry(msg, cm_id);
 606         return 0;
 607
 608 err: return -EMSGSIZE;
 609 }
 610
 611 static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin,
 612                              struct rdma_restrack_entry *res, uint32_t port)
 613 {
 614         struct ib_cq *cq = container_of(res, struct ib_cq, res);
 615         struct ib_device *dev = cq->device;
 616
 617         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq->cqe))
 618                 return -EMSGSIZE;
 619         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
 620                               atomic_read(&cq->usecnt), RDMA_NLDEV_ATTR_PAD))
 621                 return -EMSGSIZE;
 622
 623         /* Poll context is only valid for kernel CQs */
 624         if (rdma_is_kernel_res(res) &&
 625             nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_POLL_CTX, cq->poll_ctx))
 626                 return -EMSGSIZE;
 627
 628         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, (cq->dim != NULL)))
 629                 return -EMSGSIZE;
 630
 631         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id))
 632                 return -EMSGSIZE;
 633         if (!rdma_is_kernel_res(res) &&
 634             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 635                         cq->uobject->uevent.uobject.context->res.id))
 636                 return -EMSGSIZE;
 637
 638         if (fill_res_name_pid(msg, res))
 639                 return -EMSGSIZE;
 640
 641         return (dev->ops.fill_res_cq_entry) ?
 642                 dev->ops.fill_res_cq_entry(msg, cq) : 0;
 643 }
 644
 645 static int fill_res_cq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 646                                  struct rdma_restrack_entry *res, uint32_t port)
 647 {
 648         struct ib_cq *cq = container_of(res, struct ib_cq, res);
 649         struct ib_device *dev = cq->device;
 650
 651         if (!dev->ops.fill_res_cq_entry_raw)
 652                 return -EINVAL;
 653         return dev->ops.fill_res_cq_entry_raw(msg, cq);
 654 }
 655
 656 static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
 657                              struct rdma_restrack_entry *res, uint32_t port)
 658 {
 659         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 660         struct ib_device *dev = mr->pd->device;
 661
 662         if (has_cap_net_admin) {
 663                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey))
 664                         return -EMSGSIZE;
 665                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey))
 666                         return -EMSGSIZE;
 667         }
 668
 669         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length,
 670                               RDMA_NLDEV_ATTR_PAD))
 671                 return -EMSGSIZE;
 672
 673         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
 674                 return -EMSGSIZE;
 675
 676         if (!rdma_is_kernel_res(res) &&
 677             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id))
 678                 return -EMSGSIZE;
 679
 680         if (fill_res_name_pid(msg, res))
 681                 return -EMSGSIZE;
 682
 683         return (dev->ops.fill_res_mr_entry) ?
 684                        dev->ops.fill_res_mr_entry(msg, mr) :
 685                        0;
 686 }
 687
 688 static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 689                                  struct rdma_restrack_entry *res, uint32_t port)
 690 {
 691         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 692         struct ib_device *dev = mr->pd->device;
 693
 694         if (!dev->ops.fill_res_mr_entry_raw)
 695                 return -EINVAL;
 696         return dev->ops.fill_res_mr_entry_raw(msg, mr);
 697 }
 698
 699 static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,
 700                              struct rdma_restrack_entry *res, uint32_t port)
 701 {
 702         struct ib_pd *pd = container_of(res, struct ib_pd, res);
 703
 704         if (has_cap_net_admin) {
 705                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,
 706                                 pd->local_dma_lkey))
 707                         goto err;
 708                 if ((pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) &&
 709                     nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,
 710                                 pd->unsafe_global_rkey))
 711                         goto err;
 712         }
 713         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT,
 714                               atomic_read(&pd->usecnt), RDMA_NLDEV_ATTR_PAD))
 715                 goto err;
 716
 717         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id))
 718                 goto err;
 719
 720         if (!rdma_is_kernel_res(res) &&
 721             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 722                         pd->uobject->context->res.id))
 723                 goto err;
 724
 725         return fill_res_name_pid(msg, res);
 726
 727 err:    return -EMSGSIZE;
 728 }
 729
 730 static int fill_res_ctx_entry(struct sk_buff *msg, bool has_cap_net_admin,
 731                               struct rdma_restrack_entry *res, uint32_t port)
 732 {
 733         struct ib_ucontext *ctx = container_of(res, struct ib_ucontext, res);
 734
 735         if (rdma_is_kernel_res(res))
 736                 return 0;
 737
 738         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, ctx->res.id))
 739                 return -EMSGSIZE;
 740
 741         return fill_res_name_pid(msg, res);
 742 }
 743
 744 static int fill_res_range_qp_entry(struct sk_buff *msg, uint32_t min_range,
 745                                    uint32_t max_range)
 746 {
 747         struct nlattr *entry_attr;
 748
 749         if (!min_range)
 750                 return 0;
 751
 752         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);
 753         if (!entry_attr)
 754                 return -EMSGSIZE;
 755
 756         if (min_range == max_range) {
 757                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, min_range))
 758                         goto err;
 759         } else {
 760                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MIN_RANGE, min_range))
 761                         goto err;
 762                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MAX_RANGE, max_range))
 763                         goto err;
 764         }
 765         nla_nest_end(msg, entry_attr);
 766         return 0;
 767
 768 err:
 769         nla_nest_cancel(msg, entry_attr);
 770         return -EMSGSIZE;
 771 }
 772
 773 static int fill_res_srq_qps(struct sk_buff *msg, struct ib_srq *srq)
 774 {
 775         uint32_t min_range = 0, prev = 0;
 776         struct rdma_restrack_entry *res;
 777         struct rdma_restrack_root *rt;
 778         struct nlattr *table_attr;
 779         struct ib_qp *qp = NULL;
 780         unsigned long id = 0;
 781
 782         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);
 783         if (!table_attr)
 784                 return -EMSGSIZE;
 785
 786         rt = &srq->device->res[RDMA_RESTRACK_QP];
 787         xa_lock(&rt->xa);
 788         xa_for_each(&rt->xa, id, res) {
 789                 if (!rdma_restrack_get(res))
 790                         continue;
 791
 792                 qp = container_of(res, struct ib_qp, res);
 793                 if (!qp->srq || (qp->srq->res.id != srq->res.id)) {
 794                         rdma_restrack_put(res);
 795                         continue;
 796                 }
 797
 798                 if (qp->qp_num < prev)
 799                         /* qp_num should be ascending */
 800                         goto err_loop;
 801
 802                 if (min_range == 0) {
 803                         min_range = qp->qp_num;
 804                 } else if (qp->qp_num > (prev + 1)) {
 805                         if (fill_res_range_qp_entry(msg, min_range, prev))
 806                                 goto err_loop;
 807
 808                         min_range = qp->qp_num;
 809                 }
 810                 prev = qp->qp_num;
 811                 rdma_restrack_put(res);
 812         }
 813
 814         xa_unlock(&rt->xa);
 815
 816         if (fill_res_range_qp_entry(msg, min_range, prev))
 817                 goto err;
 818
 819         nla_nest_end(msg, table_attr);
 820         return 0;
 821
 822 err_loop:
 823         rdma_restrack_put(res);
 824         xa_unlock(&rt->xa);
 825 err:
 826         nla_nest_cancel(msg, table_attr);
 827         return -EMSGSIZE;
 828 }
 829
 830 static int fill_res_srq_entry(struct sk_buff *msg, bool has_cap_net_admin,
 831                               struct rdma_restrack_entry *res, uint32_t port)
 832 {
 833         struct ib_srq *srq = container_of(res, struct ib_srq, res);
 834         struct ib_device *dev = srq->device;
 835
 836         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SRQN, srq->res.id))
 837                 goto err;
 838
 839         if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, srq->srq_type))
 840                 goto err;
 841
 842         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, srq->pd->res.id))
 843                 goto err;
 844
 845         if (ib_srq_has_cq(srq->srq_type)) {
 846                 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN,
 847                                 srq->ext.cq->res.id))
 848                         goto err;
 849         }
 850
 851         if (fill_res_srq_qps(msg, srq))
 852                 goto err;
 853
 854         if (fill_res_name_pid(msg, res))
 855                 goto err;
 856
 857         if (dev->ops.fill_res_srq_entry)
 858                 return dev->ops.fill_res_srq_entry(msg, srq);
 859
 860         return 0;
 861
 862 err:
 863         return -EMSGSIZE;
 864 }
 865
 866 static int fill_res_srq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin,
 867                                  struct rdma_restrack_entry *res, uint32_t port)
 868 {
 869         struct ib_srq *srq = container_of(res, struct ib_srq, res);
 870         struct ib_device *dev = srq->device;
 871
 872         if (!dev->ops.fill_res_srq_entry_raw)
 873                 return -EINVAL;
 874         return dev->ops.fill_res_srq_entry_raw(msg, srq);
 875 }
 876
 877 static int fill_stat_counter_mode(struct sk_buff *msg,
 878                                   struct rdma_counter *counter)
 879 {
 880         struct rdma_counter_mode *m = &counter->mode;
 881
 882         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, m->mode))
 883                 return -EMSGSIZE;
 884
 885         if (m->mode == RDMA_COUNTER_MODE_AUTO) {
 886                 if ((m->mask & RDMA_COUNTER_MASK_QP_TYPE) &&
 887                     nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, m->param.qp_type))
 888                         return -EMSGSIZE;
 889
 890                 if ((m->mask & RDMA_COUNTER_MASK_PID) &&
 891                     fill_res_name_pid(msg, &counter->res))
 892                         return -EMSGSIZE;
 893         }
 894
 895         return 0;
 896 }
 897
 898 static int fill_stat_counter_qp_entry(struct sk_buff *msg, u32 qpn)
 899 {
 900         struct nlattr *entry_attr;
 901
 902         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);
 903         if (!entry_attr)
 904                 return -EMSGSIZE;
 905
 906         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn))
 907                 goto err;
 908
 909         nla_nest_end(msg, entry_attr);
 910         return 0;
 911
 912 err:
 913         nla_nest_cancel(msg, entry_attr);
 914         return -EMSGSIZE;
 915 }
 916
 917 static int fill_stat_counter_qps(struct sk_buff *msg,
 918                                  struct rdma_counter *counter)
 919 {
 920         struct rdma_restrack_entry *res;
 921         struct rdma_restrack_root *rt;
 922         struct nlattr *table_attr;
 923         struct ib_qp *qp = NULL;
 924         unsigned long id = 0;
 925         int ret = 0;
 926
 927         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP);
 928         if (!table_attr)
 929                 return -EMSGSIZE;
 930
 931         rt = &counter->device->res[RDMA_RESTRACK_QP];
 932         xa_lock(&rt->xa);
 933         xa_for_each(&rt->xa, id, res) {
 934                 qp = container_of(res, struct ib_qp, res);
 935                 if (!qp->counter || (qp->counter->id != counter->id))
 936                         continue;
 937
 938                 ret = fill_stat_counter_qp_entry(msg, qp->qp_num);
 939                 if (ret)
 940                         goto err;
 941         }
 942
 943         xa_unlock(&rt->xa);
 944         nla_nest_end(msg, table_attr);
 945         return 0;
 946
 947 err:
 948         xa_unlock(&rt->xa);
 949         nla_nest_cancel(msg, table_attr);
 950         return ret;
 951 }
 952
 953 int rdma_nl_stat_hwcounter_entry(struct sk_buff *msg, const char *name,
 954                                  u64 value)
 955 {
 956         struct nlattr *entry_attr;
 957
 958         entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY);
 959         if (!entry_attr)
 960                 return -EMSGSIZE;
 961
 962         if (nla_put_string(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME,
 963                            name))
 964                 goto err;
 965         if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE,
 966                               value, RDMA_NLDEV_ATTR_PAD))
 967                 goto err;
 968
 969         nla_nest_end(msg, entry_attr);
 970         return 0;
 971
 972 err:
 973         nla_nest_cancel(msg, entry_attr);
 974         return -EMSGSIZE;
 975 }
 976 EXPORT_SYMBOL(rdma_nl_stat_hwcounter_entry);
 977
 978 static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin,
 979                               struct rdma_restrack_entry *res, uint32_t port)
 980 {
 981         struct ib_mr *mr = container_of(res, struct ib_mr, res);
 982         struct ib_device *dev = mr->pd->device;
 983
 984         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id))
 985                 goto err;
 986
 987         if (dev->ops.fill_stat_mr_entry)
 988                 return dev->ops.fill_stat_mr_entry(msg, mr);
 989         return 0;
 990
 991 err:
 992         return -EMSGSIZE;
 993 }
 994
 995 static int fill_stat_counter_hwcounters(struct sk_buff *msg,
 996                                         struct rdma_counter *counter)
 997 {
 998         struct rdma_hw_stats *st = counter->stats;
 999         struct nlattr *table_attr;
1000         int i;
1001
1002         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
1003         if (!table_attr)
1004                 return -EMSGSIZE;
1005
1006         mutex_lock(&st->lock);
1007         for (i = 0; i < st->num_counters; i++) {
1008                 if (test_bit(i, st->is_disabled))
1009                         continue;
1010                 if (rdma_nl_stat_hwcounter_entry(msg, st->descs[i].name,
1011                                                  st->value[i]))
1012                         goto err;
1013         }
1014         mutex_unlock(&st->lock);
1015
1016         nla_nest_end(msg, table_attr);
1017         return 0;
1018
1019 err:
1020         mutex_unlock(&st->lock);
1021         nla_nest_cancel(msg, table_attr);
1022         return -EMSGSIZE;
1023 }
1024
1025 static int fill_res_counter_entry(struct sk_buff *msg, bool has_cap_net_admin,
1026                                   struct rdma_restrack_entry *res,
1027                                   uint32_t port)
1028 {
1029         struct rdma_counter *counter =
1030                 container_of(res, struct rdma_counter, res);
1031
1032         if (port && port != counter->port)
1033                 return -EAGAIN;
1034
1035         /* Dump it even query failed */
1036         rdma_counter_query_stats(counter);
1037
1038         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, counter->port) ||
1039             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, counter->id) ||
1040             fill_stat_counter_mode(msg, counter) ||
1041             fill_stat_counter_qps(msg, counter) ||
1042             fill_stat_counter_hwcounters(msg, counter))
1043                 return -EMSGSIZE;
1044
1045         return 0;
1046 }
1047
1048 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1049                           struct netlink_ext_ack *extack)
1050 {
1051         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1052         struct ib_device *device;
1053         struct sk_buff *msg;
1054         u32 index;
1055         int err;
1056
1057         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1058                                      nldev_policy, extack);
1059         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1060                 return -EINVAL;
1061
1062         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1063
1064         device = ib_device_get_by_index(sock_net(skb->sk), index);
1065         if (!device)
1066                 return -EINVAL;
1067
1068         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1069         if (!msg) {
1070                 err = -ENOMEM;
1071                 goto err;
1072         }
1073
1074         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1075                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1076                         0, 0);
1077         if (!nlh) {
1078                 err = -EMSGSIZE;
1079                 goto err_free;
1080         }
1081
1082         err = fill_dev_info(msg, device);
1083         if (err)
1084                 goto err_free;
1085
1086         nlmsg_end(msg, nlh);
1087
1088         ib_device_put(device);
1089         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1090
1091 err_free:
1092         nlmsg_free(msg);
1093 err:
1094         ib_device_put(device);
1095         return err;
1096 }
1097
1098 static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1099                           struct netlink_ext_ack *extack)
1100 {
1101         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1102         struct ib_device *device;
1103         u32 index;
1104         int err;
1105
1106         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1107                                      nldev_policy, extack);
1108         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1109                 return -EINVAL;
1110
1111         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1112         device = ib_device_get_by_index(sock_net(skb->sk), index);
1113         if (!device)
1114                 return -EINVAL;
1115
1116         if (tb[RDMA_NLDEV_ATTR_DEV_NAME]) {
1117                 char name[IB_DEVICE_NAME_MAX] = {};
1118
1119                 nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME],
1120                             IB_DEVICE_NAME_MAX);
1121                 if (strlen(name) == 0) {
1122                         err = -EINVAL;
1123                         goto done;
1124                 }
1125                 err = ib_device_rename(device, name);
1126                 goto done;
1127         }
1128
1129         if (tb[RDMA_NLDEV_NET_NS_FD]) {
1130                 u32 ns_fd;
1131
1132                 ns_fd = nla_get_u32(tb[RDMA_NLDEV_NET_NS_FD]);
1133                 err = ib_device_set_netns_put(skb, device, ns_fd);
1134                 goto put_done;
1135         }
1136
1137         if (tb[RDMA_NLDEV_ATTR_DEV_DIM]) {
1138                 u8 use_dim;
1139
1140                 use_dim = nla_get_u8(tb[RDMA_NLDEV_ATTR_DEV_DIM]);
1141                 err = ib_device_set_dim(device,  use_dim);
1142                 goto done;
1143         }
1144
1145 done:
1146         ib_device_put(device);
1147 put_done:
1148         return err;
1149 }
1150
1151 static int _nldev_get_dumpit(struct ib_device *device,
1152                              struct sk_buff *skb,
1153                              struct netlink_callback *cb,
1154                              unsigned int idx)
1155 {
1156         int start = cb->args[0];
1157         struct nlmsghdr *nlh;
1158
1159         if (idx < start)
1160                 return 0;
1161
1162         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1163                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1164                         0, NLM_F_MULTI);
1165
1166         if (!nlh || fill_dev_info(skb, device)) {
1167                 nlmsg_cancel(skb, nlh);
1168                 goto out;
1169         }
1170
1171         nlmsg_end(skb, nlh);
1172
1173         idx++;
1174
1175 out:    cb->args[0] = idx;
1176         return skb->len;
1177 }
1178
1179 static int nldev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
1180 {
1181         /*
1182          * There is no need to take lock, because
1183          * we are relying on ib_core's locking.
1184          */
1185         return ib_enum_all_devs(_nldev_get_dumpit, skb, cb);
1186 }
1187
1188 static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1189                                struct netlink_ext_ack *extack)
1190 {
1191         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1192         struct ib_device *device;
1193         struct sk_buff *msg;
1194         u32 index;
1195         u32 port;
1196         int err;
1197
1198         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1199                                      nldev_policy, extack);
1200         if (err ||
1201             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
1202             !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
1203                 return -EINVAL;
1204
1205         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1206         device = ib_device_get_by_index(sock_net(skb->sk), index);
1207         if (!device)
1208                 return -EINVAL;
1209
1210         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1211         if (!rdma_is_port_valid(device, port)) {
1212                 err = -EINVAL;
1213                 goto err;
1214         }
1215
1216         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1217         if (!msg) {
1218                 err = -ENOMEM;
1219                 goto err;
1220         }
1221
1222         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1223                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
1224                         0, 0);
1225         if (!nlh) {
1226                 err = -EMSGSIZE;
1227                 goto err_free;
1228         }
1229
1230         err = fill_port_info(msg, device, port, sock_net(skb->sk));
1231         if (err)
1232                 goto err_free;
1233
1234         nlmsg_end(msg, nlh);
1235         ib_device_put(device);
1236
1237         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1238
1239 err_free:
1240         nlmsg_free(msg);
1241 err:
1242         ib_device_put(device);
1243         return err;
1244 }
1245
1246 static int nldev_port_get_dumpit(struct sk_buff *skb,
1247                                  struct netlink_callback *cb)
1248 {
1249         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1250         struct ib_device *device;
1251         int start = cb->args[0];
1252         struct nlmsghdr *nlh;
1253         u32 idx = 0;
1254         u32 ifindex;
1255         int err;
1256         unsigned int p;
1257
1258         err = nlmsg_parse_deprecated(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1259                                      nldev_policy, NULL);
1260         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1261                 return -EINVAL;
1262
1263         ifindex = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1264         device = ib_device_get_by_index(sock_net(skb->sk), ifindex);
1265         if (!device)
1266                 return -EINVAL;
1267
1268         rdma_for_each_port (device, p) {
1269                 /*
1270                  * The dumpit function returns all information from specific
1271                  * index. This specific index is taken from the netlink
1272                  * messages request sent by user and it is available
1273                  * in cb->args[0].
1274                  *
1275                  * Usually, the user doesn't fill this field and it causes
1276                  * to return everything.
1277                  *
1278                  */
1279                 if (idx < start) {
1280                         idx++;
1281                         continue;
1282                 }
1283
1284                 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid,
1285                                 cb->nlh->nlmsg_seq,
1286                                 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1287                                                  RDMA_NLDEV_CMD_PORT_GET),
1288                                 0, NLM_F_MULTI);
1289
1290                 if (!nlh || fill_port_info(skb, device, p, sock_net(skb->sk))) {
1291                         nlmsg_cancel(skb, nlh);
1292                         goto out;
1293                 }
1294                 idx++;
1295                 nlmsg_end(skb, nlh);
1296         }
1297
1298 out:
1299         ib_device_put(device);
1300         cb->args[0] = idx;
1301         return skb->len;
1302 }
1303
1304 static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1305                               struct netlink_ext_ack *extack)
1306 {
1307         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1308         struct ib_device *device;
1309         struct sk_buff *msg;
1310         u32 index;
1311         int ret;
1312
1313         ret = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1314                                      nldev_policy, extack);
1315         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1316                 return -EINVAL;
1317
1318         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1319         device = ib_device_get_by_index(sock_net(skb->sk), index);
1320         if (!device)
1321                 return -EINVAL;
1322
1323         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1324         if (!msg) {
1325                 ret = -ENOMEM;
1326                 goto err;
1327         }
1328
1329         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1330                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
1331                         0, 0);
1332         if (!nlh) {
1333                 ret = -EMSGSIZE;
1334                 goto err_free;
1335         }
1336
1337         ret = fill_res_info(msg, device);
1338         if (ret)
1339                 goto err_free;
1340
1341         nlmsg_end(msg, nlh);
1342         ib_device_put(device);
1343         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1344
1345 err_free:
1346         nlmsg_free(msg);
1347 err:
1348         ib_device_put(device);
1349         return ret;
1350 }
1351
1352 static int _nldev_res_get_dumpit(struct ib_device *device,
1353                                  struct sk_buff *skb,
1354                                  struct netlink_callback *cb,
1355                                  unsigned int idx)
1356 {
1357         int start = cb->args[0];
1358         struct nlmsghdr *nlh;
1359
1360         if (idx < start)
1361                 return 0;
1362
1363         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1364                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
1365                         0, NLM_F_MULTI);
1366
1367         if (!nlh || fill_res_info(skb, device)) {
1368                 nlmsg_cancel(skb, nlh);
1369                 goto out;
1370         }
1371         nlmsg_end(skb, nlh);
1372
1373         idx++;
1374
1375 out:
1376         cb->args[0] = idx;
1377         return skb->len;
1378 }
1379
1380 static int nldev_res_get_dumpit(struct sk_buff *skb,
1381                                 struct netlink_callback *cb)
1382 {
1383         return ib_enum_all_devs(_nldev_res_get_dumpit, skb, cb);
1384 }
1385
1386 struct nldev_fill_res_entry {
1387         enum rdma_nldev_attr nldev_attr;
1388         u8 flags;
1389         u32 entry;
1390         u32 id;
1391 };
1392
1393 enum nldev_res_flags {
1394         NLDEV_PER_DEV = 1 << 0,
1395 };
1396
1397 static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = {
1398         [RDMA_RESTRACK_QP] = {
1399                 .nldev_attr = RDMA_NLDEV_ATTR_RES_QP,
1400                 .entry = RDMA_NLDEV_ATTR_RES_QP_ENTRY,
1401                 .id = RDMA_NLDEV_ATTR_RES_LQPN,
1402         },
1403         [RDMA_RESTRACK_CM_ID] = {
1404                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CM_ID,
1405                 .entry = RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY,
1406                 .id = RDMA_NLDEV_ATTR_RES_CM_IDN,
1407         },
1408         [RDMA_RESTRACK_CQ] = {
1409                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CQ,
1410                 .flags = NLDEV_PER_DEV,
1411                 .entry = RDMA_NLDEV_ATTR_RES_CQ_ENTRY,
1412                 .id = RDMA_NLDEV_ATTR_RES_CQN,
1413         },
1414         [RDMA_RESTRACK_MR] = {
1415                 .nldev_attr = RDMA_NLDEV_ATTR_RES_MR,
1416                 .flags = NLDEV_PER_DEV,
1417                 .entry = RDMA_NLDEV_ATTR_RES_MR_ENTRY,
1418                 .id = RDMA_NLDEV_ATTR_RES_MRN,
1419         },
1420         [RDMA_RESTRACK_PD] = {
1421                 .nldev_attr = RDMA_NLDEV_ATTR_RES_PD,
1422                 .flags = NLDEV_PER_DEV,
1423                 .entry = RDMA_NLDEV_ATTR_RES_PD_ENTRY,
1424                 .id = RDMA_NLDEV_ATTR_RES_PDN,
1425         },
1426         [RDMA_RESTRACK_COUNTER] = {
1427                 .nldev_attr = RDMA_NLDEV_ATTR_STAT_COUNTER,
1428                 .entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY,
1429                 .id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID,
1430         },
1431         [RDMA_RESTRACK_CTX] = {
1432                 .nldev_attr = RDMA_NLDEV_ATTR_RES_CTX,
1433                 .flags = NLDEV_PER_DEV,
1434                 .entry = RDMA_NLDEV_ATTR_RES_CTX_ENTRY,
1435                 .id = RDMA_NLDEV_ATTR_RES_CTXN,
1436         },
1437         [RDMA_RESTRACK_SRQ] = {
1438                 .nldev_attr = RDMA_NLDEV_ATTR_RES_SRQ,
1439                 .flags = NLDEV_PER_DEV,
1440                 .entry = RDMA_NLDEV_ATTR_RES_SRQ_ENTRY,
1441                 .id = RDMA_NLDEV_ATTR_RES_SRQN,
1442         },
1443
1444 };
1445
1446 static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1447                                struct netlink_ext_ack *extack,
1448                                enum rdma_restrack_type res_type,
1449                                res_fill_func_t fill_func)
1450 {
1451         const struct nldev_fill_res_entry *fe = &fill_entries[res_type];
1452         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1453         struct rdma_restrack_entry *res;
1454         struct ib_device *device;
1455         u32 index, id, port = 0;
1456         bool has_cap_net_admin;
1457         struct sk_buff *msg;
1458         int ret;
1459
1460         ret = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1461                                      nldev_policy, extack);
1462         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !fe->id || !tb[fe->id])
1463                 return -EINVAL;
1464
1465         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1466         device = ib_device_get_by_index(sock_net(skb->sk), index);
1467         if (!device)
1468                 return -EINVAL;
1469
1470         if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1471                 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1472                 if (!rdma_is_port_valid(device, port)) {
1473                         ret = -EINVAL;
1474                         goto err;
1475                 }
1476         }
1477
1478         if ((port && fe->flags & NLDEV_PER_DEV) ||
1479             (!port && ~fe->flags & NLDEV_PER_DEV)) {
1480                 ret = -EINVAL;
1481                 goto err;
1482         }
1483
1484         id = nla_get_u32(tb[fe->id]);
1485         res = rdma_restrack_get_byid(device, res_type, id);
1486         if (IS_ERR(res)) {
1487                 ret = PTR_ERR(res);
1488                 goto err;
1489         }
1490
1491         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1492         if (!msg) {
1493                 ret = -ENOMEM;
1494                 goto err_get;
1495         }
1496
1497         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1498                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1499                                          RDMA_NL_GET_OP(nlh->nlmsg_type)),
1500                         0, 0);
1501
1502         if (!nlh || fill_nldev_handle(msg, device)) {
1503                 ret = -EMSGSIZE;
1504                 goto err_free;
1505         }
1506
1507         has_cap_net_admin = netlink_capable(skb, CAP_NET_ADMIN);
1508
1509         ret = fill_func(msg, has_cap_net_admin, res, port);
1510         if (ret)
1511                 goto err_free;
1512
1513         rdma_restrack_put(res);
1514         nlmsg_end(msg, nlh);
1515         ib_device_put(device);
1516         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1517
1518 err_free:
1519         nlmsg_free(msg);
1520 err_get:
1521         rdma_restrack_put(res);
1522 err:
1523         ib_device_put(device);
1524         return ret;
1525 }
1526
1527 static int res_get_common_dumpit(struct sk_buff *skb,
1528                                  struct netlink_callback *cb,
1529                                  enum rdma_restrack_type res_type,
1530                                  res_fill_func_t fill_func)
1531 {
1532         const struct nldev_fill_res_entry *fe = &fill_entries[res_type];
1533         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1534         struct rdma_restrack_entry *res;
1535         struct rdma_restrack_root *rt;
1536         int err, ret = 0, idx = 0;
1537         struct nlattr *table_attr;
1538         struct nlattr *entry_attr;
1539         struct ib_device *device;
1540         int start = cb->args[0];
1541         bool has_cap_net_admin;
1542         struct nlmsghdr *nlh;
1543         unsigned long id;
1544         u32 index, port = 0;
1545         bool filled = false;
1546
1547         err = nlmsg_parse_deprecated(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1548                                      nldev_policy, NULL);
1549         /*
1550          * Right now, we are expecting the device index to get res information,
1551          * but it is possible to extend this code to return all devices in
1552          * one shot by checking the existence of RDMA_NLDEV_ATTR_DEV_INDEX.
1553          * if it doesn't exist, we will iterate over all devices.
1554          *
1555          * But it is not needed for now.
1556          */
1557         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1558                 return -EINVAL;
1559
1560         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1561         device = ib_device_get_by_index(sock_net(skb->sk), index);
1562         if (!device)
1563                 return -EINVAL;
1564
1565         /*
1566          * If no PORT_INDEX is supplied, we will return all QPs from that device
1567          */
1568         if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1569                 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1570                 if (!rdma_is_port_valid(device, port)) {
1571                         ret = -EINVAL;
1572                         goto err_index;
1573                 }
1574         }
1575
1576         nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1577                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1578                                          RDMA_NL_GET_OP(cb->nlh->nlmsg_type)),
1579                         0, NLM_F_MULTI);
1580
1581         if (!nlh || fill_nldev_handle(skb, device)) {
1582                 ret = -EMSGSIZE;
1583                 goto err;
1584         }
1585
1586         table_attr = nla_nest_start_noflag(skb, fe->nldev_attr);
1587         if (!table_attr) {
1588                 ret = -EMSGSIZE;
1589                 goto err;
1590         }
1591
1592         has_cap_net_admin = netlink_capable(cb->skb, CAP_NET_ADMIN);
1593
1594         rt = &device->res[res_type];
1595         xa_lock(&rt->xa);
1596         /*
1597          * FIXME: if the skip ahead is something common this loop should
1598          * use xas_for_each & xas_pause to optimize, we can have a lot of
1599          * objects.
1600          */
1601         xa_for_each(&rt->xa, id, res) {
1602                 if (idx < start || !rdma_restrack_get(res))
1603                         goto next;
1604
1605                 xa_unlock(&rt->xa);
1606
1607                 filled = true;
1608
1609                 entry_attr = nla_nest_start_noflag(skb, fe->entry);
1610                 if (!entry_attr) {
1611                         ret = -EMSGSIZE;
1612                         rdma_restrack_put(res);
1613                         goto msg_full;
1614                 }
1615
1616                 ret = fill_func(skb, has_cap_net_admin, res, port);
1617
1618                 rdma_restrack_put(res);
1619
1620                 if (ret) {
1621                         nla_nest_cancel(skb, entry_attr);
1622                         if (ret == -EMSGSIZE)
1623                                 goto msg_full;
1624                         if (ret == -EAGAIN)
1625                                 goto again;
1626                         goto res_err;
1627                 }
1628                 nla_nest_end(skb, entry_attr);
1629 again:          xa_lock(&rt->xa);
1630 next:           idx++;
1631         }
1632         xa_unlock(&rt->xa);
1633
1634 msg_full:
1635         nla_nest_end(skb, table_attr);
1636         nlmsg_end(skb, nlh);
1637         cb->args[0] = idx;
1638
1639         /*
1640          * No more entries to fill, cancel the message and
1641          * return 0 to mark end of dumpit.
1642          */
1643         if (!filled)
1644                 goto err;
1645
1646         ib_device_put(device);
1647         return skb->len;
1648
1649 res_err:
1650         nla_nest_cancel(skb, table_attr);
1651
1652 err:
1653         nlmsg_cancel(skb, nlh);
1654
1655 err_index:
1656         ib_device_put(device);
1657         return ret;
1658 }
1659
1660 #define RES_GET_FUNCS(name, type)                                              \
1661         static int nldev_res_get_##name##_dumpit(struct sk_buff *skb,          \
1662                                                  struct netlink_callback *cb)  \
1663         {                                                                      \
1664                 return res_get_common_dumpit(skb, cb, type,                    \
1665                                              fill_res_##name##_entry);         \
1666         }                                                                      \
1667         static int nldev_res_get_##name##_doit(struct sk_buff *skb,            \
1668                                                struct nlmsghdr *nlh,           \
1669                                                struct netlink_ext_ack *extack) \
1670         {                                                                      \
1671                 return res_get_common_doit(skb, nlh, extack, type,             \
1672                                            fill_res_##name##_entry);           \
1673         }
1674
1675 RES_GET_FUNCS(qp, RDMA_RESTRACK_QP);
1676 RES_GET_FUNCS(qp_raw, RDMA_RESTRACK_QP);
1677 RES_GET_FUNCS(cm_id, RDMA_RESTRACK_CM_ID);
1678 RES_GET_FUNCS(cq, RDMA_RESTRACK_CQ);
1679 RES_GET_FUNCS(cq_raw, RDMA_RESTRACK_CQ);
1680 RES_GET_FUNCS(pd, RDMA_RESTRACK_PD);
1681 RES_GET_FUNCS(mr, RDMA_RESTRACK_MR);
1682 RES_GET_FUNCS(mr_raw, RDMA_RESTRACK_MR);
1683 RES_GET_FUNCS(counter, RDMA_RESTRACK_COUNTER);
1684 RES_GET_FUNCS(ctx, RDMA_RESTRACK_CTX);
1685 RES_GET_FUNCS(srq, RDMA_RESTRACK_SRQ);
1686 RES_GET_FUNCS(srq_raw, RDMA_RESTRACK_SRQ);
1687
1688 static LIST_HEAD(link_ops);
1689 static DECLARE_RWSEM(link_ops_rwsem);
1690
1691 static const struct rdma_link_ops *link_ops_get(const char *type)
1692 {
1693         const struct rdma_link_ops *ops;
1694
1695         list_for_each_entry(ops, &link_ops, list) {
1696                 if (!strcmp(ops->type, type))
1697                         goto out;
1698         }
1699         ops = NULL;
1700 out:
1701         return ops;
1702 }
1703
1704 void rdma_link_register(struct rdma_link_ops *ops)
1705 {
1706         down_write(&link_ops_rwsem);
1707         if (WARN_ON_ONCE(link_ops_get(ops->type)))
1708                 goto out;
1709         list_add(&ops->list, &link_ops);
1710 out:
1711         up_write(&link_ops_rwsem);
1712 }
1713 EXPORT_SYMBOL(rdma_link_register);
1714
1715 void rdma_link_unregister(struct rdma_link_ops *ops)
1716 {
1717         down_write(&link_ops_rwsem);
1718         list_del(&ops->list);
1719         up_write(&link_ops_rwsem);
1720 }
1721 EXPORT_SYMBOL(rdma_link_unregister);
1722
1723 static int nldev_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,
1724                           struct netlink_ext_ack *extack)
1725 {
1726         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1727         char ibdev_name[IB_DEVICE_NAME_MAX];
1728         const struct rdma_link_ops *ops;
1729         char ndev_name[IFNAMSIZ];
1730         struct net_device *ndev;
1731         char type[IFNAMSIZ];
1732         int err;
1733
1734         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1735                                      nldev_policy, extack);
1736         if (err || !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
1737             !tb[RDMA_NLDEV_ATTR_LINK_TYPE] || !tb[RDMA_NLDEV_ATTR_NDEV_NAME])
1738                 return -EINVAL;
1739
1740         nla_strscpy(ibdev_name, tb[RDMA_NLDEV_ATTR_DEV_NAME],
1741                     sizeof(ibdev_name));
1742         if (strchr(ibdev_name, '%') || strlen(ibdev_name) == 0)
1743                 return -EINVAL;
1744
1745         nla_strscpy(type, tb[RDMA_NLDEV_ATTR_LINK_TYPE], sizeof(type));
1746         nla_strscpy(ndev_name, tb[RDMA_NLDEV_ATTR_NDEV_NAME],
1747                     sizeof(ndev_name));
1748
1749         ndev = dev_get_by_name(sock_net(skb->sk), ndev_name);
1750         if (!ndev)
1751                 return -ENODEV;
1752
1753         down_read(&link_ops_rwsem);
1754         ops = link_ops_get(type);
1755 #ifdef CONFIG_MODULES
1756         if (!ops) {
1757                 up_read(&link_ops_rwsem);
1758                 request_module("rdma-link-%s", type);
1759                 down_read(&link_ops_rwsem);
1760                 ops = link_ops_get(type);
1761         }
1762 #endif
1763         err = ops ? ops->newlink(ibdev_name, ndev) : -EINVAL;
1764         up_read(&link_ops_rwsem);
1765         dev_put(ndev);
1766
1767         return err;
1768 }
1769
1770 static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
1771                           struct netlink_ext_ack *extack)
1772 {
1773         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1774         struct ib_device *device;
1775         u32 index;
1776         int err;
1777
1778         err = nlmsg_parse_deprecated(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1779                                      nldev_policy, extack);
1780         if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
1781                 return -EINVAL;
1782
1783         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1784         device = ib_device_get_by_index(sock_net(skb->sk), index);
1785         if (!device)
1786                 return -EINVAL;
1787
1788         if (!(device->attrs.kernel_cap_flags & IBK_ALLOW_USER_UNREG)) {
1789                 ib_device_put(device);
1790                 return -EINVAL;
1791         }
1792
1793         ib_unregister_device_and_put(device);
1794         return 0;
1795 }
1796
1797 static int nldev_get_chardev(struct sk_buff *skb, struct nlmsghdr *nlh,
1798                              struct netlink_ext_ack *extack)
1799 {
1800         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1801         char client_name[RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE];
1802         struct ib_client_nl_info data = {};
1803         struct ib_device *ibdev = NULL;
1804         struct sk_buff *msg;
1805         u32 index;
1806         int err;
1807
1808         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
1809                           extack);
1810         if (err || !tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE])
1811                 return -EINVAL;
1812
1813         nla_strscpy(client_name, tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE],
1814                     sizeof(client_name));
1815
1816         if (tb[RDMA_NLDEV_ATTR_DEV_INDEX]) {
1817                 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
1818                 ibdev = ib_device_get_by_index(sock_net(skb->sk), index);
1819                 if (!ibdev)
1820                         return -EINVAL;
1821
1822                 if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1823                         data.port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
1824                         if (!rdma_is_port_valid(ibdev, data.port)) {
1825                                 err = -EINVAL;
1826                                 goto out_put;
1827                         }
1828                 } else {
1829                         data.port = -1;
1830                 }
1831         } else if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
1832                 return -EINVAL;
1833         }
1834
1835         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1836         if (!msg) {
1837                 err = -ENOMEM;
1838                 goto out_put;
1839         }
1840         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1841                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1842                                          RDMA_NLDEV_CMD_GET_CHARDEV),
1843                         0, 0);
1844         if (!nlh) {
1845                 err = -EMSGSIZE;
1846                 goto out_nlmsg;
1847         }
1848
1849         data.nl_msg = msg;
1850         err = ib_get_client_nl_info(ibdev, client_name, &data);
1851         if (err)
1852                 goto out_nlmsg;
1853
1854         err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV,
1855                                 huge_encode_dev(data.cdev->devt),
1856                                 RDMA_NLDEV_ATTR_PAD);
1857         if (err)
1858                 goto out_data;
1859         err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV_ABI, data.abi,
1860                                 RDMA_NLDEV_ATTR_PAD);
1861         if (err)
1862                 goto out_data;
1863         if (nla_put_string(msg, RDMA_NLDEV_ATTR_CHARDEV_NAME,
1864                            dev_name(data.cdev))) {
1865                 err = -EMSGSIZE;
1866                 goto out_data;
1867         }
1868
1869         nlmsg_end(msg, nlh);
1870         put_device(data.cdev);
1871         if (ibdev)
1872                 ib_device_put(ibdev);
1873         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1874
1875 out_data:
1876         put_device(data.cdev);
1877 out_nlmsg:
1878         nlmsg_free(msg);
1879 out_put:
1880         if (ibdev)
1881                 ib_device_put(ibdev);
1882         return err;
1883 }
1884
1885 static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1886                               struct netlink_ext_ack *extack)
1887 {
1888         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1889         struct sk_buff *msg;
1890         int err;
1891
1892         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1893                           nldev_policy, extack);
1894         if (err)
1895                 return err;
1896
1897         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1898         if (!msg)
1899                 return -ENOMEM;
1900
1901         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
1902                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
1903                                          RDMA_NLDEV_CMD_SYS_GET),
1904                         0, 0);
1905         if (!nlh) {
1906                 nlmsg_free(msg);
1907                 return -EMSGSIZE;
1908         }
1909
1910         err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_NETNS_MODE,
1911                          (u8)ib_devices_shared_netns);
1912         if (err) {
1913                 nlmsg_free(msg);
1914                 return err;
1915         }
1916
1917         err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE,
1918                          (u8)privileged_qkey);
1919         if (err) {
1920                 nlmsg_free(msg);
1921                 return err;
1922         }
1923         /*
1924          * Copy-on-fork is supported.
1925          * See commits:
1926          * 70e806e4e645 ("mm: Do early cow for pinned pages during fork() for ptes")
1927          * 4eae4efa2c29 ("hugetlb: do early cow when page pinned on src mm")
1928          * for more details. Don't backport this without them.
1929          *
1930          * Return value ignored on purpose, assume copy-on-fork is not
1931          * supported in case of failure.
1932          */
1933         nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK, 1);
1934
1935         nlmsg_end(msg, nlh);
1936         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
1937 }
1938
1939 static int nldev_set_sys_set_netns_doit(struct nlattr *tb[])
1940 {
1941         u8 enable;
1942         int err;
1943
1944         enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]);
1945         /* Only 0 and 1 are supported */
1946         if (enable > 1)
1947                 return -EINVAL;
1948
1949         err = rdma_compatdev_set(enable);
1950         return err;
1951 }
1952
1953 static int nldev_set_sys_set_pqkey_doit(struct nlattr *tb[])
1954 {
1955         u8 enable;
1956
1957         enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE]);
1958         /* Only 0 and 1 are supported */
1959         if (enable > 1)
1960                 return -EINVAL;
1961
1962         privileged_qkey = enable;
1963         return 0;
1964 }
1965
1966 static int nldev_set_sys_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
1967                                   struct netlink_ext_ack *extack)
1968 {
1969         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
1970         int err;
1971
1972         err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
1973                           nldev_policy, extack);
1974         if (err)
1975                 return -EINVAL;
1976
1977         if (tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE])
1978                 return nldev_set_sys_set_netns_doit(tb);
1979
1980         if (tb[RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE])
1981                 return nldev_set_sys_set_pqkey_doit(tb);
1982
1983         return -EINVAL;
1984 }
1985
1986
1987 static int nldev_stat_set_mode_doit(struct sk_buff *msg,
1988                                     struct netlink_ext_ack *extack,
1989                                     struct nlattr *tb[],
1990                                     struct ib_device *device, u32 port)
1991 {
1992         u32 mode, mask = 0, qpn, cntn = 0;
1993         int ret;
1994
1995         /* Currently only counter for QP is supported */
1996         if (!tb[RDMA_NLDEV_ATTR_STAT_RES] ||
1997             nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP)
1998                 return -EINVAL;
1999
2000         mode = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_MODE]);
2001         if (mode == RDMA_COUNTER_MODE_AUTO) {
2002                 if (tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK])
2003                         mask = nla_get_u32(
2004                                 tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]);
2005                 return rdma_counter_set_auto_mode(device, port, mask, extack);
2006         }
2007
2008         if (!tb[RDMA_NLDEV_ATTR_RES_LQPN])
2009                 return -EINVAL;
2010
2011         qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);
2012         if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]) {
2013                 cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);
2014                 ret = rdma_counter_bind_qpn(device, port, qpn, cntn);
2015                 if (ret)
2016                         return ret;
2017         } else {
2018                 ret = rdma_counter_bind_qpn_alloc(device, port, qpn, &cntn);
2019                 if (ret)
2020                         return ret;
2021         }
2022
2023         if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) ||
2024             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) {
2025                 ret = -EMSGSIZE;
2026                 goto err_fill;
2027         }
2028
2029         return 0;
2030
2031 err_fill:
2032         rdma_counter_unbind_qpn(device, port, qpn, cntn);
2033         return ret;
2034 }
2035
2036 static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[],
2037                                                struct ib_device *device,
2038                                                u32 port)
2039 {
2040         struct rdma_hw_stats *stats;
2041         struct nlattr *entry_attr;
2042         unsigned long *target;
2043         int rem, i, ret = 0;
2044         u32 index;
2045
2046         stats = ib_get_hw_stats_port(device, port);
2047         if (!stats)
2048                 return -EINVAL;
2049
2050         target = kcalloc(BITS_TO_LONGS(stats->num_counters),
2051                          sizeof(*stats->is_disabled), GFP_KERNEL);
2052         if (!target)
2053                 return -ENOMEM;
2054
2055         nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS],
2056                             rem) {
2057                 index = nla_get_u32(entry_attr);
2058                 if ((index >= stats->num_counters) ||
2059                     !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) {
2060                         ret = -EINVAL;
2061                         goto out;
2062                 }
2063
2064                 set_bit(index, target);
2065         }
2066
2067         for (i = 0; i < stats->num_counters; i++) {
2068                 if (!(stats->descs[i].flags & IB_STAT_FLAG_OPTIONAL))
2069                         continue;
2070
2071                 ret = rdma_counter_modify(device, port, i, test_bit(i, target));
2072                 if (ret)
2073                         goto out;
2074         }
2075
2076 out:
2077         kfree(target);
2078         return ret;
2079 }
2080
2081 static int nldev_stat_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
2082                                struct netlink_ext_ack *extack)
2083 {
2084         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2085         struct ib_device *device;
2086         struct sk_buff *msg;
2087         u32 index, port;
2088         int ret;
2089
2090         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy,
2091                           extack);
2092         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
2093             !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2094                 return -EINVAL;
2095
2096         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2097         device = ib_device_get_by_index(sock_net(skb->sk), index);
2098         if (!device)
2099                 return -EINVAL;
2100
2101         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2102         if (!rdma_is_port_valid(device, port)) {
2103                 ret = -EINVAL;
2104                 goto err_put_device;
2105         }
2106
2107         if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] &&
2108             !tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) {
2109                 ret = -EINVAL;
2110                 goto err_put_device;
2111         }
2112
2113         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2114         if (!msg) {
2115                 ret = -ENOMEM;
2116                 goto err_put_device;
2117         }
2118         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2119                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2120                                          RDMA_NLDEV_CMD_STAT_SET),
2121                         0, 0);
2122         if (!nlh || fill_nldev_handle(msg, device) ||
2123             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
2124                 ret = -EMSGSIZE;
2125                 goto err_free_msg;
2126         }
2127
2128         if (tb[RDMA_NLDEV_ATTR_STAT_MODE]) {
2129                 ret = nldev_stat_set_mode_doit(msg, extack, tb, device, port);
2130                 if (ret)
2131                         goto err_free_msg;
2132         }
2133
2134         if (tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) {
2135                 ret = nldev_stat_set_counter_dynamic_doit(tb, device, port);
2136                 if (ret)
2137                         goto err_free_msg;
2138         }
2139
2140         nlmsg_end(msg, nlh);
2141         ib_device_put(device);
2142         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2143
2144 err_free_msg:
2145         nlmsg_free(msg);
2146 err_put_device:
2147         ib_device_put(device);
2148         return ret;
2149 }
2150
2151 static int nldev_stat_del_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
2152                                struct netlink_ext_ack *extack)
2153 {
2154         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2155         struct ib_device *device;
2156         struct sk_buff *msg;
2157         u32 index, port, qpn, cntn;
2158         int ret;
2159
2160         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2161                           nldev_policy, extack);
2162         if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES] ||
2163             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX] ||
2164             !tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID] ||
2165             !tb[RDMA_NLDEV_ATTR_RES_LQPN])
2166                 return -EINVAL;
2167
2168         if (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP)
2169                 return -EINVAL;
2170
2171         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2172         device = ib_device_get_by_index(sock_net(skb->sk), index);
2173         if (!device)
2174                 return -EINVAL;
2175
2176         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2177         if (!rdma_is_port_valid(device, port)) {
2178                 ret = -EINVAL;
2179                 goto err;
2180         }
2181
2182         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2183         if (!msg) {
2184                 ret = -ENOMEM;
2185                 goto err;
2186         }
2187         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2188                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2189                                          RDMA_NLDEV_CMD_STAT_SET),
2190                         0, 0);
2191         if (!nlh) {
2192                 ret = -EMSGSIZE;
2193                 goto err_fill;
2194         }
2195
2196         cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]);
2197         qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]);
2198         if (fill_nldev_handle(msg, device) ||
2199             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||
2200             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) ||
2201             nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) {
2202                 ret = -EMSGSIZE;
2203                 goto err_fill;
2204         }
2205
2206         ret = rdma_counter_unbind_qpn(device, port, qpn, cntn);
2207         if (ret)
2208                 goto err_fill;
2209
2210         nlmsg_end(msg, nlh);
2211         ib_device_put(device);
2212         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2213
2214 err_fill:
2215         nlmsg_free(msg);
2216 err:
2217         ib_device_put(device);
2218         return ret;
2219 }
2220
2221 static int stat_get_doit_default_counter(struct sk_buff *skb,
2222                                          struct nlmsghdr *nlh,
2223                                          struct netlink_ext_ack *extack,
2224                                          struct nlattr *tb[])
2225 {
2226         struct rdma_hw_stats *stats;
2227         struct nlattr *table_attr;
2228         struct ib_device *device;
2229         int ret, num_cnts, i;
2230         struct sk_buff *msg;
2231         u32 index, port;
2232         u64 v;
2233
2234         if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2235                 return -EINVAL;
2236
2237         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2238         device = ib_device_get_by_index(sock_net(skb->sk), index);
2239         if (!device)
2240                 return -EINVAL;
2241
2242         if (!device->ops.alloc_hw_port_stats || !device->ops.get_hw_stats) {
2243                 ret = -EINVAL;
2244                 goto err;
2245         }
2246
2247         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2248         stats = ib_get_hw_stats_port(device, port);
2249         if (!stats) {
2250                 ret = -EINVAL;
2251                 goto err;
2252         }
2253
2254         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2255         if (!msg) {
2256                 ret = -ENOMEM;
2257                 goto err;
2258         }
2259
2260         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2261                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2262                                          RDMA_NLDEV_CMD_STAT_GET),
2263                         0, 0);
2264
2265         if (!nlh || fill_nldev_handle(msg, device) ||
2266             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) {
2267                 ret = -EMSGSIZE;
2268                 goto err_msg;
2269         }
2270
2271         mutex_lock(&stats->lock);
2272
2273         num_cnts = device->ops.get_hw_stats(device, stats, port, 0);
2274         if (num_cnts < 0) {
2275                 ret = -EINVAL;
2276                 goto err_stats;
2277         }
2278
2279         table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
2280         if (!table_attr) {
2281                 ret = -EMSGSIZE;
2282                 goto err_stats;
2283         }
2284         for (i = 0; i < num_cnts; i++) {
2285                 if (test_bit(i, stats->is_disabled))
2286                         continue;
2287
2288                 v = stats->value[i] +
2289                         rdma_counter_get_hwstat_value(device, port, i);
2290                 if (rdma_nl_stat_hwcounter_entry(msg,
2291                                                  stats->descs[i].name, v)) {
2292                         ret = -EMSGSIZE;
2293                         goto err_table;
2294                 }
2295         }
2296         nla_nest_end(msg, table_attr);
2297
2298         mutex_unlock(&stats->lock);
2299         nlmsg_end(msg, nlh);
2300         ib_device_put(device);
2301         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2302
2303 err_table:
2304         nla_nest_cancel(msg, table_attr);
2305 err_stats:
2306         mutex_unlock(&stats->lock);
2307 err_msg:
2308         nlmsg_free(msg);
2309 err:
2310         ib_device_put(device);
2311         return ret;
2312 }
2313
2314 static int stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh,
2315                             struct netlink_ext_ack *extack, struct nlattr *tb[])
2316
2317 {
2318         static enum rdma_nl_counter_mode mode;
2319         static enum rdma_nl_counter_mask mask;
2320         struct ib_device *device;
2321         struct sk_buff *msg;
2322         u32 index, port;
2323         int ret;
2324
2325         if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID])
2326                 return nldev_res_get_counter_doit(skb, nlh, extack);
2327
2328         if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] ||
2329             !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2330                 return -EINVAL;
2331
2332         index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2333         device = ib_device_get_by_index(sock_net(skb->sk), index);
2334         if (!device)
2335                 return -EINVAL;
2336
2337         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2338         if (!rdma_is_port_valid(device, port)) {
2339                 ret = -EINVAL;
2340                 goto err;
2341         }
2342
2343         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2344         if (!msg) {
2345                 ret = -ENOMEM;
2346                 goto err;
2347         }
2348
2349         nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2350                         RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2351                                          RDMA_NLDEV_CMD_STAT_GET),
2352                         0, 0);
2353         if (!nlh) {
2354                 ret = -EMSGSIZE;
2355                 goto err_msg;
2356         }
2357
2358         ret = rdma_counter_get_mode(device, port, &mode, &mask);
2359         if (ret)
2360                 goto err_msg;
2361
2362         if (fill_nldev_handle(msg, device) ||
2363             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) ||
2364             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, mode)) {
2365                 ret = -EMSGSIZE;
2366                 goto err_msg;
2367         }
2368
2369         if ((mode == RDMA_COUNTER_MODE_AUTO) &&
2370             nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK, mask)) {
2371                 ret = -EMSGSIZE;
2372                 goto err_msg;
2373         }
2374
2375         nlmsg_end(msg, nlh);
2376         ib_device_put(device);
2377         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2378
2379 err_msg:
2380         nlmsg_free(msg);
2381 err:
2382         ib_device_put(device);
2383         return ret;
2384 }
2385
2386 static int nldev_stat_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
2387                                struct netlink_ext_ack *extack)
2388 {
2389         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2390         int ret;
2391
2392         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2393                           nldev_policy, extack);
2394         if (ret)
2395                 return -EINVAL;
2396
2397         if (!tb[RDMA_NLDEV_ATTR_STAT_RES])
2398                 return stat_get_doit_default_counter(skb, nlh, extack, tb);
2399
2400         switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) {
2401         case RDMA_NLDEV_ATTR_RES_QP:
2402                 ret = stat_get_doit_qp(skb, nlh, extack, tb);
2403                 break;
2404         case RDMA_NLDEV_ATTR_RES_MR:
2405                 ret = res_get_common_doit(skb, nlh, extack, RDMA_RESTRACK_MR,
2406                                           fill_stat_mr_entry);
2407                 break;
2408         default:
2409                 ret = -EINVAL;
2410                 break;
2411         }
2412
2413         return ret;
2414 }
2415
2416 static int nldev_stat_get_dumpit(struct sk_buff *skb,
2417                                  struct netlink_callback *cb)
2418 {
2419         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2420         int ret;
2421
2422         ret = nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2423                           nldev_policy, NULL);
2424         if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES])
2425                 return -EINVAL;
2426
2427         switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) {
2428         case RDMA_NLDEV_ATTR_RES_QP:
2429                 ret = nldev_res_get_counter_dumpit(skb, cb);
2430                 break;
2431         case RDMA_NLDEV_ATTR_RES_MR:
2432                 ret = res_get_common_dumpit(skb, cb, RDMA_RESTRACK_MR,
2433                                             fill_stat_mr_entry);
2434                 break;
2435         default:
2436                 ret = -EINVAL;
2437                 break;
2438         }
2439
2440         return ret;
2441 }
2442
2443 static int nldev_stat_get_counter_status_doit(struct sk_buff *skb,
2444                                               struct nlmsghdr *nlh,
2445                                               struct netlink_ext_ack *extack)
2446 {
2447         struct nlattr *tb[RDMA_NLDEV_ATTR_MAX], *table, *entry;
2448         struct rdma_hw_stats *stats;
2449         struct ib_device *device;
2450         struct sk_buff *msg;
2451         u32 devid, port;
2452         int ret, i;
2453
2454         ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2455                           nldev_policy, extack);
2456         if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] ||
2457             !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
2458                 return -EINVAL;
2459
2460         devid = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
2461         device = ib_device_get_by_index(sock_net(skb->sk), devid);
2462         if (!device)
2463                 return -EINVAL;
2464
2465         port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
2466         if (!rdma_is_port_valid(device, port)) {
2467                 ret = -EINVAL;
2468                 goto err;
2469         }
2470
2471         stats = ib_get_hw_stats_port(device, port);
2472         if (!stats) {
2473                 ret = -EINVAL;
2474                 goto err;
2475         }
2476
2477         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2478         if (!msg) {
2479                 ret = -ENOMEM;
2480                 goto err;
2481         }
2482
2483         nlh = nlmsg_put(
2484                 msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
2485                 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_STAT_GET_STATUS),
2486                 0, 0);
2487
2488         ret = -EMSGSIZE;
2489         if (!nlh || fill_nldev_handle(msg, device) ||
2490             nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
2491                 goto err_msg;
2492
2493         table = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS);
2494         if (!table)
2495                 goto err_msg;
2496
2497         mutex_lock(&stats->lock);
2498         for (i = 0; i < stats->num_counters; i++) {
2499                 entry = nla_nest_start(msg,
2500                                        RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY);
2501                 if (!entry)
2502                         goto err_msg_table;
2503
2504                 if (nla_put_string(msg,
2505                                    RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME,
2506                                    stats->descs[i].name) ||
2507                     nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX, i))
2508                         goto err_msg_entry;
2509
2510                 if ((stats->descs[i].flags & IB_STAT_FLAG_OPTIONAL) &&
2511                     (nla_put_u8(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC,
2512                                 !test_bit(i, stats->is_disabled))))
2513                         goto err_msg_entry;
2514
2515                 nla_nest_end(msg, entry);
2516         }
2517         mutex_unlock(&stats->lock);
2518
2519         nla_nest_end(msg, table);
2520         nlmsg_end(msg, nlh);
2521         ib_device_put(device);
2522         return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid);
2523
2524 err_msg_entry:
2525         nla_nest_cancel(msg, entry);
2526 err_msg_table:
2527         mutex_unlock(&stats->lock);
2528         nla_nest_cancel(msg, table);
2529 err_msg:
2530         nlmsg_free(msg);
2531 err:
2532         ib_device_put(device);
2533         return ret;
2534 }
2535
2536 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
2537         [RDMA_NLDEV_CMD_GET] = {
2538                 .doit = nldev_get_doit,
2539                 .dump = nldev_get_dumpit,
2540         },
2541         [RDMA_NLDEV_CMD_GET_CHARDEV] = {
2542                 .doit = nldev_get_chardev,
2543         },
2544         [RDMA_NLDEV_CMD_SET] = {
2545                 .doit = nldev_set_doit,
2546                 .flags = RDMA_NL_ADMIN_PERM,
2547         },
2548         [RDMA_NLDEV_CMD_NEWLINK] = {
2549                 .doit = nldev_newlink,
2550                 .flags = RDMA_NL_ADMIN_PERM,
2551         },
2552         [RDMA_NLDEV_CMD_DELLINK] = {
2553                 .doit = nldev_dellink,
2554                 .flags = RDMA_NL_ADMIN_PERM,
2555         },
2556         [RDMA_NLDEV_CMD_PORT_GET] = {
2557                 .doit = nldev_port_get_doit,
2558                 .dump = nldev_port_get_dumpit,
2559         },
2560         [RDMA_NLDEV_CMD_RES_GET] = {
2561                 .doit = nldev_res_get_doit,
2562                 .dump = nldev_res_get_dumpit,
2563         },
2564         [RDMA_NLDEV_CMD_RES_QP_GET] = {
2565                 .doit = nldev_res_get_qp_doit,
2566                 .dump = nldev_res_get_qp_dumpit,
2567         },
2568         [RDMA_NLDEV_CMD_RES_CM_ID_GET] = {
2569                 .doit = nldev_res_get_cm_id_doit,
2570                 .dump = nldev_res_get_cm_id_dumpit,
2571         },
2572         [RDMA_NLDEV_CMD_RES_CQ_GET] = {
2573                 .doit = nldev_res_get_cq_doit,
2574                 .dump = nldev_res_get_cq_dumpit,
2575         },
2576         [RDMA_NLDEV_CMD_RES_MR_GET] = {
2577                 .doit = nldev_res_get_mr_doit,
2578                 .dump = nldev_res_get_mr_dumpit,
2579         },
2580         [RDMA_NLDEV_CMD_RES_PD_GET] = {
2581                 .doit = nldev_res_get_pd_doit,
2582                 .dump = nldev_res_get_pd_dumpit,
2583         },
2584         [RDMA_NLDEV_CMD_RES_CTX_GET] = {
2585                 .doit = nldev_res_get_ctx_doit,
2586                 .dump = nldev_res_get_ctx_dumpit,
2587         },
2588         [RDMA_NLDEV_CMD_RES_SRQ_GET] = {
2589                 .doit = nldev_res_get_srq_doit,
2590                 .dump = nldev_res_get_srq_dumpit,
2591         },
2592         [RDMA_NLDEV_CMD_SYS_GET] = {
2593                 .doit = nldev_sys_get_doit,
2594         },
2595         [RDMA_NLDEV_CMD_SYS_SET] = {
2596                 .doit = nldev_set_sys_set_doit,
2597                 .flags = RDMA_NL_ADMIN_PERM,
2598         },
2599         [RDMA_NLDEV_CMD_STAT_SET] = {
2600                 .doit = nldev_stat_set_doit,
2601                 .flags = RDMA_NL_ADMIN_PERM,
2602         },
2603         [RDMA_NLDEV_CMD_STAT_GET] = {
2604                 .doit = nldev_stat_get_doit,
2605                 .dump = nldev_stat_get_dumpit,
2606         },
2607         [RDMA_NLDEV_CMD_STAT_DEL] = {
2608                 .doit = nldev_stat_del_doit,
2609                 .flags = RDMA_NL_ADMIN_PERM,
2610         },
2611         [RDMA_NLDEV_CMD_RES_QP_GET_RAW] = {
2612                 .doit = nldev_res_get_qp_raw_doit,
2613                 .dump = nldev_res_get_qp_raw_dumpit,
2614                 .flags = RDMA_NL_ADMIN_PERM,
2615         },
2616         [RDMA_NLDEV_CMD_RES_CQ_GET_RAW] = {
2617                 .doit = nldev_res_get_cq_raw_doit,
2618                 .dump = nldev_res_get_cq_raw_dumpit,
2619                 .flags = RDMA_NL_ADMIN_PERM,
2620         },
2621         [RDMA_NLDEV_CMD_RES_MR_GET_RAW] = {
2622                 .doit = nldev_res_get_mr_raw_doit,
2623                 .dump = nldev_res_get_mr_raw_dumpit,
2624                 .flags = RDMA_NL_ADMIN_PERM,
2625         },
2626         [RDMA_NLDEV_CMD_RES_SRQ_GET_RAW] = {
2627                 .doit = nldev_res_get_srq_raw_doit,
2628                 .dump = nldev_res_get_srq_raw_dumpit,
2629                 .flags = RDMA_NL_ADMIN_PERM,
2630         },
2631         [RDMA_NLDEV_CMD_STAT_GET_STATUS] = {
2632                 .doit = nldev_stat_get_counter_status_doit,
2633         },
2634 };
2635
2636 void __init nldev_init(void)
2637 {
2638         rdma_nl_register(RDMA_NL_NLDEV, nldev_cb_table);
2639 }
2640
2641 void nldev_exit(void)
2642 {
2643         rdma_nl_unregister(RDMA_NL_NLDEV);
2644 }
2645
2646 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_NLDEV, 5);