net/smc/smc_llc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  Link Layer Control (LLC)
   6  *
   7  *  Copyright IBM Corp. 2016
   8  *
   9  *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
  10  *              Ursula Braun <ubraun@linux.vnet.ibm.com>
  11  */
  12
  13 #include <net/tcp.h>
  14 #include <rdma/ib_verbs.h>
  15
  16 #include "smc.h"
  17 #include "smc_core.h"
  18 #include "smc_clc.h"
  19 #include "smc_llc.h"
  20 #include "smc_pnet.h"
  21
  22 #define SMC_LLC_DATA_LEN                40
  23
  24 struct smc_llc_hdr {
  25         struct smc_wr_rx_hdr common;
  26         union {
  27                 struct {
  28                         u8 length;      /* 44 */
  29         #if defined(__BIG_ENDIAN_BITFIELD)
  30                         u8 reserved:4,
  31                            add_link_rej_rsn:4;
  32 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  33                         u8 add_link_rej_rsn:4,
  34                            reserved:4;
  35 #endif
  36                 };
  37                 u16 length_v2;  /* 44 - 8192*/
  38         };
  39         u8 flags;
  40 } __packed;             /* format defined in
  41                          * IBM Shared Memory Communications Version 2
  42                          * (https://www.ibm.com/support/pages/node/6326337)
  43                          */
  44
  45 #define SMC_LLC_FLAG_NO_RMBE_EYEC       0x03
  46
  47 struct smc_llc_msg_confirm_link {       /* type 0x01 */
  48         struct smc_llc_hdr hd;
  49         u8 sender_mac[ETH_ALEN];
  50         u8 sender_gid[SMC_GID_SIZE];
  51         u8 sender_qp_num[3];
  52         u8 link_num;
  53         u8 link_uid[SMC_LGR_ID_SIZE];
  54         u8 max_links;
  55         u8 reserved[9];
  56 };
  57
  58 #define SMC_LLC_FLAG_ADD_LNK_REJ        0x40
  59 #define SMC_LLC_REJ_RSN_NO_ALT_PATH     1
  60
  61 #define SMC_LLC_ADD_LNK_MAX_LINKS       2
  62
  63 struct smc_llc_msg_add_link {           /* type 0x02 */
  64         struct smc_llc_hdr hd;
  65         u8 sender_mac[ETH_ALEN];
  66         u8 reserved2[2];
  67         u8 sender_gid[SMC_GID_SIZE];
  68         u8 sender_qp_num[3];
  69         u8 link_num;
  70 #if defined(__BIG_ENDIAN_BITFIELD)
  71         u8 reserved3 : 4,
  72            qp_mtu   : 4;
  73 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  74         u8 qp_mtu   : 4,
  75            reserved3 : 4;
  76 #endif
  77         u8 initial_psn[3];
  78         u8 reserved[8];
  79 };
  80
  81 struct smc_llc_msg_add_link_cont_rt {
  82         __be32 rmb_key;
  83         __be32 rmb_key_new;
  84         __be64 rmb_vaddr_new;
  85 };
  86
  87 struct smc_llc_msg_add_link_v2_ext {
  88 #if defined(__BIG_ENDIAN_BITFIELD)
  89         u8 v2_direct : 1,
  90            reserved  : 7;
  91 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  92         u8 reserved  : 7,
  93            v2_direct : 1;
  94 #endif
  95         u8 reserved2;
  96         u8 client_target_gid[SMC_GID_SIZE];
  97         u8 reserved3[8];
  98         u16 num_rkeys;
  99         struct smc_llc_msg_add_link_cont_rt rt[];
 100 } __packed;             /* format defined in
 101                          * IBM Shared Memory Communications Version 2
 102                          * (https://www.ibm.com/support/pages/node/6326337)
 103                          */
 104
 105 struct smc_llc_msg_req_add_link_v2 {
 106         struct smc_llc_hdr hd;
 107         u8 reserved[20];
 108         u8 gid_cnt;
 109         u8 reserved2[3];
 110         u8 gid[][SMC_GID_SIZE];
 111 };
 112
 113 #define SMC_LLC_RKEYS_PER_CONT_MSG      2
 114
 115 struct smc_llc_msg_add_link_cont {      /* type 0x03 */
 116         struct smc_llc_hdr hd;
 117         u8 link_num;
 118         u8 num_rkeys;
 119         u8 reserved2[2];
 120         struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
 121         u8 reserved[4];
 122 } __packed;                     /* format defined in RFC7609 */
 123
 124 #define SMC_LLC_FLAG_DEL_LINK_ALL       0x40
 125 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY   0x20
 126
 127 struct smc_llc_msg_del_link {           /* type 0x04 */
 128         struct smc_llc_hdr hd;
 129         u8 link_num;
 130         __be32 reason;
 131         u8 reserved[35];
 132 } __packed;                     /* format defined in RFC7609 */
 133
 134 struct smc_llc_msg_test_link {          /* type 0x07 */
 135         struct smc_llc_hdr hd;
 136         u8 user_data[16];
 137         u8 reserved[24];
 138 };
 139
 140 struct smc_rmb_rtoken {
 141         union {
 142                 u8 num_rkeys;   /* first rtoken byte of CONFIRM LINK msg */
 143                                 /* is actually the num of rtokens, first */
 144                                 /* rtoken is always for the current link */
 145                 u8 link_id;     /* link id of the rtoken */
 146         };
 147         __be32 rmb_key;
 148         __be64 rmb_vaddr;
 149 } __packed;                     /* format defined in RFC7609 */
 150
 151 #define SMC_LLC_RKEYS_PER_MSG           3
 152 #define SMC_LLC_RKEYS_PER_MSG_V2        255
 153
 154 struct smc_llc_msg_confirm_rkey {       /* type 0x06 */
 155         struct smc_llc_hdr hd;
 156         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
 157         u8 reserved;
 158 };
 159
 160 #define SMC_LLC_DEL_RKEY_MAX    8
 161 #define SMC_LLC_FLAG_RKEY_RETRY 0x10
 162 #define SMC_LLC_FLAG_RKEY_NEG   0x20
 163
 164 struct smc_llc_msg_delete_rkey {        /* type 0x09 */
 165         struct smc_llc_hdr hd;
 166         u8 num_rkeys;
 167         u8 err_mask;
 168         u8 reserved[2];
 169         __be32 rkey[8];
 170         u8 reserved2[4];
 171 };
 172
 173 struct smc_llc_msg_delete_rkey_v2 {     /* type 0x29 */
 174         struct smc_llc_hdr hd;
 175         u8 num_rkeys;
 176         u8 num_inval_rkeys;
 177         u8 reserved[2];
 178         __be32 rkey[];
 179 };
 180
 181 union smc_llc_msg {
 182         struct smc_llc_msg_confirm_link confirm_link;
 183         struct smc_llc_msg_add_link add_link;
 184         struct smc_llc_msg_req_add_link_v2 req_add_link;
 185         struct smc_llc_msg_add_link_cont add_link_cont;
 186         struct smc_llc_msg_del_link delete_link;
 187
 188         struct smc_llc_msg_confirm_rkey confirm_rkey;
 189         struct smc_llc_msg_delete_rkey delete_rkey;
 190
 191         struct smc_llc_msg_test_link test_link;
 192         struct {
 193                 struct smc_llc_hdr hdr;
 194                 u8 data[SMC_LLC_DATA_LEN];
 195         } raw;
 196 };
 197
 198 #define SMC_LLC_FLAG_RESP               0x80
 199
 200 struct smc_llc_qentry {
 201         struct list_head list;
 202         struct smc_link *link;
 203         union smc_llc_msg msg;
 204 };
 205
 206 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
 207
 208 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
 209 {
 210         struct smc_llc_qentry *qentry = flow->qentry;
 211
 212         flow->qentry = NULL;
 213         return qentry;
 214 }
 215
 216 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
 217 {
 218         struct smc_llc_qentry *qentry;
 219
 220         if (flow->qentry) {
 221                 qentry = flow->qentry;
 222                 flow->qentry = NULL;
 223                 kfree(qentry);
 224         }
 225 }
 226
 227 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
 228                                            struct smc_llc_qentry *qentry)
 229 {
 230         flow->qentry = qentry;
 231 }
 232
 233 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,
 234                                   struct smc_llc_qentry *qentry)
 235 {
 236         u8 msg_type = qentry->msg.raw.hdr.common.llc_type;
 237
 238         if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) &&
 239             flow_type != msg_type && !lgr->delayed_event) {
 240                 lgr->delayed_event = qentry;
 241                 return;
 242         }
 243         /* drop parallel or already-in-progress llc requests */
 244         if (flow_type != msg_type)
 245                 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel "
 246                              "LLC msg: msg %d flow %d role %d\n",
 247                              SMC_LGR_ID_SIZE, &lgr->id,
 248                              lgr->net->net_cookie,
 249                              qentry->msg.raw.hdr.common.type,
 250                              flow_type, lgr->role);
 251         kfree(qentry);
 252 }
 253
 254 /* try to start a new llc flow, initiated by an incoming llc msg */
 255 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
 256                                struct smc_llc_qentry *qentry)
 257 {
 258         struct smc_link_group *lgr = qentry->link->lgr;
 259
 260         spin_lock_bh(&lgr->llc_flow_lock);
 261         if (flow->type) {
 262                 /* a flow is already active */
 263                 smc_llc_flow_parallel(lgr, flow->type, qentry);
 264                 spin_unlock_bh(&lgr->llc_flow_lock);
 265                 return false;
 266         }
 267         switch (qentry->msg.raw.hdr.common.llc_type) {
 268         case SMC_LLC_ADD_LINK:
 269                 flow->type = SMC_LLC_FLOW_ADD_LINK;
 270                 break;
 271         case SMC_LLC_DELETE_LINK:
 272                 flow->type = SMC_LLC_FLOW_DEL_LINK;
 273                 break;
 274         case SMC_LLC_CONFIRM_RKEY:
 275         case SMC_LLC_DELETE_RKEY:
 276                 flow->type = SMC_LLC_FLOW_RKEY;
 277                 break;
 278         default:
 279                 flow->type = SMC_LLC_FLOW_NONE;
 280         }
 281         smc_llc_flow_qentry_set(flow, qentry);
 282         spin_unlock_bh(&lgr->llc_flow_lock);
 283         return true;
 284 }
 285
 286 /* start a new local llc flow, wait till current flow finished */
 287 int smc_llc_flow_initiate(struct smc_link_group *lgr,
 288                           enum smc_llc_flowtype type)
 289 {
 290         enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
 291         int rc;
 292
 293         /* all flows except confirm_rkey and delete_rkey are exclusive,
 294          * confirm/delete rkey flows can run concurrently (local and remote)
 295          */
 296         if (type == SMC_LLC_FLOW_RKEY)
 297                 allowed_remote = SMC_LLC_FLOW_RKEY;
 298 again:
 299         if (list_empty(&lgr->list))
 300                 return -ENODEV;
 301         spin_lock_bh(&lgr->llc_flow_lock);
 302         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 303             (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 304              lgr->llc_flow_rmt.type == allowed_remote)) {
 305                 lgr->llc_flow_lcl.type = type;
 306                 spin_unlock_bh(&lgr->llc_flow_lock);
 307                 return 0;
 308         }
 309         spin_unlock_bh(&lgr->llc_flow_lock);
 310         rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||
 311                                 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 312                                  (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 313                                   lgr->llc_flow_rmt.type == allowed_remote))),
 314                                 SMC_LLC_WAIT_TIME * 10);
 315         if (!rc)
 316                 return -ETIMEDOUT;
 317         goto again;
 318 }
 319
 320 /* finish the current llc flow */
 321 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
 322 {
 323         spin_lock_bh(&lgr->llc_flow_lock);
 324         memset(flow, 0, sizeof(*flow));
 325         flow->type = SMC_LLC_FLOW_NONE;
 326         spin_unlock_bh(&lgr->llc_flow_lock);
 327         if (!list_empty(&lgr->list) && lgr->delayed_event &&
 328             flow == &lgr->llc_flow_lcl)
 329                 schedule_work(&lgr->llc_event_work);
 330         else
 331                 wake_up(&lgr->llc_flow_waiter);
 332 }
 333
 334 /* lnk is optional and used for early wakeup when link goes down, useful in
 335  * cases where we wait for a response on the link after we sent a request
 336  */
 337 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
 338                                     struct smc_link *lnk,
 339                                     int time_out, u8 exp_msg)
 340 {
 341         struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
 342         u8 rcv_msg;
 343
 344         wait_event_timeout(lgr->llc_msg_waiter,
 345                            (flow->qentry ||
 346                             (lnk && !smc_link_usable(lnk)) ||
 347                             list_empty(&lgr->list)),
 348                            time_out);
 349         if (!flow->qentry ||
 350             (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
 351                 smc_llc_flow_qentry_del(flow);
 352                 goto out;
 353         }
 354         rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type;
 355         if (exp_msg && rcv_msg != exp_msg) {
 356                 if (exp_msg == SMC_LLC_ADD_LINK &&
 357                     rcv_msg == SMC_LLC_DELETE_LINK) {
 358                         /* flow_start will delay the unexpected msg */
 359                         smc_llc_flow_start(&lgr->llc_flow_lcl,
 360                                            smc_llc_flow_qentry_clr(flow));
 361                         return NULL;
 362                 }
 363                 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: "
 364                              "msg %d exp %d flow %d role %d flags %x\n",
 365                              SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie,
 366                              rcv_msg, exp_msg,
 367                              flow->type, lgr->role,
 368                              flow->qentry->msg.raw.hdr.flags);
 369                 smc_llc_flow_qentry_del(flow);
 370         }
 371 out:
 372         return flow->qentry;
 373 }
 374
 375 /********************************** send *************************************/
 376
 377 struct smc_llc_tx_pend {
 378 };
 379
 380 /* handler for send/transmission completion of an LLC msg */
 381 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
 382                                struct smc_link *link,
 383                                enum ib_wc_status wc_status)
 384 {
 385         /* future work: handle wc_status error for recovery and failover */
 386 }
 387
 388 /**
 389  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
 390  * @link: Pointer to SMC link used for sending LLC control message.
 391  * @wr_buf: Out variable returning pointer to work request payload buffer.
 392  * @pend: Out variable returning pointer to private pending WR tracking.
 393  *        It's the context the transmit complete handler will get.
 394  *
 395  * Reserves and pre-fills an entry for a pending work request send/tx.
 396  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
 397  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
 398  *
 399  * Return: 0 on success, otherwise an error value.
 400  */
 401 static int smc_llc_add_pending_send(struct smc_link *link,
 402                                     struct smc_wr_buf **wr_buf,
 403                                     struct smc_wr_tx_pend_priv **pend)
 404 {
 405         int rc;
 406
 407         rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
 408                                      pend);
 409         if (rc < 0)
 410                 return rc;
 411         BUILD_BUG_ON_MSG(
 412                 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
 413                 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
 414         BUILD_BUG_ON_MSG(
 415                 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
 416                 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
 417         BUILD_BUG_ON_MSG(
 418                 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
 419                 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
 420         return 0;
 421 }
 422
 423 static int smc_llc_add_pending_send_v2(struct smc_link *link,
 424                                        struct smc_wr_v2_buf **wr_buf,
 425                                        struct smc_wr_tx_pend_priv **pend)
 426 {
 427         int rc;
 428
 429         rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend);
 430         if (rc < 0)
 431                 return rc;
 432         return 0;
 433 }
 434
 435 static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr,
 436                                  struct smc_link_group *lgr, size_t len)
 437 {
 438         if (lgr->smc_version == SMC_V2) {
 439                 hdr->common.llc_version = SMC_V2;
 440                 hdr->length_v2 = len;
 441         } else {
 442                 hdr->common.llc_version = 0;
 443                 hdr->length = len;
 444         }
 445 }
 446
 447 /* high-level API to send LLC confirm link */
 448 int smc_llc_send_confirm_link(struct smc_link *link,
 449                               enum smc_llc_reqresp reqresp)
 450 {
 451         struct smc_llc_msg_confirm_link *confllc;
 452         struct smc_wr_tx_pend_priv *pend;
 453         struct smc_wr_buf *wr_buf;
 454         int rc;
 455
 456         if (!smc_wr_tx_link_hold(link))
 457                 return -ENOLINK;
 458         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 459         if (rc)
 460                 goto put_out;
 461         confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
 462         memset(confllc, 0, sizeof(*confllc));
 463         confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK;
 464         smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc));
 465         confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
 466         if (reqresp == SMC_LLC_RESP)
 467                 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
 468         memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
 469                ETH_ALEN);
 470         memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
 471         hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
 472         confllc->link_num = link->link_id;
 473         memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
 474         confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS;
 475         /* send llc message */
 476         rc = smc_wr_tx_send(link, pend);
 477 put_out:
 478         smc_wr_tx_link_put(link);
 479         return rc;
 480 }
 481
 482 /* send LLC confirm rkey request */
 483 static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
 484                                      struct smc_buf_desc *rmb_desc)
 485 {
 486         struct smc_llc_msg_confirm_rkey *rkeyllc;
 487         struct smc_wr_tx_pend_priv *pend;
 488         struct smc_wr_buf *wr_buf;
 489         struct smc_link *link;
 490         int i, rc, rtok_ix;
 491
 492         if (!smc_wr_tx_link_hold(send_link))
 493                 return -ENOLINK;
 494         rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
 495         if (rc)
 496                 goto put_out;
 497         rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
 498         memset(rkeyllc, 0, sizeof(*rkeyllc));
 499         rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY;
 500         smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc));
 501
 502         rtok_ix = 1;
 503         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 504                 link = &send_link->lgr->lnk[i];
 505                 if (smc_link_active(link) && link != send_link) {
 506                         rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
 507                         rkeyllc->rtoken[rtok_ix].rmb_key =
 508                                 htonl(rmb_desc->mr[link->link_idx]->rkey);
 509                         rkeyllc->rtoken[rtok_ix].rmb_vaddr = rmb_desc->is_vm ?
 510                                 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
 511                                 cpu_to_be64((u64)sg_dma_address
 512                                             (rmb_desc->sgt[link->link_idx].sgl));
 513                         rtok_ix++;
 514                 }
 515         }
 516         /* rkey of send_link is in rtoken[0] */
 517         rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
 518         rkeyllc->rtoken[0].rmb_key =
 519                 htonl(rmb_desc->mr[send_link->link_idx]->rkey);
 520         rkeyllc->rtoken[0].rmb_vaddr = rmb_desc->is_vm ?
 521                 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
 522                 cpu_to_be64((u64)sg_dma_address
 523                             (rmb_desc->sgt[send_link->link_idx].sgl));
 524         /* send llc message */
 525         rc = smc_wr_tx_send(send_link, pend);
 526 put_out:
 527         smc_wr_tx_link_put(send_link);
 528         return rc;
 529 }
 530
 531 /* send LLC delete rkey request */
 532 static int smc_llc_send_delete_rkey(struct smc_link *link,
 533                                     struct smc_buf_desc *rmb_desc)
 534 {
 535         struct smc_llc_msg_delete_rkey *rkeyllc;
 536         struct smc_wr_tx_pend_priv *pend;
 537         struct smc_wr_buf *wr_buf;
 538         int rc;
 539
 540         if (!smc_wr_tx_link_hold(link))
 541                 return -ENOLINK;
 542         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 543         if (rc)
 544                 goto put_out;
 545         rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
 546         memset(rkeyllc, 0, sizeof(*rkeyllc));
 547         rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY;
 548         smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc));
 549         rkeyllc->num_rkeys = 1;
 550         rkeyllc->rkey[0] = htonl(rmb_desc->mr[link->link_idx]->rkey);
 551         /* send llc message */
 552         rc = smc_wr_tx_send(link, pend);
 553 put_out:
 554         smc_wr_tx_link_put(link);
 555         return rc;
 556 }
 557
 558 /* return first buffer from any of the next buf lists */
 559 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
 560                                                   int *buf_lst)
 561 {
 562         struct smc_buf_desc *buf_pos;
 563
 564         while (*buf_lst < SMC_RMBE_SIZES) {
 565                 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
 566                                                    struct smc_buf_desc, list);
 567                 if (buf_pos)
 568                         return buf_pos;
 569                 (*buf_lst)++;
 570         }
 571         return NULL;
 572 }
 573
 574 /* return next rmb from buffer lists */
 575 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
 576                                                  int *buf_lst,
 577                                                  struct smc_buf_desc *buf_pos)
 578 {
 579         struct smc_buf_desc *buf_next;
 580
 581         if (!buf_pos)
 582                 return _smc_llc_get_next_rmb(lgr, buf_lst);
 583
 584         if (list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
 585                 (*buf_lst)++;
 586                 return _smc_llc_get_next_rmb(lgr, buf_lst);
 587         }
 588         buf_next = list_next_entry(buf_pos, list);
 589         return buf_next;
 590 }
 591
 592 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
 593                                                   int *buf_lst)
 594 {
 595         *buf_lst = 0;
 596         return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
 597 }
 598
 599 static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext,
 600                                struct smc_link *link, struct smc_link *link_new)
 601 {
 602         struct smc_link_group *lgr = link->lgr;
 603         struct smc_buf_desc *buf_pos;
 604         int prim_lnk_idx, lnk_idx, i;
 605         struct smc_buf_desc *rmb;
 606         int len = sizeof(*ext);
 607         int buf_lst;
 608
 609         ext->v2_direct = !lgr->uses_gateway;
 610         memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE);
 611
 612         prim_lnk_idx = link->link_idx;
 613         lnk_idx = link_new->link_idx;
 614         down_write(&lgr->rmbs_lock);
 615         ext->num_rkeys = lgr->conns_num;
 616         if (!ext->num_rkeys)
 617                 goto out;
 618         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
 619         for (i = 0; i < ext->num_rkeys; i++) {
 620                 while (buf_pos && !(buf_pos)->used)
 621                         buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
 622                 if (!buf_pos)
 623                         break;
 624                 rmb = buf_pos;
 625                 ext->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
 626                 ext->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
 627                 ext->rt[i].rmb_vaddr_new = rmb->is_vm ?
 628                         cpu_to_be64((uintptr_t)rmb->cpu_addr) :
 629                         cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
 630                 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
 631         }
 632         len += i * sizeof(ext->rt[0]);
 633 out:
 634         up_write(&lgr->rmbs_lock);
 635         return len;
 636 }
 637
 638 /* send ADD LINK request or response */
 639 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
 640                           struct smc_link *link_new,
 641                           enum smc_llc_reqresp reqresp)
 642 {
 643         struct smc_llc_msg_add_link_v2_ext *ext = NULL;
 644         struct smc_llc_msg_add_link *addllc;
 645         struct smc_wr_tx_pend_priv *pend;
 646         int len = sizeof(*addllc);
 647         int rc;
 648
 649         if (!smc_wr_tx_link_hold(link))
 650                 return -ENOLINK;
 651         if (link->lgr->smc_version == SMC_V2) {
 652                 struct smc_wr_v2_buf *wr_buf;
 653
 654                 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
 655                 if (rc)
 656                         goto put_out;
 657                 addllc = (struct smc_llc_msg_add_link *)wr_buf;
 658                 ext = (struct smc_llc_msg_add_link_v2_ext *)
 659                                                 &wr_buf->raw[sizeof(*addllc)];
 660                 memset(ext, 0, SMC_WR_TX_SIZE);
 661         } else {
 662                 struct smc_wr_buf *wr_buf;
 663
 664                 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 665                 if (rc)
 666                         goto put_out;
 667                 addllc = (struct smc_llc_msg_add_link *)wr_buf;
 668         }
 669
 670         memset(addllc, 0, sizeof(*addllc));
 671         addllc->hd.common.llc_type = SMC_LLC_ADD_LINK;
 672         if (reqresp == SMC_LLC_RESP)
 673                 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
 674         memcpy(addllc->sender_mac, mac, ETH_ALEN);
 675         memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
 676         if (link_new) {
 677                 addllc->link_num = link_new->link_id;
 678                 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
 679                 hton24(addllc->initial_psn, link_new->psn_initial);
 680                 if (reqresp == SMC_LLC_REQ)
 681                         addllc->qp_mtu = link_new->path_mtu;
 682                 else
 683                         addllc->qp_mtu = min(link_new->path_mtu,
 684                                              link_new->peer_mtu);
 685         }
 686         if (ext && link_new)
 687                 len += smc_llc_fill_ext_v2(ext, link, link_new);
 688         smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len);
 689         /* send llc message */
 690         if (link->lgr->smc_version == SMC_V2)
 691                 rc = smc_wr_tx_v2_send(link, pend, len);
 692         else
 693                 rc = smc_wr_tx_send(link, pend);
 694 put_out:
 695         smc_wr_tx_link_put(link);
 696         return rc;
 697 }
 698
 699 /* send DELETE LINK request or response */
 700 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
 701                              enum smc_llc_reqresp reqresp, bool orderly,
 702                              u32 reason)
 703 {
 704         struct smc_llc_msg_del_link *delllc;
 705         struct smc_wr_tx_pend_priv *pend;
 706         struct smc_wr_buf *wr_buf;
 707         int rc;
 708
 709         if (!smc_wr_tx_link_hold(link))
 710                 return -ENOLINK;
 711         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 712         if (rc)
 713                 goto put_out;
 714         delllc = (struct smc_llc_msg_del_link *)wr_buf;
 715
 716         memset(delllc, 0, sizeof(*delllc));
 717         delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK;
 718         smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc));
 719         if (reqresp == SMC_LLC_RESP)
 720                 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
 721         if (orderly)
 722                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
 723         if (link_del_id)
 724                 delllc->link_num = link_del_id;
 725         else
 726                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
 727         delllc->reason = htonl(reason);
 728         /* send llc message */
 729         rc = smc_wr_tx_send(link, pend);
 730 put_out:
 731         smc_wr_tx_link_put(link);
 732         return rc;
 733 }
 734
 735 /* send LLC test link request */
 736 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
 737 {
 738         struct smc_llc_msg_test_link *testllc;
 739         struct smc_wr_tx_pend_priv *pend;
 740         struct smc_wr_buf *wr_buf;
 741         int rc;
 742
 743         if (!smc_wr_tx_link_hold(link))
 744                 return -ENOLINK;
 745         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 746         if (rc)
 747                 goto put_out;
 748         testllc = (struct smc_llc_msg_test_link *)wr_buf;
 749         memset(testllc, 0, sizeof(*testllc));
 750         testllc->hd.common.llc_type = SMC_LLC_TEST_LINK;
 751         smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc));
 752         memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
 753         /* send llc message */
 754         rc = smc_wr_tx_send(link, pend);
 755 put_out:
 756         smc_wr_tx_link_put(link);
 757         return rc;
 758 }
 759
 760 /* schedule an llc send on link, may wait for buffers */
 761 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
 762 {
 763         struct smc_wr_tx_pend_priv *pend;
 764         struct smc_wr_buf *wr_buf;
 765         int rc;
 766
 767         if (!smc_wr_tx_link_hold(link))
 768                 return -ENOLINK;
 769         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 770         if (rc)
 771                 goto put_out;
 772         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 773         rc = smc_wr_tx_send(link, pend);
 774 put_out:
 775         smc_wr_tx_link_put(link);
 776         return rc;
 777 }
 778
 779 /* schedule an llc send on link, may wait for buffers,
 780  * and wait for send completion notification.
 781  * @return 0 on success
 782  */
 783 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
 784 {
 785         struct smc_wr_tx_pend_priv *pend;
 786         struct smc_wr_buf *wr_buf;
 787         int rc;
 788
 789         if (!smc_wr_tx_link_hold(link))
 790                 return -ENOLINK;
 791         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 792         if (rc)
 793                 goto put_out;
 794         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 795         rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
 796 put_out:
 797         smc_wr_tx_link_put(link);
 798         return rc;
 799 }
 800
 801 /********************************* receive ***********************************/
 802
 803 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
 804                                   enum smc_lgr_type lgr_new_t)
 805 {
 806         int i;
 807
 808         if (lgr->type == SMC_LGR_SYMMETRIC ||
 809             (lgr->type != SMC_LGR_SINGLE &&
 810              (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 811               lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
 812                 return -EMLINK;
 813
 814         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 815             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
 816                 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
 817                         if (lgr->lnk[i].state == SMC_LNK_UNUSED)
 818                                 return i;
 819         } else {
 820                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
 821                         if (lgr->lnk[i].state == SMC_LNK_UNUSED)
 822                                 return i;
 823         }
 824         return -EMLINK;
 825 }
 826
 827 /* send one add_link_continue msg */
 828 static int smc_llc_add_link_cont(struct smc_link *link,
 829                                  struct smc_link *link_new, u8 *num_rkeys_todo,
 830                                  int *buf_lst, struct smc_buf_desc **buf_pos)
 831 {
 832         struct smc_llc_msg_add_link_cont *addc_llc;
 833         struct smc_link_group *lgr = link->lgr;
 834         int prim_lnk_idx, lnk_idx, i, rc;
 835         struct smc_wr_tx_pend_priv *pend;
 836         struct smc_wr_buf *wr_buf;
 837         struct smc_buf_desc *rmb;
 838         u8 n;
 839
 840         if (!smc_wr_tx_link_hold(link))
 841                 return -ENOLINK;
 842         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 843         if (rc)
 844                 goto put_out;
 845         addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
 846         memset(addc_llc, 0, sizeof(*addc_llc));
 847
 848         prim_lnk_idx = link->link_idx;
 849         lnk_idx = link_new->link_idx;
 850         addc_llc->link_num = link_new->link_id;
 851         addc_llc->num_rkeys = *num_rkeys_todo;
 852         n = *num_rkeys_todo;
 853         for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
 854                 while (*buf_pos && !(*buf_pos)->used)
 855                         *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
 856                 if (!*buf_pos) {
 857                         addc_llc->num_rkeys = addc_llc->num_rkeys -
 858                                               *num_rkeys_todo;
 859                         *num_rkeys_todo = 0;
 860                         break;
 861                 }
 862                 rmb = *buf_pos;
 863
 864                 addc_llc->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
 865                 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
 866                 addc_llc->rt[i].rmb_vaddr_new = rmb->is_vm ?
 867                         cpu_to_be64((uintptr_t)rmb->cpu_addr) :
 868                         cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
 869
 870                 (*num_rkeys_todo)--;
 871                 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
 872         }
 873         addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT;
 874         addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
 875         if (lgr->role == SMC_CLNT)
 876                 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
 877         rc = smc_wr_tx_send(link, pend);
 878 put_out:
 879         smc_wr_tx_link_put(link);
 880         return rc;
 881 }
 882
 883 static int smc_llc_cli_rkey_exchange(struct smc_link *link,
 884                                      struct smc_link *link_new)
 885 {
 886         struct smc_llc_msg_add_link_cont *addc_llc;
 887         struct smc_link_group *lgr = link->lgr;
 888         u8 max, num_rkeys_send, num_rkeys_recv;
 889         struct smc_llc_qentry *qentry;
 890         struct smc_buf_desc *buf_pos;
 891         int buf_lst;
 892         int rc = 0;
 893         int i;
 894
 895         down_write(&lgr->rmbs_lock);
 896         num_rkeys_send = lgr->conns_num;
 897         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
 898         do {
 899                 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
 900                                       SMC_LLC_ADD_LINK_CONT);
 901                 if (!qentry) {
 902                         rc = -ETIMEDOUT;
 903                         break;
 904                 }
 905                 addc_llc = &qentry->msg.add_link_cont;
 906                 num_rkeys_recv = addc_llc->num_rkeys;
 907                 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
 908                 for (i = 0; i < max; i++) {
 909                         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
 910                                        addc_llc->rt[i].rmb_key,
 911                                        addc_llc->rt[i].rmb_vaddr_new,
 912                                        addc_llc->rt[i].rmb_key_new);
 913                         num_rkeys_recv--;
 914                 }
 915                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 916                 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
 917                                            &buf_lst, &buf_pos);
 918                 if (rc)
 919                         break;
 920         } while (num_rkeys_send || num_rkeys_recv);
 921
 922         up_write(&lgr->rmbs_lock);
 923         return rc;
 924 }
 925
 926 /* prepare and send an add link reject response */
 927 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
 928 {
 929         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
 930         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
 931         qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
 932         smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
 933                              sizeof(qentry->msg));
 934         return smc_llc_send_message(qentry->link, &qentry->msg);
 935 }
 936
 937 static int smc_llc_cli_conf_link(struct smc_link *link,
 938                                  struct smc_init_info *ini,
 939                                  struct smc_link *link_new,
 940                                  enum smc_lgr_type lgr_new_t)
 941 {
 942         struct smc_link_group *lgr = link->lgr;
 943         struct smc_llc_qentry *qentry = NULL;
 944         int rc = 0;
 945
 946         /* receive CONFIRM LINK request over RoCE fabric */
 947         qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
 948         if (!qentry) {
 949                 rc = smc_llc_send_delete_link(link, link_new->link_id,
 950                                               SMC_LLC_REQ, false,
 951                                               SMC_LLC_DEL_LOST_PATH);
 952                 return -ENOLINK;
 953         }
 954         if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
 955                 /* received DELETE_LINK instead */
 956                 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
 957                 smc_llc_send_message(link, &qentry->msg);
 958                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 959                 return -ENOLINK;
 960         }
 961         smc_llc_save_peer_uid(qentry);
 962         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 963
 964         rc = smc_ib_modify_qp_rts(link_new);
 965         if (rc) {
 966                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 967                                          false, SMC_LLC_DEL_LOST_PATH);
 968                 return -ENOLINK;
 969         }
 970         smc_wr_remember_qp_attr(link_new);
 971
 972         rc = smcr_buf_reg_lgr(link_new);
 973         if (rc) {
 974                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 975                                          false, SMC_LLC_DEL_LOST_PATH);
 976                 return -ENOLINK;
 977         }
 978
 979         /* send CONFIRM LINK response over RoCE fabric */
 980         rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
 981         if (rc) {
 982                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 983                                          false, SMC_LLC_DEL_LOST_PATH);
 984                 return -ENOLINK;
 985         }
 986         smc_llc_link_active(link_new);
 987         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 988             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
 989                 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
 990         else
 991                 smcr_lgr_set_type(lgr, lgr_new_t);
 992         return 0;
 993 }
 994
 995 static void smc_llc_save_add_link_rkeys(struct smc_link *link,
 996                                         struct smc_link *link_new)
 997 {
 998         struct smc_llc_msg_add_link_v2_ext *ext;
 999         struct smc_link_group *lgr = link->lgr;
1000         int max, i;
1001
1002         ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 +
1003                                                      SMC_WR_TX_SIZE);
1004         max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1005         down_write(&lgr->rmbs_lock);
1006         for (i = 0; i < max; i++) {
1007                 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1008                                ext->rt[i].rmb_key,
1009                                ext->rt[i].rmb_vaddr_new,
1010                                ext->rt[i].rmb_key_new);
1011         }
1012         up_write(&lgr->rmbs_lock);
1013 }
1014
1015 static void smc_llc_save_add_link_info(struct smc_link *link,
1016                                        struct smc_llc_msg_add_link *add_llc)
1017 {
1018         link->peer_qpn = ntoh24(add_llc->sender_qp_num);
1019         memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
1020         memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
1021         link->peer_psn = ntoh24(add_llc->initial_psn);
1022         link->peer_mtu = add_llc->qp_mtu;
1023 }
1024
1025 /* as an SMC client, process an add link request */
1026 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
1027 {
1028         struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
1029         enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1030         struct smc_link_group *lgr = smc_get_lgr(link);
1031         struct smc_init_info *ini = NULL;
1032         struct smc_link *lnk_new = NULL;
1033         int lnk_idx, rc = 0;
1034
1035         if (!llc->qp_mtu)
1036                 goto out_reject;
1037
1038         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1039         if (!ini) {
1040                 rc = -ENOMEM;
1041                 goto out_reject;
1042         }
1043
1044         ini->vlan_id = lgr->vlan_id;
1045         if (lgr->smc_version == SMC_V2) {
1046                 ini->check_smcrv2 = true;
1047                 ini->smcrv2.saddr = lgr->saddr;
1048                 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid);
1049         }
1050         smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1051         if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1052             (lgr->smc_version == SMC_V2 ||
1053              !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1054                 if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2)
1055                         goto out_reject;
1056                 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1057         }
1058         if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1059                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1060                 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1061                 ini->smcrv2.ib_port_v2 = link->ibport;
1062         } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1063                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1064                 ini->ib_dev = link->smcibdev;
1065                 ini->ib_port = link->ibport;
1066         }
1067         lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1068         if (lnk_idx < 0)
1069                 goto out_reject;
1070         lnk_new = &lgr->lnk[lnk_idx];
1071         rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini);
1072         if (rc)
1073                 goto out_reject;
1074         smc_llc_save_add_link_info(lnk_new, llc);
1075         lnk_new->link_id = llc->link_num;       /* SMC server assigns link id */
1076         smc_llc_link_set_uid(lnk_new);
1077
1078         rc = smc_ib_ready_link(lnk_new);
1079         if (rc)
1080                 goto out_clear_lnk;
1081
1082         rc = smcr_buf_map_lgr(lnk_new);
1083         if (rc)
1084                 goto out_clear_lnk;
1085
1086         rc = smc_llc_send_add_link(link,
1087                                    lnk_new->smcibdev->mac[lnk_new->ibport - 1],
1088                                    lnk_new->gid, lnk_new, SMC_LLC_RESP);
1089         if (rc)
1090                 goto out_clear_lnk;
1091         if (lgr->smc_version == SMC_V2) {
1092                 smc_llc_save_add_link_rkeys(link, lnk_new);
1093         } else {
1094                 rc = smc_llc_cli_rkey_exchange(link, lnk_new);
1095                 if (rc) {
1096                         rc = 0;
1097                         goto out_clear_lnk;
1098                 }
1099         }
1100         rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t);
1101         if (!rc)
1102                 goto out;
1103 out_clear_lnk:
1104         lnk_new->state = SMC_LNK_INACTIVE;
1105         smcr_link_clear(lnk_new, false);
1106 out_reject:
1107         smc_llc_cli_add_link_reject(qentry);
1108 out:
1109         kfree(ini);
1110         kfree(qentry);
1111         return rc;
1112 }
1113
1114 static void smc_llc_send_request_add_link(struct smc_link *link)
1115 {
1116         struct smc_llc_msg_req_add_link_v2 *llc;
1117         struct smc_wr_tx_pend_priv *pend;
1118         struct smc_wr_v2_buf *wr_buf;
1119         struct smc_gidlist gidlist;
1120         int rc, len, i;
1121
1122         if (!smc_wr_tx_link_hold(link))
1123                 return;
1124         if (link->lgr->type == SMC_LGR_SYMMETRIC ||
1125             link->lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1126                 goto put_out;
1127
1128         smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid);
1129         if (gidlist.len <= 1)
1130                 goto put_out;
1131
1132         rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
1133         if (rc)
1134                 goto put_out;
1135         llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf;
1136         memset(llc, 0, SMC_WR_TX_SIZE);
1137
1138         llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK;
1139         for (i = 0; i < gidlist.len; i++)
1140                 memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0]));
1141         llc->gid_cnt = gidlist.len;
1142         len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0]));
1143         smc_llc_init_msg_hdr(&llc->hd, link->lgr, len);
1144         rc = smc_wr_tx_v2_send(link, pend, len);
1145         if (!rc)
1146                 /* set REQ_ADD_LINK flow and wait for response from peer */
1147                 link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK;
1148 put_out:
1149         smc_wr_tx_link_put(link);
1150 }
1151
1152 /* as an SMC client, invite server to start the add_link processing */
1153 static void smc_llc_cli_add_link_invite(struct smc_link *link,
1154                                         struct smc_llc_qentry *qentry)
1155 {
1156         struct smc_link_group *lgr = smc_get_lgr(link);
1157         struct smc_init_info *ini = NULL;
1158
1159         if (lgr->smc_version == SMC_V2) {
1160                 smc_llc_send_request_add_link(link);
1161                 goto out;
1162         }
1163
1164         if (lgr->type == SMC_LGR_SYMMETRIC ||
1165             lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1166                 goto out;
1167
1168         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1169         if (!ini)
1170                 goto out;
1171
1172         ini->vlan_id = lgr->vlan_id;
1173         smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1174         if (!ini->ib_dev)
1175                 goto out;
1176
1177         smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1],
1178                               ini->ib_gid, NULL, SMC_LLC_REQ);
1179 out:
1180         kfree(ini);
1181         kfree(qentry);
1182 }
1183
1184 static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc)
1185 {
1186         int i;
1187
1188         for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++)
1189                 if (llc->raw.data[i])
1190                         return false;
1191         return true;
1192 }
1193
1194 static bool smc_llc_is_local_add_link(union smc_llc_msg *llc)
1195 {
1196         if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK &&
1197             smc_llc_is_empty_llc_message(llc))
1198                 return true;
1199         return false;
1200 }
1201
1202 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
1203 {
1204         struct smc_llc_qentry *qentry;
1205
1206         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1207
1208         down_write(&lgr->llc_conf_mutex);
1209         if (smc_llc_is_local_add_link(&qentry->msg))
1210                 smc_llc_cli_add_link_invite(qentry->link, qentry);
1211         else
1212                 smc_llc_cli_add_link(qentry->link, qentry);
1213         up_write(&lgr->llc_conf_mutex);
1214 }
1215
1216 static int smc_llc_active_link_count(struct smc_link_group *lgr)
1217 {
1218         int i, link_count = 0;
1219
1220         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1221                 if (!smc_link_active(&lgr->lnk[i]))
1222                         continue;
1223                 link_count++;
1224         }
1225         return link_count;
1226 }
1227
1228 /* find the asymmetric link when 3 links are established  */
1229 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
1230 {
1231         int asym_idx = -ENOENT;
1232         int i, j, k;
1233         bool found;
1234
1235         /* determine asymmetric link */
1236         found = false;
1237         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1238                 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
1239                         if (!smc_link_usable(&lgr->lnk[i]) ||
1240                             !smc_link_usable(&lgr->lnk[j]))
1241                                 continue;
1242                         if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
1243                                     SMC_GID_SIZE)) {
1244                                 found = true;   /* asym_lnk is i or j */
1245                                 break;
1246                         }
1247                 }
1248                 if (found)
1249                         break;
1250         }
1251         if (!found)
1252                 goto out; /* no asymmetric link */
1253         for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
1254                 if (!smc_link_usable(&lgr->lnk[k]))
1255                         continue;
1256                 if (k != i &&
1257                     !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
1258                             SMC_GID_SIZE)) {
1259                         asym_idx = i;
1260                         break;
1261                 }
1262                 if (k != j &&
1263                     !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
1264                             SMC_GID_SIZE)) {
1265                         asym_idx = j;
1266                         break;
1267                 }
1268         }
1269 out:
1270         return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
1271 }
1272
1273 static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
1274 {
1275         struct smc_link *lnk_new = NULL, *lnk_asym;
1276         struct smc_llc_qentry *qentry;
1277         int rc;
1278
1279         lnk_asym = smc_llc_find_asym_link(lgr);
1280         if (!lnk_asym)
1281                 return; /* no asymmetric link */
1282         if (!smc_link_downing(&lnk_asym->state))
1283                 return;
1284         lnk_new = smc_switch_conns(lgr, lnk_asym, false);
1285         smc_wr_tx_wait_no_pending_sends(lnk_asym);
1286         if (!lnk_new)
1287                 goto out_free;
1288         /* change flow type from ADD_LINK into DEL_LINK */
1289         lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
1290         rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
1291                                       true, SMC_LLC_DEL_NO_ASYM_NEEDED);
1292         if (rc) {
1293                 smcr_link_down_cond(lnk_new);
1294                 goto out_free;
1295         }
1296         qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
1297                               SMC_LLC_DELETE_LINK);
1298         if (!qentry) {
1299                 smcr_link_down_cond(lnk_new);
1300                 goto out_free;
1301         }
1302         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1303 out_free:
1304         smcr_link_clear(lnk_asym, true);
1305 }
1306
1307 static int smc_llc_srv_rkey_exchange(struct smc_link *link,
1308                                      struct smc_link *link_new)
1309 {
1310         struct smc_llc_msg_add_link_cont *addc_llc;
1311         struct smc_link_group *lgr = link->lgr;
1312         u8 max, num_rkeys_send, num_rkeys_recv;
1313         struct smc_llc_qentry *qentry = NULL;
1314         struct smc_buf_desc *buf_pos;
1315         int buf_lst;
1316         int rc = 0;
1317         int i;
1318
1319         down_write(&lgr->rmbs_lock);
1320         num_rkeys_send = lgr->conns_num;
1321         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
1322         do {
1323                 smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
1324                                       &buf_lst, &buf_pos);
1325                 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1326                                       SMC_LLC_ADD_LINK_CONT);
1327                 if (!qentry) {
1328                         rc = -ETIMEDOUT;
1329                         goto out;
1330                 }
1331                 addc_llc = &qentry->msg.add_link_cont;
1332                 num_rkeys_recv = addc_llc->num_rkeys;
1333                 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1334                 for (i = 0; i < max; i++) {
1335                         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1336                                        addc_llc->rt[i].rmb_key,
1337                                        addc_llc->rt[i].rmb_vaddr_new,
1338                                        addc_llc->rt[i].rmb_key_new);
1339                         num_rkeys_recv--;
1340                 }
1341                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1342         } while (num_rkeys_send || num_rkeys_recv);
1343 out:
1344         up_write(&lgr->rmbs_lock);
1345         return rc;
1346 }
1347
1348 static int smc_llc_srv_conf_link(struct smc_link *link,
1349                                  struct smc_link *link_new,
1350                                  enum smc_lgr_type lgr_new_t)
1351 {
1352         struct smc_link_group *lgr = link->lgr;
1353         struct smc_llc_qentry *qentry = NULL;
1354         int rc;
1355
1356         /* send CONFIRM LINK request over the RoCE fabric */
1357         rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1358         if (rc)
1359                 return -ENOLINK;
1360         /* receive CONFIRM LINK response over the RoCE fabric */
1361         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0);
1362         if (!qentry ||
1363             qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
1364                 /* send DELETE LINK */
1365                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1366                                          false, SMC_LLC_DEL_LOST_PATH);
1367                 if (qentry)
1368                         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1369                 return -ENOLINK;
1370         }
1371         smc_llc_save_peer_uid(qentry);
1372         smc_llc_link_active(link_new);
1373         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1374             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1375                 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1376         else
1377                 smcr_lgr_set_type(lgr, lgr_new_t);
1378         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1379         return 0;
1380 }
1381
1382 static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry)
1383 {
1384         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
1385         smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
1386                              sizeof(qentry->msg));
1387         memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data));
1388         smc_llc_send_message(qentry->link, &qentry->msg);
1389 }
1390
1391 int smc_llc_srv_add_link(struct smc_link *link,
1392                          struct smc_llc_qentry *req_qentry)
1393 {
1394         enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1395         struct smc_link_group *lgr = link->lgr;
1396         struct smc_llc_msg_add_link *add_llc;
1397         struct smc_llc_qentry *qentry = NULL;
1398         bool send_req_add_link_resp = false;
1399         struct smc_link *link_new = NULL;
1400         struct smc_init_info *ini = NULL;
1401         int lnk_idx, rc = 0;
1402
1403         if (req_qentry &&
1404             req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK)
1405                 send_req_add_link_resp = true;
1406
1407         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1408         if (!ini) {
1409                 rc = -ENOMEM;
1410                 goto out;
1411         }
1412
1413         /* ignore client add link recommendation, start new flow */
1414         ini->vlan_id = lgr->vlan_id;
1415         if (lgr->smc_version == SMC_V2) {
1416                 ini->check_smcrv2 = true;
1417                 ini->smcrv2.saddr = lgr->saddr;
1418                 if (send_req_add_link_resp) {
1419                         struct smc_llc_msg_req_add_link_v2 *req_add =
1420                                 &req_qentry->msg.req_add_link;
1421
1422                         ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]);
1423                 }
1424         }
1425         smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1426         if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1427                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1428                 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1429                 ini->smcrv2.ib_port_v2 = link->ibport;
1430         } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1431                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1432                 ini->ib_dev = link->smcibdev;
1433                 ini->ib_port = link->ibport;
1434         }
1435         lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1436         if (lnk_idx < 0) {
1437                 rc = 0;
1438                 goto out;
1439         }
1440
1441         rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini);
1442         if (rc)
1443                 goto out;
1444         link_new = &lgr->lnk[lnk_idx];
1445
1446         rc = smcr_buf_map_lgr(link_new);
1447         if (rc)
1448                 goto out_err;
1449
1450         rc = smc_llc_send_add_link(link,
1451                                    link_new->smcibdev->mac[link_new->ibport-1],
1452                                    link_new->gid, link_new, SMC_LLC_REQ);
1453         if (rc)
1454                 goto out_err;
1455         send_req_add_link_resp = false;
1456         /* receive ADD LINK response over the RoCE fabric */
1457         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1458         if (!qentry) {
1459                 rc = -ETIMEDOUT;
1460                 goto out_err;
1461         }
1462         add_llc = &qentry->msg.add_link;
1463         if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1464                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1465                 rc = -ENOLINK;
1466                 goto out_err;
1467         }
1468         if (lgr->type == SMC_LGR_SINGLE &&
1469             (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1470              (lgr->smc_version == SMC_V2 ||
1471               !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) {
1472                 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1473         }
1474         smc_llc_save_add_link_info(link_new, add_llc);
1475         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1476
1477         rc = smc_ib_ready_link(link_new);
1478         if (rc)
1479                 goto out_err;
1480         rc = smcr_buf_reg_lgr(link_new);
1481         if (rc)
1482                 goto out_err;
1483         if (lgr->smc_version == SMC_V2) {
1484                 smc_llc_save_add_link_rkeys(link, link_new);
1485         } else {
1486                 rc = smc_llc_srv_rkey_exchange(link, link_new);
1487                 if (rc)
1488                         goto out_err;
1489         }
1490         rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
1491         if (rc)
1492                 goto out_err;
1493         kfree(ini);
1494         return 0;
1495 out_err:
1496         if (link_new) {
1497                 link_new->state = SMC_LNK_INACTIVE;
1498                 smcr_link_clear(link_new, false);
1499         }
1500 out:
1501         kfree(ini);
1502         if (send_req_add_link_resp)
1503                 smc_llc_send_req_add_link_response(req_qentry);
1504         return rc;
1505 }
1506
1507 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1508 {
1509         struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1510         struct smc_llc_qentry *qentry;
1511         int rc;
1512
1513         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1514
1515         down_write(&lgr->llc_conf_mutex);
1516         rc = smc_llc_srv_add_link(link, qentry);
1517         if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1518                 /* delete any asymmetric link */
1519                 smc_llc_delete_asym_link(lgr);
1520         }
1521         up_write(&lgr->llc_conf_mutex);
1522         kfree(qentry);
1523 }
1524
1525 /* enqueue a local add_link req to trigger a new add_link flow */
1526 void smc_llc_add_link_local(struct smc_link *link)
1527 {
1528         struct smc_llc_msg_add_link add_llc = {};
1529
1530         add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK;
1531         smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc));
1532         /* no dev and port needed */
1533         smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc);
1534 }
1535
1536 /* worker to process an add link message */
1537 static void smc_llc_add_link_work(struct work_struct *work)
1538 {
1539         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1540                                                   llc_add_link_work);
1541
1542         if (list_empty(&lgr->list)) {
1543                 /* link group is terminating */
1544                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1545                 goto out;
1546         }
1547
1548         if (lgr->role == SMC_CLNT)
1549                 smc_llc_process_cli_add_link(lgr);
1550         else
1551                 smc_llc_process_srv_add_link(lgr);
1552 out:
1553         if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK)
1554                 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1555 }
1556
1557 /* enqueue a local del_link msg to trigger a new del_link flow,
1558  * called only for role SMC_SERV
1559  */
1560 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1561 {
1562         struct smc_llc_msg_del_link del_llc = {};
1563
1564         del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1565         smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc));
1566         del_llc.link_num = del_link_id;
1567         del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1568         del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1569         smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc);
1570 }
1571
1572 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1573 {
1574         struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1575         struct smc_llc_msg_del_link *del_llc;
1576         struct smc_llc_qentry *qentry;
1577         int active_links;
1578         int lnk_idx;
1579
1580         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1581         lnk = qentry->link;
1582         del_llc = &qentry->msg.delete_link;
1583
1584         if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1585                 smc_lgr_terminate_sched(lgr);
1586                 goto out;
1587         }
1588         down_write(&lgr->llc_conf_mutex);
1589         /* delete single link */
1590         for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1591                 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1592                         continue;
1593                 lnk_del = &lgr->lnk[lnk_idx];
1594                 break;
1595         }
1596         del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1597         if (!lnk_del) {
1598                 /* link was not found */
1599                 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1600                 smc_llc_send_message(lnk, &qentry->msg);
1601                 goto out_unlock;
1602         }
1603         lnk_asym = smc_llc_find_asym_link(lgr);
1604
1605         del_llc->reason = 0;
1606         smc_llc_send_message(lnk, &qentry->msg); /* response */
1607
1608         if (smc_link_downing(&lnk_del->state))
1609                 smc_switch_conns(lgr, lnk_del, false);
1610         smcr_link_clear(lnk_del, true);
1611
1612         active_links = smc_llc_active_link_count(lgr);
1613         if (lnk_del == lnk_asym) {
1614                 /* expected deletion of asym link, don't change lgr state */
1615         } else if (active_links == 1) {
1616                 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1617         } else if (!active_links) {
1618                 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1619                 smc_lgr_terminate_sched(lgr);
1620         }
1621 out_unlock:
1622         up_write(&lgr->llc_conf_mutex);
1623 out:
1624         kfree(qentry);
1625 }
1626
1627 /* try to send a DELETE LINK ALL request on any active link,
1628  * waiting for send completion
1629  */
1630 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1631 {
1632         struct smc_llc_msg_del_link delllc = {};
1633         int i;
1634
1635         delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1636         smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc));
1637         if (ord)
1638                 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1639         delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1640         delllc.reason = htonl(rsn);
1641
1642         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1643                 if (!smc_link_sendable(&lgr->lnk[i]))
1644                         continue;
1645                 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1646                         break;
1647         }
1648 }
1649
1650 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1651 {
1652         struct smc_llc_msg_del_link *del_llc;
1653         struct smc_link *lnk, *lnk_del;
1654         struct smc_llc_qentry *qentry;
1655         int active_links;
1656         int i;
1657
1658         down_write(&lgr->llc_conf_mutex);
1659         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1660         lnk = qentry->link;
1661         del_llc = &qentry->msg.delete_link;
1662
1663         if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1664                 /* delete entire lgr */
1665                 smc_llc_send_link_delete_all(lgr, true, ntohl(
1666                                               qentry->msg.delete_link.reason));
1667                 smc_lgr_terminate_sched(lgr);
1668                 goto out;
1669         }
1670         /* delete single link */
1671         lnk_del = NULL;
1672         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1673                 if (lgr->lnk[i].link_id == del_llc->link_num) {
1674                         lnk_del = &lgr->lnk[i];
1675                         break;
1676                 }
1677         }
1678         if (!lnk_del)
1679                 goto out; /* asymmetric link already deleted */
1680
1681         if (smc_link_downing(&lnk_del->state)) {
1682                 if (smc_switch_conns(lgr, lnk_del, false))
1683                         smc_wr_tx_wait_no_pending_sends(lnk_del);
1684         }
1685         if (!list_empty(&lgr->list)) {
1686                 /* qentry is either a request from peer (send it back to
1687                  * initiate the DELETE_LINK processing), or a locally
1688                  * enqueued DELETE_LINK request (forward it)
1689                  */
1690                 if (!smc_llc_send_message(lnk, &qentry->msg)) {
1691                         struct smc_llc_qentry *qentry2;
1692
1693                         qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1694                                                SMC_LLC_DELETE_LINK);
1695                         if (qentry2)
1696                                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1697                 }
1698         }
1699         smcr_link_clear(lnk_del, true);
1700
1701         active_links = smc_llc_active_link_count(lgr);
1702         if (active_links == 1) {
1703                 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1704         } else if (!active_links) {
1705                 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1706                 smc_lgr_terminate_sched(lgr);
1707         }
1708
1709         if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1710                 /* trigger setup of asymm alt link */
1711                 smc_llc_add_link_local(lnk);
1712         }
1713 out:
1714         up_write(&lgr->llc_conf_mutex);
1715         kfree(qentry);
1716 }
1717
1718 static void smc_llc_delete_link_work(struct work_struct *work)
1719 {
1720         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1721                                                   llc_del_link_work);
1722
1723         if (list_empty(&lgr->list)) {
1724                 /* link group is terminating */
1725                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1726                 goto out;
1727         }
1728
1729         if (lgr->role == SMC_CLNT)
1730                 smc_llc_process_cli_delete_link(lgr);
1731         else
1732                 smc_llc_process_srv_delete_link(lgr);
1733 out:
1734         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1735 }
1736
1737 /* process a confirm_rkey request from peer, remote flow */
1738 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
1739 {
1740         struct smc_llc_msg_confirm_rkey *llc;
1741         struct smc_llc_qentry *qentry;
1742         struct smc_link *link;
1743         int num_entries;
1744         int rk_idx;
1745         int i;
1746
1747         qentry = lgr->llc_flow_rmt.qentry;
1748         llc = &qentry->msg.confirm_rkey;
1749         link = qentry->link;
1750
1751         num_entries = llc->rtoken[0].num_rkeys;
1752         if (num_entries > SMC_LLC_RKEYS_PER_MSG)
1753                 goto out_err;
1754         /* first rkey entry is for receiving link */
1755         rk_idx = smc_rtoken_add(link,
1756                                 llc->rtoken[0].rmb_vaddr,
1757                                 llc->rtoken[0].rmb_key);
1758         if (rk_idx < 0)
1759                 goto out_err;
1760
1761         for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1762                 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1763                                 llc->rtoken[i].rmb_vaddr,
1764                                 llc->rtoken[i].rmb_key);
1765         /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1766         goto out;
1767 out_err:
1768         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1769         llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1770 out:
1771         llc->hd.flags |= SMC_LLC_FLAG_RESP;
1772         smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1773         smc_llc_send_message(link, &qentry->msg);
1774         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1775 }
1776
1777 /* process a delete_rkey request from peer, remote flow */
1778 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
1779 {
1780         struct smc_llc_msg_delete_rkey *llc;
1781         struct smc_llc_qentry *qentry;
1782         struct smc_link *link;
1783         u8 err_mask = 0;
1784         int i, max;
1785
1786         qentry = lgr->llc_flow_rmt.qentry;
1787         llc = &qentry->msg.delete_rkey;
1788         link = qentry->link;
1789
1790         if (lgr->smc_version == SMC_V2) {
1791                 struct smc_llc_msg_delete_rkey_v2 *llcv2;
1792
1793                 memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc));
1794                 llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2;
1795                 llcv2->num_inval_rkeys = 0;
1796
1797                 max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1798                 for (i = 0; i < max; i++) {
1799                         if (smc_rtoken_delete(link, llcv2->rkey[i]))
1800                                 llcv2->num_inval_rkeys++;
1801                 }
1802                 memset(&llc->rkey[0], 0, sizeof(llc->rkey));
1803                 memset(&llc->reserved2, 0, sizeof(llc->reserved2));
1804                 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1805                 if (llcv2->num_inval_rkeys) {
1806                         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1807                         llc->err_mask = llcv2->num_inval_rkeys;
1808                 }
1809                 goto finish;
1810         }
1811
1812         max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1813         for (i = 0; i < max; i++) {
1814                 if (smc_rtoken_delete(link, llc->rkey[i]))
1815                         err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1816         }
1817         if (err_mask) {
1818                 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1819                 llc->err_mask = err_mask;
1820         }
1821 finish:
1822         llc->hd.flags |= SMC_LLC_FLAG_RESP;
1823         smc_llc_send_message(link, &qentry->msg);
1824         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1825 }
1826
1827 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1828 {
1829         pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: "
1830                             "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id,
1831                             lgr->net->net_cookie, type);
1832         smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1833         smc_lgr_terminate_sched(lgr);
1834 }
1835
1836 /* flush the llc event queue */
1837 static void smc_llc_event_flush(struct smc_link_group *lgr)
1838 {
1839         struct smc_llc_qentry *qentry, *q;
1840
1841         spin_lock_bh(&lgr->llc_event_q_lock);
1842         list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1843                 list_del_init(&qentry->list);
1844                 kfree(qentry);
1845         }
1846         spin_unlock_bh(&lgr->llc_event_q_lock);
1847 }
1848
1849 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1850 {
1851         union smc_llc_msg *llc = &qentry->msg;
1852         struct smc_link *link = qentry->link;
1853         struct smc_link_group *lgr = link->lgr;
1854
1855         if (!smc_link_usable(link))
1856                 goto out;
1857
1858         switch (llc->raw.hdr.common.llc_type) {
1859         case SMC_LLC_TEST_LINK:
1860                 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1861                 smc_llc_send_message(link, llc);
1862                 break;
1863         case SMC_LLC_ADD_LINK:
1864                 if (list_empty(&lgr->list))
1865                         goto out;       /* lgr is terminating */
1866                 if (lgr->role == SMC_CLNT) {
1867                         if (smc_llc_is_local_add_link(llc)) {
1868                                 if (lgr->llc_flow_lcl.type ==
1869                                     SMC_LLC_FLOW_ADD_LINK)
1870                                         break;  /* add_link in progress */
1871                                 if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1872                                                        qentry)) {
1873                                         schedule_work(&lgr->llc_add_link_work);
1874                                 }
1875                                 return;
1876                         }
1877                         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1878                             !lgr->llc_flow_lcl.qentry) {
1879                                 /* a flow is waiting for this message */
1880                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1881                                                         qentry);
1882                                 wake_up(&lgr->llc_msg_waiter);
1883                                 return;
1884                         }
1885                         if (lgr->llc_flow_lcl.type ==
1886                                         SMC_LLC_FLOW_REQ_ADD_LINK) {
1887                                 /* server started add_link processing */
1888                                 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1889                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1890                                                         qentry);
1891                                 schedule_work(&lgr->llc_add_link_work);
1892                                 return;
1893                         }
1894                         if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1895                                 schedule_work(&lgr->llc_add_link_work);
1896                         }
1897                 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1898                         /* as smc server, handle client suggestion */
1899                         schedule_work(&lgr->llc_add_link_work);
1900                 }
1901                 return;
1902         case SMC_LLC_CONFIRM_LINK:
1903         case SMC_LLC_ADD_LINK_CONT:
1904                 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1905                         /* a flow is waiting for this message */
1906                         smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1907                         wake_up(&lgr->llc_msg_waiter);
1908                         return;
1909                 }
1910                 break;
1911         case SMC_LLC_DELETE_LINK:
1912                 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1913                     !lgr->llc_flow_lcl.qentry) {
1914                         /* DEL LINK REQ during ADD LINK SEQ */
1915                         smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1916                         wake_up(&lgr->llc_msg_waiter);
1917                 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1918                         schedule_work(&lgr->llc_del_link_work);
1919                 }
1920                 return;
1921         case SMC_LLC_CONFIRM_RKEY:
1922                 /* new request from remote, assign to remote flow */
1923                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1924                         /* process here, does not wait for more llc msgs */
1925                         smc_llc_rmt_conf_rkey(lgr);
1926                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1927                 }
1928                 return;
1929         case SMC_LLC_CONFIRM_RKEY_CONT:
1930                 /* not used because max links is 3, and 3 rkeys fit into
1931                  * one CONFIRM_RKEY message
1932                  */
1933                 break;
1934         case SMC_LLC_DELETE_RKEY:
1935                 /* new request from remote, assign to remote flow */
1936                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1937                         /* process here, does not wait for more llc msgs */
1938                         smc_llc_rmt_delete_rkey(lgr);
1939                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1940                 }
1941                 return;
1942         case SMC_LLC_REQ_ADD_LINK:
1943                 /* handle response here, smc_llc_flow_stop() cannot be called
1944                  * in tasklet context
1945                  */
1946                 if (lgr->role == SMC_CLNT &&
1947                     lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK &&
1948                     (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) {
1949                         smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl);
1950                 } else if (lgr->role == SMC_SERV) {
1951                         if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1952                                 /* as smc server, handle client suggestion */
1953                                 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1954                                 schedule_work(&lgr->llc_add_link_work);
1955                         }
1956                         return;
1957                 }
1958                 break;
1959         default:
1960                 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
1961                 break;
1962         }
1963 out:
1964         kfree(qentry);
1965 }
1966
1967 /* worker to process llc messages on the event queue */
1968 static void smc_llc_event_work(struct work_struct *work)
1969 {
1970         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1971                                                   llc_event_work);
1972         struct smc_llc_qentry *qentry;
1973
1974         if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
1975                 qentry = lgr->delayed_event;
1976                 lgr->delayed_event = NULL;
1977                 if (smc_link_usable(qentry->link))
1978                         smc_llc_event_handler(qentry);
1979                 else
1980                         kfree(qentry);
1981         }
1982
1983 again:
1984         spin_lock_bh(&lgr->llc_event_q_lock);
1985         if (!list_empty(&lgr->llc_event_q)) {
1986                 qentry = list_first_entry(&lgr->llc_event_q,
1987                                           struct smc_llc_qentry, list);
1988                 list_del_init(&qentry->list);
1989                 spin_unlock_bh(&lgr->llc_event_q_lock);
1990                 smc_llc_event_handler(qentry);
1991                 goto again;
1992         }
1993         spin_unlock_bh(&lgr->llc_event_q_lock);
1994 }
1995
1996 /* process llc responses in tasklet context */
1997 static void smc_llc_rx_response(struct smc_link *link,
1998                                 struct smc_llc_qentry *qentry)
1999 {
2000         enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;
2001         struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;
2002         u8 llc_type = qentry->msg.raw.hdr.common.llc_type;
2003
2004         switch (llc_type) {
2005         case SMC_LLC_TEST_LINK:
2006                 if (smc_link_active(link))
2007                         complete(&link->llc_testlink_resp);
2008                 break;
2009         case SMC_LLC_ADD_LINK:
2010         case SMC_LLC_ADD_LINK_CONT:
2011         case SMC_LLC_CONFIRM_LINK:
2012                 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry)
2013                         break;  /* drop out-of-flow response */
2014                 goto assign;
2015         case SMC_LLC_DELETE_LINK:
2016                 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry)
2017                         break;  /* drop out-of-flow response */
2018                 goto assign;
2019         case SMC_LLC_CONFIRM_RKEY:
2020         case SMC_LLC_DELETE_RKEY:
2021                 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry)
2022                         break;  /* drop out-of-flow response */
2023                 goto assign;
2024         case SMC_LLC_CONFIRM_RKEY_CONT:
2025                 /* not used because max links is 3 */
2026                 break;
2027         default:
2028                 smc_llc_protocol_violation(link->lgr,
2029                                            qentry->msg.raw.hdr.common.type);
2030                 break;
2031         }
2032         kfree(qentry);
2033         return;
2034 assign:
2035         /* assign responses to the local flow, we requested them */
2036         smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
2037         wake_up(&link->lgr->llc_msg_waiter);
2038 }
2039
2040 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
2041 {
2042         struct smc_link_group *lgr = link->lgr;
2043         struct smc_llc_qentry *qentry;
2044         unsigned long flags;
2045
2046         qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
2047         if (!qentry)
2048                 return;
2049         qentry->link = link;
2050         INIT_LIST_HEAD(&qentry->list);
2051         memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
2052
2053         /* process responses immediately */
2054         if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) &&
2055             llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) {
2056                 smc_llc_rx_response(link, qentry);
2057                 return;
2058         }
2059
2060         /* add requests to event queue */
2061         spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
2062         list_add_tail(&qentry->list, &lgr->llc_event_q);
2063         spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
2064         queue_work(system_highpri_wq, &lgr->llc_event_work);
2065 }
2066
2067 /* copy received msg and add it to the event queue */
2068 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
2069 {
2070         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
2071         union smc_llc_msg *llc = buf;
2072
2073         if (wc->byte_len < sizeof(*llc))
2074                 return; /* short message */
2075         if (!llc->raw.hdr.common.llc_version) {
2076                 if (llc->raw.hdr.length != sizeof(*llc))
2077                         return; /* invalid message */
2078         } else {
2079                 if (llc->raw.hdr.length_v2 < sizeof(*llc))
2080                         return; /* invalid message */
2081         }
2082
2083         smc_llc_enqueue(link, llc);
2084 }
2085
2086 /***************************** worker, utils *********************************/
2087
2088 static void smc_llc_testlink_work(struct work_struct *work)
2089 {
2090         struct smc_link *link = container_of(to_delayed_work(work),
2091                                              struct smc_link, llc_testlink_wrk);
2092         unsigned long next_interval;
2093         unsigned long expire_time;
2094         u8 user_data[16] = { 0 };
2095         int rc;
2096
2097         if (!smc_link_active(link))
2098                 return;         /* don't reschedule worker */
2099         expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
2100         if (time_is_after_jiffies(expire_time)) {
2101                 next_interval = expire_time - jiffies;
2102                 goto out;
2103         }
2104         reinit_completion(&link->llc_testlink_resp);
2105         smc_llc_send_test_link(link, user_data);
2106         /* receive TEST LINK response over RoCE fabric */
2107         rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
2108                                                        SMC_LLC_WAIT_TIME);
2109         if (!smc_link_active(link))
2110                 return;         /* link state changed */
2111         if (rc <= 0) {
2112                 smcr_link_down_cond_sched(link);
2113                 return;
2114         }
2115         next_interval = link->llc_testlink_time;
2116 out:
2117         schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
2118 }
2119
2120 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
2121 {
2122         struct net *net = sock_net(smc->clcsock->sk);
2123
2124         INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
2125         INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
2126         INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
2127         INIT_LIST_HEAD(&lgr->llc_event_q);
2128         spin_lock_init(&lgr->llc_event_q_lock);
2129         spin_lock_init(&lgr->llc_flow_lock);
2130         init_waitqueue_head(&lgr->llc_flow_waiter);
2131         init_waitqueue_head(&lgr->llc_msg_waiter);
2132         init_rwsem(&lgr->llc_conf_mutex);
2133         lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time);
2134 }
2135
2136 /* called after lgr was removed from lgr_list */
2137 void smc_llc_lgr_clear(struct smc_link_group *lgr)
2138 {
2139         smc_llc_event_flush(lgr);
2140         wake_up_all(&lgr->llc_flow_waiter);
2141         wake_up_all(&lgr->llc_msg_waiter);
2142         cancel_work_sync(&lgr->llc_event_work);
2143         cancel_work_sync(&lgr->llc_add_link_work);
2144         cancel_work_sync(&lgr->llc_del_link_work);
2145         if (lgr->delayed_event) {
2146                 kfree(lgr->delayed_event);
2147                 lgr->delayed_event = NULL;
2148         }
2149 }
2150
2151 int smc_llc_link_init(struct smc_link *link)
2152 {
2153         init_completion(&link->llc_testlink_resp);
2154         INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2155         return 0;
2156 }
2157
2158 void smc_llc_link_active(struct smc_link *link)
2159 {
2160         pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, "
2161                             "peerid %*phN, ibdev %s, ibport %d\n",
2162                             SMC_LGR_ID_SIZE, &link->lgr->id,
2163                             link->lgr->net->net_cookie,
2164                             SMC_LGR_ID_SIZE, &link->link_uid,
2165                             SMC_LGR_ID_SIZE, &link->peer_link_uid,
2166                             link->smcibdev->ibdev->name, link->ibport);
2167         link->state = SMC_LNK_ACTIVE;
2168         if (link->lgr->llc_testlink_time) {
2169                 link->llc_testlink_time = link->lgr->llc_testlink_time;
2170                 schedule_delayed_work(&link->llc_testlink_wrk,
2171                                       link->llc_testlink_time);
2172         }
2173 }
2174
2175 /* called in worker context */
2176 void smc_llc_link_clear(struct smc_link *link, bool log)
2177 {
2178         if (log)
2179                 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN"
2180                                     ", peerid %*phN, ibdev %s, ibport %d\n",
2181                                     SMC_LGR_ID_SIZE, &link->lgr->id,
2182                                     link->lgr->net->net_cookie,
2183                                     SMC_LGR_ID_SIZE, &link->link_uid,
2184                                     SMC_LGR_ID_SIZE, &link->peer_link_uid,
2185                                     link->smcibdev->ibdev->name, link->ibport);
2186         complete(&link->llc_testlink_resp);
2187         cancel_delayed_work_sync(&link->llc_testlink_wrk);
2188 }
2189
2190 /* register a new rtoken at the remote peer (for all links) */
2191 int smc_llc_do_confirm_rkey(struct smc_link *send_link,
2192                             struct smc_buf_desc *rmb_desc)
2193 {
2194         struct smc_link_group *lgr = send_link->lgr;
2195         struct smc_llc_qentry *qentry = NULL;
2196         int rc = 0;
2197
2198         rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
2199         if (rc)
2200                 goto out;
2201         /* receive CONFIRM RKEY response from server over RoCE fabric */
2202         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2203                               SMC_LLC_CONFIRM_RKEY);
2204         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2205                 rc = -EFAULT;
2206 out:
2207         if (qentry)
2208                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2209         return rc;
2210 }
2211
2212 /* unregister an rtoken at the remote peer */
2213 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
2214                            struct smc_buf_desc *rmb_desc)
2215 {
2216         struct smc_llc_qentry *qentry = NULL;
2217         struct smc_link *send_link;
2218         int rc = 0;
2219
2220         send_link = smc_llc_usable_link(lgr);
2221         if (!send_link)
2222                 return -ENOLINK;
2223
2224         /* protected by llc_flow control */
2225         rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
2226         if (rc)
2227                 goto out;
2228         /* receive DELETE RKEY response from server over RoCE fabric */
2229         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2230                               SMC_LLC_DELETE_RKEY);
2231         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2232                 rc = -EFAULT;
2233 out:
2234         if (qentry)
2235                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2236         return rc;
2237 }
2238
2239 void smc_llc_link_set_uid(struct smc_link *link)
2240 {
2241         __be32 link_uid;
2242
2243         link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
2244         memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
2245 }
2246
2247 /* save peers link user id, used for debug purposes */
2248 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
2249 {
2250         memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
2251                SMC_LGR_ID_SIZE);
2252 }
2253
2254 /* evaluate confirm link request or response */
2255 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
2256                            enum smc_llc_reqresp type)
2257 {
2258         if (type == SMC_LLC_REQ) {      /* SMC server assigns link_id */
2259                 qentry->link->link_id = qentry->msg.confirm_link.link_num;
2260                 smc_llc_link_set_uid(qentry->link);
2261         }
2262         if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
2263                 return -ENOTSUPP;
2264         return 0;
2265 }
2266
2267 /***************************** init, exit, misc ******************************/
2268
2269 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
2270         {
2271                 .handler        = smc_llc_rx_handler,
2272                 .type           = SMC_LLC_CONFIRM_LINK
2273         },
2274         {
2275                 .handler        = smc_llc_rx_handler,
2276                 .type           = SMC_LLC_TEST_LINK
2277         },
2278         {
2279                 .handler        = smc_llc_rx_handler,
2280                 .type           = SMC_LLC_ADD_LINK
2281         },
2282         {
2283                 .handler        = smc_llc_rx_handler,
2284                 .type           = SMC_LLC_ADD_LINK_CONT
2285         },
2286         {
2287                 .handler        = smc_llc_rx_handler,
2288                 .type           = SMC_LLC_DELETE_LINK
2289         },
2290         {
2291                 .handler        = smc_llc_rx_handler,
2292                 .type           = SMC_LLC_CONFIRM_RKEY
2293         },
2294         {
2295                 .handler        = smc_llc_rx_handler,
2296                 .type           = SMC_LLC_CONFIRM_RKEY_CONT
2297         },
2298         {
2299                 .handler        = smc_llc_rx_handler,
2300                 .type           = SMC_LLC_DELETE_RKEY
2301         },
2302         /* V2 types */
2303         {
2304                 .handler        = smc_llc_rx_handler,
2305                 .type           = SMC_LLC_CONFIRM_LINK_V2
2306         },
2307         {
2308                 .handler        = smc_llc_rx_handler,
2309                 .type           = SMC_LLC_TEST_LINK_V2
2310         },
2311         {
2312                 .handler        = smc_llc_rx_handler,
2313                 .type           = SMC_LLC_ADD_LINK_V2
2314         },
2315         {
2316                 .handler        = smc_llc_rx_handler,
2317                 .type           = SMC_LLC_DELETE_LINK_V2
2318         },
2319         {
2320                 .handler        = smc_llc_rx_handler,
2321                 .type           = SMC_LLC_REQ_ADD_LINK_V2
2322         },
2323         {
2324                 .handler        = smc_llc_rx_handler,
2325                 .type           = SMC_LLC_CONFIRM_RKEY_V2
2326         },
2327         {
2328                 .handler        = smc_llc_rx_handler,
2329                 .type           = SMC_LLC_DELETE_RKEY_V2
2330         },
2331         {
2332                 .handler        = NULL,
2333         }
2334 };
2335
2336 int __init smc_llc_init(void)
2337 {
2338         struct smc_wr_rx_handler *handler;
2339         int rc = 0;
2340
2341         for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
2342                 INIT_HLIST_NODE(&handler->list);
2343                 rc = smc_wr_rx_register_handler(handler);
2344                 if (rc)
2345                         break;
2346         }
2347         return rc;
2348 }