net/smc/af_smc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   6  *  applies to SOCK_STREAM sockets only
   7  *  offers an alternative communication option for TCP-protocol sockets
   8  *  applicable with RoCE-cards only
   9  *
  10  *  Initial restrictions:
  11  *    - support for alternate links postponed
  12  *
  13  *  Copyright IBM Corp. 2016, 2018
  14  *
  15  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  16  *              based on prototype from Frank Blaschka
  17  */
  18
  19 #define KMSG_COMPONENT "smc"
  20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  21
  22 #include <linux/module.h>
  23 #include <linux/socket.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/in.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/if_vlan.h>
  28 #include <linux/rcupdate_wait.h>
  29 #include <linux/ctype.h>
  30
  31 #include <net/sock.h>
  32 #include <net/tcp.h>
  33 #include <net/smc.h>
  34 #include <asm/ioctls.h>
  35
  36 #include <net/net_namespace.h>
  37 #include <net/netns/generic.h>
  38 #include "smc_netns.h"
  39
  40 #include "smc.h"
  41 #include "smc_clc.h"
  42 #include "smc_llc.h"
  43 #include "smc_cdc.h"
  44 #include "smc_core.h"
  45 #include "smc_ib.h"
  46 #include "smc_ism.h"
  47 #include "smc_pnet.h"
  48 #include "smc_tx.h"
  49 #include "smc_rx.h"
  50 #include "smc_close.h"
  51
  52 static DEFINE_MUTEX(smc_server_lgr_pending);    /* serialize link group
  53                                                  * creation on server
  54                                                  */
  55 static DEFINE_MUTEX(smc_client_lgr_pending);    /* serialize link group
  56                                                  * creation on client
  57                                                  */
  58
  59 struct workqueue_struct *smc_hs_wq;     /* wq for handshake work */
  60 struct workqueue_struct *smc_close_wq;  /* wq for close work */
  61
  62 static void smc_tcp_listen_work(struct work_struct *);
  63 static void smc_connect_work(struct work_struct *);
  64
  65 static void smc_set_keepalive(struct sock *sk, int val)
  66 {
  67         struct smc_sock *smc = smc_sk(sk);
  68
  69         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  70 }
  71
  72 static struct smc_hashinfo smc_v4_hashinfo = {
  73         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  74 };
  75
  76 static struct smc_hashinfo smc_v6_hashinfo = {
  77         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  78 };
  79
  80 int smc_hash_sk(struct sock *sk)
  81 {
  82         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  83         struct hlist_head *head;
  84
  85         head = &h->ht;
  86
  87         write_lock_bh(&h->lock);
  88         sk_add_node(sk, head);
  89         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  90         write_unlock_bh(&h->lock);
  91
  92         return 0;
  93 }
  94 EXPORT_SYMBOL_GPL(smc_hash_sk);
  95
  96 void smc_unhash_sk(struct sock *sk)
  97 {
  98         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  99
 100         write_lock_bh(&h->lock);
 101         if (sk_del_node_init(sk))
 102                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 103         write_unlock_bh(&h->lock);
 104 }
 105 EXPORT_SYMBOL_GPL(smc_unhash_sk);
 106
 107 struct proto smc_proto = {
 108         .name           = "SMC",
 109         .owner          = THIS_MODULE,
 110         .keepalive      = smc_set_keepalive,
 111         .hash           = smc_hash_sk,
 112         .unhash         = smc_unhash_sk,
 113         .obj_size       = sizeof(struct smc_sock),
 114         .h.smc_hash     = &smc_v4_hashinfo,
 115         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 116 };
 117 EXPORT_SYMBOL_GPL(smc_proto);
 118
 119 struct proto smc_proto6 = {
 120         .name           = "SMC6",
 121         .owner          = THIS_MODULE,
 122         .keepalive      = smc_set_keepalive,
 123         .hash           = smc_hash_sk,
 124         .unhash         = smc_unhash_sk,
 125         .obj_size       = sizeof(struct smc_sock),
 126         .h.smc_hash     = &smc_v6_hashinfo,
 127         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 128 };
 129 EXPORT_SYMBOL_GPL(smc_proto6);
 130
 131 static void smc_restore_fallback_changes(struct smc_sock *smc)
 132 {
 133         if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
 134                 smc->clcsock->file->private_data = smc->sk.sk_socket;
 135                 smc->clcsock->file = NULL;
 136         }
 137 }
 138
 139 static int __smc_release(struct smc_sock *smc)
 140 {
 141         struct sock *sk = &smc->sk;
 142         int rc = 0;
 143
 144         if (!smc->use_fallback) {
 145                 rc = smc_close_active(smc);
 146                 smc_sock_set_flag(sk, SOCK_DEAD);
 147                 sk->sk_shutdown |= SHUTDOWN_MASK;
 148         } else {
 149                 if (sk->sk_state != SMC_CLOSED) {
 150                         if (sk->sk_state != SMC_LISTEN &&
 151                             sk->sk_state != SMC_INIT)
 152                                 sock_put(sk); /* passive closing */
 153                         if (sk->sk_state == SMC_LISTEN) {
 154                                 /* wake up clcsock accept */
 155                                 rc = kernel_sock_shutdown(smc->clcsock,
 156                                                           SHUT_RDWR);
 157                         }
 158                         sk->sk_state = SMC_CLOSED;
 159                         sk->sk_state_change(sk);
 160                 }
 161                 smc_restore_fallback_changes(smc);
 162         }
 163
 164         sk->sk_prot->unhash(sk);
 165
 166         if (sk->sk_state == SMC_CLOSED) {
 167                 if (smc->clcsock) {
 168                         release_sock(sk);
 169                         smc_clcsock_release(smc);
 170                         lock_sock(sk);
 171                 }
 172                 if (!smc->use_fallback)
 173                         smc_conn_free(&smc->conn);
 174         }
 175
 176         return rc;
 177 }
 178
 179 static int smc_release(struct socket *sock)
 180 {
 181         struct sock *sk = sock->sk;
 182         struct smc_sock *smc;
 183         int old_state, rc = 0;
 184
 185         if (!sk)
 186                 goto out;
 187
 188         sock_hold(sk); /* sock_put below */
 189         smc = smc_sk(sk);
 190
 191         old_state = sk->sk_state;
 192
 193         /* cleanup for a dangling non-blocking connect */
 194         if (smc->connect_nonblock && old_state == SMC_INIT)
 195                 tcp_abort(smc->clcsock->sk, ECONNABORTED);
 196
 197         if (cancel_work_sync(&smc->connect_work))
 198                 sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */
 199
 200         if (sk->sk_state == SMC_LISTEN)
 201                 /* smc_close_non_accepted() is called and acquires
 202                  * sock lock for child sockets again
 203                  */
 204                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 205         else
 206                 lock_sock(sk);
 207
 208         if (old_state == SMC_INIT && sk->sk_state == SMC_ACTIVE &&
 209             !smc->use_fallback)
 210                 smc_close_active_abort(smc);
 211
 212         rc = __smc_release(smc);
 213
 214         /* detach socket */
 215         sock_orphan(sk);
 216         sock->sk = NULL;
 217         release_sock(sk);
 218
 219         sock_put(sk); /* sock_hold above */
 220         sock_put(sk); /* final sock_put */
 221 out:
 222         return rc;
 223 }
 224
 225 static void smc_destruct(struct sock *sk)
 226 {
 227         if (sk->sk_state != SMC_CLOSED)
 228                 return;
 229         if (!sock_flag(sk, SOCK_DEAD))
 230                 return;
 231
 232         sk_refcnt_debug_dec(sk);
 233 }
 234
 235 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 236                                    int protocol)
 237 {
 238         struct smc_sock *smc;
 239         struct proto *prot;
 240         struct sock *sk;
 241
 242         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 243         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 244         if (!sk)
 245                 return NULL;
 246
 247         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 248         sk->sk_state = SMC_INIT;
 249         sk->sk_destruct = smc_destruct;
 250         sk->sk_protocol = protocol;
 251         smc = smc_sk(sk);
 252         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 253         INIT_WORK(&smc->connect_work, smc_connect_work);
 254         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 255         INIT_LIST_HEAD(&smc->accept_q);
 256         spin_lock_init(&smc->accept_q_lock);
 257         spin_lock_init(&smc->conn.send_lock);
 258         sk->sk_prot->hash(sk);
 259         sk_refcnt_debug_inc(sk);
 260         mutex_init(&smc->clcsock_release_lock);
 261
 262         return sk;
 263 }
 264
 265 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 266                     int addr_len)
 267 {
 268         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 269         struct sock *sk = sock->sk;
 270         struct smc_sock *smc;
 271         int rc;
 272
 273         smc = smc_sk(sk);
 274
 275         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 276         rc = -EINVAL;
 277         if (addr_len < sizeof(struct sockaddr_in))
 278                 goto out;
 279
 280         rc = -EAFNOSUPPORT;
 281         if (addr->sin_family != AF_INET &&
 282             addr->sin_family != AF_INET6 &&
 283             addr->sin_family != AF_UNSPEC)
 284                 goto out;
 285         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 286         if (addr->sin_family == AF_UNSPEC &&
 287             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 288                 goto out;
 289
 290         lock_sock(sk);
 291
 292         /* Check if socket is already active */
 293         rc = -EINVAL;
 294         if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
 295                 goto out_rel;
 296
 297         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 298         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 299
 300 out_rel:
 301         release_sock(sk);
 302 out:
 303         return rc;
 304 }
 305
 306 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 307                                    unsigned long mask)
 308 {
 309         /* options we don't get control via setsockopt for */
 310         nsk->sk_type = osk->sk_type;
 311         nsk->sk_sndbuf = osk->sk_sndbuf;
 312         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 313         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 314         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 315         nsk->sk_mark = osk->sk_mark;
 316         nsk->sk_priority = osk->sk_priority;
 317         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 318         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 319         nsk->sk_err = osk->sk_err;
 320
 321         nsk->sk_flags &= ~mask;
 322         nsk->sk_flags |= osk->sk_flags & mask;
 323 }
 324
 325 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 326                              (1UL << SOCK_KEEPOPEN) | \
 327                              (1UL << SOCK_LINGER) | \
 328                              (1UL << SOCK_BROADCAST) | \
 329                              (1UL << SOCK_TIMESTAMP) | \
 330                              (1UL << SOCK_DBG) | \
 331                              (1UL << SOCK_RCVTSTAMP) | \
 332                              (1UL << SOCK_RCVTSTAMPNS) | \
 333                              (1UL << SOCK_LOCALROUTE) | \
 334                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 335                              (1UL << SOCK_RXQ_OVFL) | \
 336                              (1UL << SOCK_WIFI_STATUS) | \
 337                              (1UL << SOCK_NOFCS) | \
 338                              (1UL << SOCK_FILTER_LOCKED) | \
 339                              (1UL << SOCK_TSTAMP_NEW))
 340 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 341  * clc socket (since smc is not called for these options from net/core)
 342  */
 343 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 344 {
 345         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 346 }
 347
 348 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 349                              (1UL << SOCK_KEEPOPEN) | \
 350                              (1UL << SOCK_LINGER) | \
 351                              (1UL << SOCK_DBG))
 352 /* copy only settings and flags relevant for smc from clc to smc socket */
 353 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 354 {
 355         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 356 }
 357
 358 /* register the new rmb on all links */
 359 static int smcr_lgr_reg_rmbs(struct smc_link *link,
 360                              struct smc_buf_desc *rmb_desc)
 361 {
 362         struct smc_link_group *lgr = link->lgr;
 363         int i, rc = 0;
 364
 365         rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
 366         if (rc)
 367                 return rc;
 368         /* protect against parallel smc_llc_cli_rkey_exchange() and
 369          * parallel smcr_link_reg_rmb()
 370          */
 371         mutex_lock(&lgr->llc_conf_mutex);
 372         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 373                 if (!smc_link_active(&lgr->lnk[i]))
 374                         continue;
 375                 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
 376                 if (rc)
 377                         goto out;
 378         }
 379
 380         /* exchange confirm_rkey msg with peer */
 381         rc = smc_llc_do_confirm_rkey(link, rmb_desc);
 382         if (rc) {
 383                 rc = -EFAULT;
 384                 goto out;
 385         }
 386         rmb_desc->is_conf_rkey = true;
 387 out:
 388         mutex_unlock(&lgr->llc_conf_mutex);
 389         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
 390         return rc;
 391 }
 392
 393 static int smcr_clnt_conf_first_link(struct smc_sock *smc)
 394 {
 395         struct smc_link *link = smc->conn.lnk;
 396         struct smc_llc_qentry *qentry;
 397         int rc;
 398
 399         /* Receive CONFIRM LINK request from server over RoCE fabric.
 400          * Increasing the client's timeout by twice as much as the server's
 401          * timeout by default can temporarily avoid decline messages of
 402          * both sides crossing or colliding
 403          */
 404         qentry = smc_llc_wait(link->lgr, NULL, 2 * SMC_LLC_WAIT_TIME,
 405                               SMC_LLC_CONFIRM_LINK);
 406         if (!qentry) {
 407                 struct smc_clc_msg_decline dclc;
 408
 409                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 410                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 411                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
 412         }
 413         smc_llc_save_peer_uid(qentry);
 414         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
 415         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
 416         if (rc)
 417                 return SMC_CLC_DECL_RMBE_EC;
 418
 419         rc = smc_ib_modify_qp_rts(link);
 420         if (rc)
 421                 return SMC_CLC_DECL_ERR_RDYLNK;
 422
 423         smc_wr_remember_qp_attr(link);
 424
 425         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
 426                 return SMC_CLC_DECL_ERR_REGRMB;
 427
 428         /* confirm_rkey is implicit on 1st contact */
 429         smc->conn.rmb_desc->is_conf_rkey = true;
 430
 431         /* send CONFIRM LINK response over RoCE fabric */
 432         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 433         if (rc < 0)
 434                 return SMC_CLC_DECL_TIMEOUT_CL;
 435
 436         smc_llc_link_active(link);
 437         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
 438
 439         /* optional 2nd link, receive ADD LINK request from server */
 440         qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
 441                               SMC_LLC_ADD_LINK);
 442         if (!qentry) {
 443                 struct smc_clc_msg_decline dclc;
 444
 445                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 446                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
 447                 if (rc == -EAGAIN)
 448                         rc = 0; /* no DECLINE received, go with one link */
 449                 return rc;
 450         }
 451         smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
 452         smc_llc_cli_add_link(link, qentry);
 453         return 0;
 454 }
 455
 456 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 457                                      struct smc_clc_msg_accept_confirm *clc)
 458 {
 459         int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
 460
 461         smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
 462         smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
 463         smc->conn.peer_rmbe_size = bufsize;
 464         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 465         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 466 }
 467
 468 static bool smc_isascii(char *hostname)
 469 {
 470         int i;
 471
 472         for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
 473                 if (!isascii(hostname[i]))
 474                         return false;
 475         return true;
 476 }
 477
 478 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 479                                      struct smc_clc_msg_accept_confirm *clc)
 480 {
 481         int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
 482
 483         smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
 484         smc->conn.peer_token = clc->d0.token;
 485         /* msg header takes up space in the buffer */
 486         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 487         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 488         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 489         if (clc->hdr.version > SMC_V1 &&
 490             (clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) {
 491                 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
 492                         (struct smc_clc_msg_accept_confirm_v2 *)clc;
 493                 struct smc_clc_first_contact_ext *fce =
 494                         (struct smc_clc_first_contact_ext *)
 495                                 (((u8 *)clc_v2) + sizeof(*clc_v2));
 496
 497                 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->eid,
 498                        SMC_MAX_EID_LEN);
 499                 smc->conn.lgr->peer_os = fce->os_type;
 500                 smc->conn.lgr->peer_smc_release = fce->release;
 501                 if (smc_isascii(fce->hostname))
 502                         memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
 503                                SMC_MAX_HOSTNAME_LEN);
 504         }
 505 }
 506
 507 static void smc_conn_save_peer_info(struct smc_sock *smc,
 508                                     struct smc_clc_msg_accept_confirm *clc)
 509 {
 510         if (smc->conn.lgr->is_smcd)
 511                 smcd_conn_save_peer_info(smc, clc);
 512         else
 513                 smcr_conn_save_peer_info(smc, clc);
 514 }
 515
 516 static void smc_link_save_peer_info(struct smc_link *link,
 517                                     struct smc_clc_msg_accept_confirm *clc)
 518 {
 519         link->peer_qpn = ntoh24(clc->r0.qpn);
 520         memcpy(link->peer_gid, clc->r0.lcl.gid, SMC_GID_SIZE);
 521         memcpy(link->peer_mac, clc->r0.lcl.mac, sizeof(link->peer_mac));
 522         link->peer_psn = ntoh24(clc->r0.psn);
 523         link->peer_mtu = clc->r0.qp_mtu;
 524 }
 525
 526 static void smc_switch_to_fallback(struct smc_sock *smc)
 527 {
 528         wait_queue_head_t *smc_wait = sk_sleep(&smc->sk);
 529         wait_queue_head_t *clc_wait = sk_sleep(smc->clcsock->sk);
 530         unsigned long flags;
 531
 532         smc->use_fallback = true;
 533         if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
 534                 smc->clcsock->file = smc->sk.sk_socket->file;
 535                 smc->clcsock->file->private_data = smc->clcsock;
 536                 smc->clcsock->wq.fasync_list =
 537                         smc->sk.sk_socket->wq.fasync_list;
 538
 539                 /* There may be some entries remaining in
 540                  * smc socket->wq, which should be removed
 541                  * to clcsocket->wq during the fallback.
 542                  */
 543                 spin_lock_irqsave(&smc_wait->lock, flags);
 544                 spin_lock_nested(&clc_wait->lock, SINGLE_DEPTH_NESTING);
 545                 list_splice_init(&smc_wait->head, &clc_wait->head);
 546                 spin_unlock(&clc_wait->lock);
 547                 spin_unlock_irqrestore(&smc_wait->lock, flags);
 548         }
 549 }
 550
 551 /* fall back during connect */
 552 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 553 {
 554         smc_switch_to_fallback(smc);
 555         smc->fallback_rsn = reason_code;
 556         smc_copy_sock_settings_to_clc(smc);
 557         smc->connect_nonblock = 0;
 558         if (smc->sk.sk_state == SMC_INIT)
 559                 smc->sk.sk_state = SMC_ACTIVE;
 560         return 0;
 561 }
 562
 563 /* decline and fall back during connect */
 564 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
 565                                         u8 version)
 566 {
 567         int rc;
 568
 569         if (reason_code < 0) { /* error, fallback is not possible */
 570                 if (smc->sk.sk_state == SMC_INIT)
 571                         sock_put(&smc->sk); /* passive closing */
 572                 return reason_code;
 573         }
 574         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 575                 rc = smc_clc_send_decline(smc, reason_code, version);
 576                 if (rc < 0) {
 577                         if (smc->sk.sk_state == SMC_INIT)
 578                                 sock_put(&smc->sk); /* passive closing */
 579                         return rc;
 580                 }
 581         }
 582         return smc_connect_fallback(smc, reason_code);
 583 }
 584
 585 /* abort connecting */
 586 static void smc_connect_abort(struct smc_sock *smc, int local_first)
 587 {
 588         if (local_first)
 589                 smc_lgr_cleanup_early(&smc->conn);
 590         else
 591                 smc_conn_free(&smc->conn);
 592 }
 593
 594 /* check if there is a rdma device available for this connection. */
 595 /* called for connect and listen */
 596 static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
 597 {
 598         /* PNET table look up: search active ib_device and port
 599          * within same PNETID that also contains the ethernet device
 600          * used for the internal TCP socket
 601          */
 602         smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
 603         if (!ini->ib_dev)
 604                 return SMC_CLC_DECL_NOSMCRDEV;
 605         return 0;
 606 }
 607
 608 /* check if there is an ISM device available for this connection. */
 609 /* called for connect and listen */
 610 static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
 611 {
 612         /* Find ISM device with same PNETID as connecting interface  */
 613         smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
 614         if (!ini->ism_dev[0])
 615                 return SMC_CLC_DECL_NOSMCDDEV;
 616         else
 617                 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
 618         return 0;
 619 }
 620
 621 /* is chid unique for the ism devices that are already determined? */
 622 static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
 623                                            int cnt)
 624 {
 625         int i = (!ini->ism_dev[0]) ? 1 : 0;
 626
 627         for (; i < cnt; i++)
 628                 if (ini->ism_chid[i] == chid)
 629                         return false;
 630         return true;
 631 }
 632
 633 /* determine possible V2 ISM devices (either without PNETID or with PNETID plus
 634  * PNETID matching net_device)
 635  */
 636 static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
 637                                        struct smc_init_info *ini)
 638 {
 639         int rc = SMC_CLC_DECL_NOSMCDDEV;
 640         struct smcd_dev *smcd;
 641         int i = 1;
 642         u16 chid;
 643
 644         if (smcd_indicated(ini->smc_type_v1))
 645                 rc = 0;         /* already initialized for V1 */
 646         mutex_lock(&smcd_dev_list.mutex);
 647         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
 648                 if (smcd->going_away || smcd == ini->ism_dev[0])
 649                         continue;
 650                 chid = smc_ism_get_chid(smcd);
 651                 if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
 652                         continue;
 653                 if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
 654                     smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
 655                         ini->ism_dev[i] = smcd;
 656                         ini->ism_chid[i] = chid;
 657                         ini->is_smcd = true;
 658                         rc = 0;
 659                         i++;
 660                         if (i > SMC_MAX_ISM_DEVS)
 661                                 break;
 662                 }
 663         }
 664         mutex_unlock(&smcd_dev_list.mutex);
 665         ini->ism_offered_cnt = i - 1;
 666         if (!ini->ism_dev[0] && !ini->ism_dev[1])
 667                 ini->smcd_version = 0;
 668
 669         return rc;
 670 }
 671
 672 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 673 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 674                                       struct smc_init_info *ini)
 675 {
 676         if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
 677                 return SMC_CLC_DECL_ISMVLANERR;
 678         return 0;
 679 }
 680
 681 static int smc_find_proposal_devices(struct smc_sock *smc,
 682                                      struct smc_init_info *ini)
 683 {
 684         int rc = 0;
 685
 686         /* check if there is an ism device available */
 687         if (ini->smcd_version & SMC_V1) {
 688                 if (smc_find_ism_device(smc, ini) ||
 689                     smc_connect_ism_vlan_setup(smc, ini)) {
 690                         if (ini->smc_type_v1 == SMC_TYPE_B)
 691                                 ini->smc_type_v1 = SMC_TYPE_R;
 692                         else
 693                                 ini->smc_type_v1 = SMC_TYPE_N;
 694                 } /* else ISM V1 is supported for this connection */
 695                 if (smc_find_rdma_device(smc, ini)) {
 696                         if (ini->smc_type_v1 == SMC_TYPE_B)
 697                                 ini->smc_type_v1 = SMC_TYPE_D;
 698                         else
 699                                 ini->smc_type_v1 = SMC_TYPE_N;
 700                 } /* else RDMA is supported for this connection */
 701         }
 702         if (smc_ism_v2_capable && smc_find_ism_v2_device_clnt(smc, ini))
 703                 ini->smc_type_v2 = SMC_TYPE_N;
 704
 705         /* if neither ISM nor RDMA are supported, fallback */
 706         if (!smcr_indicated(ini->smc_type_v1) &&
 707             ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
 708                 rc = SMC_CLC_DECL_NOSMCDEV;
 709
 710         return rc;
 711 }
 712
 713 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 714  * used, the VLAN ID will be registered again during the connection setup.
 715  */
 716 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
 717                                         struct smc_init_info *ini)
 718 {
 719         if (!smcd_indicated(ini->smc_type_v1))
 720                 return 0;
 721         if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
 722                 return SMC_CLC_DECL_CNFERR;
 723         return 0;
 724 }
 725
 726 #define SMC_CLC_MAX_ACCEPT_LEN \
 727         (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
 728          sizeof(struct smc_clc_first_contact_ext) + \
 729          sizeof(struct smc_clc_msg_trail))
 730
 731 /* CLC handshake during connect */
 732 static int smc_connect_clc(struct smc_sock *smc,
 733                            struct smc_clc_msg_accept_confirm_v2 *aclc2,
 734                            struct smc_init_info *ini)
 735 {
 736         int rc = 0;
 737
 738         /* do inband token exchange */
 739         rc = smc_clc_send_proposal(smc, ini);
 740         if (rc)
 741                 return rc;
 742         /* receive SMC Accept CLC message */
 743         return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
 744                                 SMC_CLC_ACCEPT, CLC_WAIT_TIME);
 745 }
 746
 747 /* setup for RDMA connection of client */
 748 static int smc_connect_rdma(struct smc_sock *smc,
 749                             struct smc_clc_msg_accept_confirm *aclc,
 750                             struct smc_init_info *ini)
 751 {
 752         int i, reason_code = 0;
 753         struct smc_link *link;
 754
 755         ini->is_smcd = false;
 756         ini->ib_lcl = &aclc->r0.lcl;
 757         ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
 758         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
 759
 760         mutex_lock(&smc_client_lgr_pending);
 761         reason_code = smc_conn_create(smc, ini);
 762         if (reason_code) {
 763                 mutex_unlock(&smc_client_lgr_pending);
 764                 return reason_code;
 765         }
 766
 767         smc_conn_save_peer_info(smc, aclc);
 768
 769         if (ini->first_contact_local) {
 770                 link = smc->conn.lnk;
 771         } else {
 772                 /* set link that was assigned by server */
 773                 link = NULL;
 774                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 775                         struct smc_link *l = &smc->conn.lgr->lnk[i];
 776
 777                         if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
 778                             !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
 779                                     SMC_GID_SIZE) &&
 780                             !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
 781                                     sizeof(l->peer_mac))) {
 782                                 link = l;
 783                                 break;
 784                         }
 785                 }
 786                 if (!link) {
 787                         reason_code = SMC_CLC_DECL_NOSRVLINK;
 788                         goto connect_abort;
 789                 }
 790                 smc->conn.lnk = link;
 791         }
 792
 793         /* create send buffer and rmb */
 794         if (smc_buf_create(smc, false)) {
 795                 reason_code = SMC_CLC_DECL_MEM;
 796                 goto connect_abort;
 797         }
 798
 799         if (ini->first_contact_local)
 800                 smc_link_save_peer_info(link, aclc);
 801
 802         if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
 803                 reason_code = SMC_CLC_DECL_ERR_RTOK;
 804                 goto connect_abort;
 805         }
 806
 807         smc_close_init(smc);
 808         smc_rx_init(smc);
 809
 810         if (ini->first_contact_local) {
 811                 if (smc_ib_ready_link(link)) {
 812                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
 813                         goto connect_abort;
 814                 }
 815         } else {
 816                 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
 817                         reason_code = SMC_CLC_DECL_ERR_REGRMB;
 818                         goto connect_abort;
 819                 }
 820         }
 821         smc_rmb_sync_sg_for_device(&smc->conn);
 822
 823         reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
 824                                            SMC_V1);
 825         if (reason_code)
 826                 goto connect_abort;
 827
 828         smc_tx_init(smc);
 829
 830         if (ini->first_contact_local) {
 831                 /* QP confirmation over RoCE fabric */
 832                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
 833                 reason_code = smcr_clnt_conf_first_link(smc);
 834                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
 835                 if (reason_code)
 836                         goto connect_abort;
 837         }
 838         mutex_unlock(&smc_client_lgr_pending);
 839
 840         smc_copy_sock_settings_to_clc(smc);
 841         smc->connect_nonblock = 0;
 842         if (smc->sk.sk_state == SMC_INIT)
 843                 smc->sk.sk_state = SMC_ACTIVE;
 844
 845         return 0;
 846 connect_abort:
 847         smc_connect_abort(smc, ini->first_contact_local);
 848         mutex_unlock(&smc_client_lgr_pending);
 849         smc->connect_nonblock = 0;
 850
 851         return reason_code;
 852 }
 853
 854 /* The server has chosen one of the proposed ISM devices for the communication.
 855  * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
 856  */
 857 static int
 858 smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
 859                                struct smc_init_info *ini)
 860 {
 861         int i;
 862
 863         for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
 864                 if (ini->ism_chid[i] == ntohs(aclc->chid)) {
 865                         ini->ism_selected = i;
 866                         return 0;
 867                 }
 868         }
 869
 870         return -EPROTO;
 871 }
 872
 873 /* setup for ISM connection of client */
 874 static int smc_connect_ism(struct smc_sock *smc,
 875                            struct smc_clc_msg_accept_confirm *aclc,
 876                            struct smc_init_info *ini)
 877 {
 878         int rc = 0;
 879
 880         ini->is_smcd = true;
 881         ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
 882
 883         if (aclc->hdr.version == SMC_V2) {
 884                 struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
 885                         (struct smc_clc_msg_accept_confirm_v2 *)aclc;
 886
 887                 rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
 888                 if (rc)
 889                         return rc;
 890         }
 891         ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
 892
 893         /* there is only one lgr role for SMC-D; use server lock */
 894         mutex_lock(&smc_server_lgr_pending);
 895         rc = smc_conn_create(smc, ini);
 896         if (rc) {
 897                 mutex_unlock(&smc_server_lgr_pending);
 898                 return rc;
 899         }
 900
 901         /* Create send and receive buffers */
 902         rc = smc_buf_create(smc, true);
 903         if (rc) {
 904                 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
 905                 goto connect_abort;
 906         }
 907
 908         smc_conn_save_peer_info(smc, aclc);
 909         smc_close_init(smc);
 910         smc_rx_init(smc);
 911         smc_tx_init(smc);
 912
 913         rc = smc_clc_send_confirm(smc, ini->first_contact_local,
 914                                   aclc->hdr.version);
 915         if (rc)
 916                 goto connect_abort;
 917         mutex_unlock(&smc_server_lgr_pending);
 918
 919         smc_copy_sock_settings_to_clc(smc);
 920         smc->connect_nonblock = 0;
 921         if (smc->sk.sk_state == SMC_INIT)
 922                 smc->sk.sk_state = SMC_ACTIVE;
 923
 924         return 0;
 925 connect_abort:
 926         smc_connect_abort(smc, ini->first_contact_local);
 927         mutex_unlock(&smc_server_lgr_pending);
 928         smc->connect_nonblock = 0;
 929
 930         return rc;
 931 }
 932
 933 /* check if received accept type and version matches a proposed one */
 934 static int smc_connect_check_aclc(struct smc_init_info *ini,
 935                                   struct smc_clc_msg_accept_confirm *aclc)
 936 {
 937         if ((aclc->hdr.typev1 == SMC_TYPE_R &&
 938              !smcr_indicated(ini->smc_type_v1)) ||
 939             (aclc->hdr.typev1 == SMC_TYPE_D &&
 940              ((!smcd_indicated(ini->smc_type_v1) &&
 941                !smcd_indicated(ini->smc_type_v2)) ||
 942               (aclc->hdr.version == SMC_V1 &&
 943                !smcd_indicated(ini->smc_type_v1)) ||
 944               (aclc->hdr.version == SMC_V2 &&
 945                !smcd_indicated(ini->smc_type_v2)))))
 946                 return SMC_CLC_DECL_MODEUNSUPP;
 947
 948         return 0;
 949 }
 950
 951 /* perform steps before actually connecting */
 952 static int __smc_connect(struct smc_sock *smc)
 953 {
 954         u8 version = smc_ism_v2_capable ? SMC_V2 : SMC_V1;
 955         struct smc_clc_msg_accept_confirm_v2 *aclc2;
 956         struct smc_clc_msg_accept_confirm *aclc;
 957         struct smc_init_info *ini = NULL;
 958         u8 *buf = NULL;
 959         int rc = 0;
 960
 961         if (smc->use_fallback)
 962                 return smc_connect_fallback(smc, smc->fallback_rsn);
 963
 964         /* if peer has not signalled SMC-capability, fall back */
 965         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 966                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 967
 968         /* IPSec connections opt out of SMC optimizations */
 969         if (using_ipsec(smc))
 970                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
 971                                                     version);
 972
 973         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
 974         if (!ini)
 975                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
 976                                                     version);
 977
 978         ini->smcd_version = SMC_V1;
 979         ini->smcd_version |= smc_ism_v2_capable ? SMC_V2 : 0;
 980         ini->smc_type_v1 = SMC_TYPE_B;
 981         ini->smc_type_v2 = smc_ism_v2_capable ? SMC_TYPE_D : SMC_TYPE_N;
 982
 983         /* get vlan id from IP device */
 984         if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
 985                 ini->smcd_version &= ~SMC_V1;
 986                 ini->smc_type_v1 = SMC_TYPE_N;
 987                 if (!ini->smcd_version) {
 988                         rc = SMC_CLC_DECL_GETVLANERR;
 989                         goto fallback;
 990                 }
 991         }
 992
 993         rc = smc_find_proposal_devices(smc, ini);
 994         if (rc)
 995                 goto fallback;
 996
 997         buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
 998         if (!buf) {
 999                 rc = SMC_CLC_DECL_MEM;
1000                 goto fallback;
1001         }
1002         aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1003         aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1004
1005         /* perform CLC handshake */
1006         rc = smc_connect_clc(smc, aclc2, ini);
1007         if (rc)
1008                 goto vlan_cleanup;
1009
1010         /* check if smc modes and versions of CLC proposal and accept match */
1011         rc = smc_connect_check_aclc(ini, aclc);
1012         version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1013         ini->smcd_version = version;
1014         if (rc)
1015                 goto vlan_cleanup;
1016
1017         /* depending on previous steps, connect using rdma or ism */
1018         if (aclc->hdr.typev1 == SMC_TYPE_R)
1019                 rc = smc_connect_rdma(smc, aclc, ini);
1020         else if (aclc->hdr.typev1 == SMC_TYPE_D)
1021                 rc = smc_connect_ism(smc, aclc, ini);
1022         if (rc)
1023                 goto vlan_cleanup;
1024
1025         smc_connect_ism_vlan_cleanup(smc, ini);
1026         kfree(buf);
1027         kfree(ini);
1028         return 0;
1029
1030 vlan_cleanup:
1031         smc_connect_ism_vlan_cleanup(smc, ini);
1032         kfree(buf);
1033 fallback:
1034         kfree(ini);
1035         return smc_connect_decline_fallback(smc, rc, version);
1036 }
1037
1038 static void smc_connect_work(struct work_struct *work)
1039 {
1040         struct smc_sock *smc = container_of(work, struct smc_sock,
1041                                             connect_work);
1042         long timeo = smc->sk.sk_sndtimeo;
1043         int rc = 0;
1044
1045         if (!timeo)
1046                 timeo = MAX_SCHEDULE_TIMEOUT;
1047         lock_sock(smc->clcsock->sk);
1048         if (smc->clcsock->sk->sk_err) {
1049                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
1050         } else if ((1 << smc->clcsock->sk->sk_state) &
1051                                         (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1052                 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1053                 if ((rc == -EPIPE) &&
1054                     ((1 << smc->clcsock->sk->sk_state) &
1055                                         (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1056                         rc = 0;
1057         }
1058         release_sock(smc->clcsock->sk);
1059         lock_sock(&smc->sk);
1060         if (rc != 0 || smc->sk.sk_err) {
1061                 smc->sk.sk_state = SMC_CLOSED;
1062                 if (rc == -EPIPE || rc == -EAGAIN)
1063                         smc->sk.sk_err = EPIPE;
1064                 else if (rc == -ECONNREFUSED)
1065                         smc->sk.sk_err = ECONNREFUSED;
1066                 else if (signal_pending(current))
1067                         smc->sk.sk_err = -sock_intr_errno(timeo);
1068                 sock_put(&smc->sk); /* passive closing */
1069                 goto out;
1070         }
1071
1072         rc = __smc_connect(smc);
1073         if (rc < 0)
1074                 smc->sk.sk_err = -rc;
1075
1076 out:
1077         if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1078                 if (smc->sk.sk_err) {
1079                         smc->sk.sk_state_change(&smc->sk);
1080                 } else { /* allow polling before and after fallback decision */
1081                         smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1082                         smc->sk.sk_write_space(&smc->sk);
1083                 }
1084         }
1085         release_sock(&smc->sk);
1086 }
1087
1088 static int smc_connect(struct socket *sock, struct sockaddr *addr,
1089                        int alen, int flags)
1090 {
1091         struct sock *sk = sock->sk;
1092         struct smc_sock *smc;
1093         int rc = -EINVAL;
1094
1095         smc = smc_sk(sk);
1096
1097         /* separate smc parameter checking to be safe */
1098         if (alen < sizeof(addr->sa_family))
1099                 goto out_err;
1100         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
1101                 goto out_err;
1102
1103         lock_sock(sk);
1104         switch (sk->sk_state) {
1105         default:
1106                 goto out;
1107         case SMC_ACTIVE:
1108                 rc = -EISCONN;
1109                 goto out;
1110         case SMC_INIT:
1111                 rc = 0;
1112                 break;
1113         }
1114
1115         smc_copy_sock_settings_to_clc(smc);
1116         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1117         if (smc->connect_nonblock) {
1118                 rc = -EALREADY;
1119                 goto out;
1120         }
1121         rc = kernel_connect(smc->clcsock, addr, alen, flags);
1122         if (rc && rc != -EINPROGRESS)
1123                 goto out;
1124
1125         if (smc->use_fallback)
1126                 goto out;
1127         sock_hold(&smc->sk); /* sock put in passive closing */
1128         if (flags & O_NONBLOCK) {
1129                 if (queue_work(smc_hs_wq, &smc->connect_work))
1130                         smc->connect_nonblock = 1;
1131                 rc = -EINPROGRESS;
1132         } else {
1133                 rc = __smc_connect(smc);
1134                 if (rc < 0)
1135                         goto out;
1136                 else
1137                         rc = 0; /* success cases including fallback */
1138         }
1139
1140 out:
1141         release_sock(sk);
1142 out_err:
1143         return rc;
1144 }
1145
1146 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1147 {
1148         struct socket *new_clcsock = NULL;
1149         struct sock *lsk = &lsmc->sk;
1150         struct sock *new_sk;
1151         int rc = -EINVAL;
1152
1153         release_sock(lsk);
1154         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
1155         if (!new_sk) {
1156                 rc = -ENOMEM;
1157                 lsk->sk_err = ENOMEM;
1158                 *new_smc = NULL;
1159                 lock_sock(lsk);
1160                 goto out;
1161         }
1162         *new_smc = smc_sk(new_sk);
1163
1164         mutex_lock(&lsmc->clcsock_release_lock);
1165         if (lsmc->clcsock)
1166                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1167         mutex_unlock(&lsmc->clcsock_release_lock);
1168         lock_sock(lsk);
1169         if  (rc < 0 && rc != -EAGAIN)
1170                 lsk->sk_err = -rc;
1171         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1172                 new_sk->sk_prot->unhash(new_sk);
1173                 if (new_clcsock)
1174                         sock_release(new_clcsock);
1175                 new_sk->sk_state = SMC_CLOSED;
1176                 smc_sock_set_flag(new_sk, SOCK_DEAD);
1177                 sock_put(new_sk); /* final */
1178                 *new_smc = NULL;
1179                 goto out;
1180         }
1181
1182         /* new clcsock has inherited the smc listen-specific sk_data_ready
1183          * function; switch it back to the original sk_data_ready function
1184          */
1185         new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
1186         (*new_smc)->clcsock = new_clcsock;
1187 out:
1188         return rc;
1189 }
1190
1191 /* add a just created sock to the accept queue of the listen sock as
1192  * candidate for a following socket accept call from user space
1193  */
1194 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1195 {
1196         struct smc_sock *par = smc_sk(parent);
1197
1198         sock_hold(sk); /* sock_put in smc_accept_unlink () */
1199         spin_lock(&par->accept_q_lock);
1200         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1201         spin_unlock(&par->accept_q_lock);
1202         sk_acceptq_added(parent);
1203 }
1204
1205 /* remove a socket from the accept queue of its parental listening socket */
1206 static void smc_accept_unlink(struct sock *sk)
1207 {
1208         struct smc_sock *par = smc_sk(sk)->listen_smc;
1209
1210         spin_lock(&par->accept_q_lock);
1211         list_del_init(&smc_sk(sk)->accept_q);
1212         spin_unlock(&par->accept_q_lock);
1213         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1214         sock_put(sk); /* sock_hold in smc_accept_enqueue */
1215 }
1216
1217 /* remove a sock from the accept queue to bind it to a new socket created
1218  * for a socket accept call from user space
1219  */
1220 struct sock *smc_accept_dequeue(struct sock *parent,
1221                                 struct socket *new_sock)
1222 {
1223         struct smc_sock *isk, *n;
1224         struct sock *new_sk;
1225
1226         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1227                 new_sk = (struct sock *)isk;
1228
1229                 smc_accept_unlink(new_sk);
1230                 if (new_sk->sk_state == SMC_CLOSED) {
1231                         new_sk->sk_prot->unhash(new_sk);
1232                         if (isk->clcsock) {
1233                                 sock_release(isk->clcsock);
1234                                 isk->clcsock = NULL;
1235                         }
1236                         sock_put(new_sk); /* final */
1237                         continue;
1238                 }
1239                 if (new_sock) {
1240                         sock_graft(new_sk, new_sock);
1241                         if (isk->use_fallback) {
1242                                 smc_sk(new_sk)->clcsock->file = new_sock->file;
1243                                 isk->clcsock->file->private_data = isk->clcsock;
1244                         }
1245                 }
1246                 return new_sk;
1247         }
1248         return NULL;
1249 }
1250
1251 /* clean up for a created but never accepted sock */
1252 void smc_close_non_accepted(struct sock *sk)
1253 {
1254         struct smc_sock *smc = smc_sk(sk);
1255
1256         sock_hold(sk); /* sock_put below */
1257         lock_sock(sk);
1258         if (!sk->sk_lingertime)
1259                 /* wait for peer closing */
1260                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1261         __smc_release(smc);
1262         release_sock(sk);
1263         sock_put(sk); /* sock_hold above */
1264         sock_put(sk); /* final sock_put */
1265 }
1266
1267 static int smcr_serv_conf_first_link(struct smc_sock *smc)
1268 {
1269         struct smc_link *link = smc->conn.lnk;
1270         struct smc_llc_qentry *qentry;
1271         int rc;
1272
1273         if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
1274                 return SMC_CLC_DECL_ERR_REGRMB;
1275
1276         /* send CONFIRM LINK request to client over the RoCE fabric */
1277         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
1278         if (rc < 0)
1279                 return SMC_CLC_DECL_TIMEOUT_CL;
1280
1281         /* receive CONFIRM LINK response from client over the RoCE fabric */
1282         qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1283                               SMC_LLC_CONFIRM_LINK);
1284         if (!qentry) {
1285                 struct smc_clc_msg_decline dclc;
1286
1287                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1288                                       SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1289                 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
1290         }
1291         smc_llc_save_peer_uid(qentry);
1292         rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1293         smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1294         if (rc)
1295                 return SMC_CLC_DECL_RMBE_EC;
1296
1297         /* confirm_rkey is implicit on 1st contact */
1298         smc->conn.rmb_desc->is_conf_rkey = true;
1299
1300         smc_llc_link_active(link);
1301         smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1302
1303         /* initial contact - try to establish second link */
1304         smc_llc_srv_add_link(link);
1305         return 0;
1306 }
1307
1308 /* listen worker: finish */
1309 static void smc_listen_out(struct smc_sock *new_smc)
1310 {
1311         struct smc_sock *lsmc = new_smc->listen_smc;
1312         struct sock *newsmcsk = &new_smc->sk;
1313
1314         if (lsmc->sk.sk_state == SMC_LISTEN) {
1315                 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1316                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1317                 release_sock(&lsmc->sk);
1318         } else { /* no longer listening */
1319                 smc_close_non_accepted(newsmcsk);
1320         }
1321
1322         /* Wake up accept */
1323         lsmc->sk.sk_data_ready(&lsmc->sk);
1324         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1325 }
1326
1327 /* listen worker: finish in state connected */
1328 static void smc_listen_out_connected(struct smc_sock *new_smc)
1329 {
1330         struct sock *newsmcsk = &new_smc->sk;
1331
1332         if (newsmcsk->sk_state == SMC_INIT)
1333                 newsmcsk->sk_state = SMC_ACTIVE;
1334
1335         smc_listen_out(new_smc);
1336 }
1337
1338 /* listen worker: finish in error state */
1339 static void smc_listen_out_err(struct smc_sock *new_smc)
1340 {
1341         struct sock *newsmcsk = &new_smc->sk;
1342
1343         if (newsmcsk->sk_state == SMC_INIT)
1344                 sock_put(&new_smc->sk); /* passive closing */
1345         newsmcsk->sk_state = SMC_CLOSED;
1346
1347         smc_listen_out(new_smc);
1348 }
1349
1350 /* listen worker: decline and fall back if possible */
1351 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1352                                int local_first, u8 version)
1353 {
1354         /* RDMA setup failed, switch back to TCP */
1355         if (local_first)
1356                 smc_lgr_cleanup_early(&new_smc->conn);
1357         else
1358                 smc_conn_free(&new_smc->conn);
1359         if (reason_code < 0) { /* error, no fallback possible */
1360                 smc_listen_out_err(new_smc);
1361                 return;
1362         }
1363         smc_switch_to_fallback(new_smc);
1364         new_smc->fallback_rsn = reason_code;
1365         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1366                 if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
1367                         smc_listen_out_err(new_smc);
1368                         return;
1369                 }
1370         }
1371         smc_listen_out_connected(new_smc);
1372 }
1373
1374 /* listen worker: version checking */
1375 static int smc_listen_v2_check(struct smc_sock *new_smc,
1376                                struct smc_clc_msg_proposal *pclc,
1377                                struct smc_init_info *ini)
1378 {
1379         struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1380         struct smc_clc_v2_extension *pclc_v2_ext;
1381
1382         ini->smc_type_v1 = pclc->hdr.typev1;
1383         ini->smc_type_v2 = pclc->hdr.typev2;
1384         ini->smcd_version = ini->smc_type_v1 != SMC_TYPE_N ? SMC_V1 : 0;
1385         if (pclc->hdr.version > SMC_V1)
1386                 ini->smcd_version |=
1387                                 ini->smc_type_v2 != SMC_TYPE_N ? SMC_V2 : 0;
1388         if (!smc_ism_v2_capable) {
1389                 ini->smcd_version &= ~SMC_V2;
1390                 goto out;
1391         }
1392         pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1393         if (!pclc_v2_ext) {
1394                 ini->smcd_version &= ~SMC_V2;
1395                 goto out;
1396         }
1397         pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
1398         if (!pclc_smcd_v2_ext)
1399                 ini->smcd_version &= ~SMC_V2;
1400
1401 out:
1402         if (!ini->smcd_version) {
1403                 if (pclc->hdr.typev1 == SMC_TYPE_B ||
1404                     pclc->hdr.typev2 == SMC_TYPE_B)
1405                         return SMC_CLC_DECL_NOSMCDEV;
1406                 if (pclc->hdr.typev1 == SMC_TYPE_D ||
1407                     pclc->hdr.typev2 == SMC_TYPE_D)
1408                         return SMC_CLC_DECL_NOSMCDDEV;
1409                 return SMC_CLC_DECL_NOSMCRDEV;
1410         }
1411
1412         return 0;
1413 }
1414
1415 /* listen worker: check prefixes */
1416 static int smc_listen_prfx_check(struct smc_sock *new_smc,
1417                                  struct smc_clc_msg_proposal *pclc)
1418 {
1419         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1420         struct socket *newclcsock = new_smc->clcsock;
1421
1422         if (pclc->hdr.typev1 == SMC_TYPE_N)
1423                 return 0;
1424         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1425         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1426                 return SMC_CLC_DECL_DIFFPREFIX;
1427
1428         return 0;
1429 }
1430
1431 /* listen worker: initialize connection and buffers */
1432 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1433                                 struct smc_init_info *ini)
1434 {
1435         int rc;
1436
1437         /* allocate connection / link group */
1438         rc = smc_conn_create(new_smc, ini);
1439         if (rc)
1440                 return rc;
1441
1442         /* create send buffer and rmb */
1443         if (smc_buf_create(new_smc, false))
1444                 return SMC_CLC_DECL_MEM;
1445
1446         return 0;
1447 }
1448
1449 /* listen worker: initialize connection and buffers for SMC-D */
1450 static int smc_listen_ism_init(struct smc_sock *new_smc,
1451                                struct smc_init_info *ini)
1452 {
1453         int rc;
1454
1455         rc = smc_conn_create(new_smc, ini);
1456         if (rc)
1457                 return rc;
1458
1459         /* Create send and receive buffers */
1460         rc = smc_buf_create(new_smc, true);
1461         if (rc) {
1462                 if (ini->first_contact_local)
1463                         smc_lgr_cleanup_early(&new_smc->conn);
1464                 else
1465                         smc_conn_free(&new_smc->conn);
1466                 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
1467                                          SMC_CLC_DECL_MEM;
1468         }
1469
1470         return 0;
1471 }
1472
1473 static bool smc_is_already_selected(struct smcd_dev *smcd,
1474                                     struct smc_init_info *ini,
1475                                     int matches)
1476 {
1477         int i;
1478
1479         for (i = 0; i < matches; i++)
1480                 if (smcd == ini->ism_dev[i])
1481                         return true;
1482
1483         return false;
1484 }
1485
1486 /* check for ISM devices matching proposed ISM devices */
1487 static void smc_check_ism_v2_match(struct smc_init_info *ini,
1488                                    u16 proposed_chid, u64 proposed_gid,
1489                                    unsigned int *matches)
1490 {
1491         struct smcd_dev *smcd;
1492
1493         list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1494                 if (smcd->going_away)
1495                         continue;
1496                 if (smc_is_already_selected(smcd, ini, *matches))
1497                         continue;
1498                 if (smc_ism_get_chid(smcd) == proposed_chid &&
1499                     !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
1500                         ini->ism_peer_gid[*matches] = proposed_gid;
1501                         ini->ism_dev[*matches] = smcd;
1502                         (*matches)++;
1503                         break;
1504                 }
1505         }
1506 }
1507
1508 static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
1509                                         struct smc_clc_msg_proposal *pclc,
1510                                         struct smc_init_info *ini)
1511 {
1512         struct smc_clc_smcd_v2_extension *smcd_v2_ext;
1513         struct smc_clc_v2_extension *smc_v2_ext;
1514         struct smc_clc_msg_smcd *pclc_smcd;
1515         unsigned int matches = 0;
1516         u8 smcd_version;
1517         u8 *eid = NULL;
1518         int i;
1519
1520         if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
1521                 goto not_found;
1522
1523         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1524         smc_v2_ext = smc_get_clc_v2_ext(pclc);
1525         smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
1526         if (!smcd_v2_ext ||
1527             !smc_v2_ext->hdr.flag.seid) /* no system EID support for SMCD */
1528                 goto not_found;
1529
1530         mutex_lock(&smcd_dev_list.mutex);
1531         if (pclc_smcd->ism.chid)
1532                 /* check for ISM device matching proposed native ISM device */
1533                 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
1534                                        ntohll(pclc_smcd->ism.gid), &matches);
1535         for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
1536                 /* check for ISM devices matching proposed non-native ISM
1537                  * devices
1538                  */
1539                 smc_check_ism_v2_match(ini,
1540                                        ntohs(smcd_v2_ext->gidchid[i - 1].chid),
1541                                        ntohll(smcd_v2_ext->gidchid[i - 1].gid),
1542                                        &matches);
1543         }
1544         mutex_unlock(&smcd_dev_list.mutex);
1545
1546         if (ini->ism_dev[0]) {
1547                 smc_ism_get_system_eid(ini->ism_dev[0], &eid);
1548                 if (memcmp(eid, smcd_v2_ext->system_eid, SMC_MAX_EID_LEN))
1549                         goto not_found;
1550         } else {
1551                 goto not_found;
1552         }
1553
1554         /* separate - outside the smcd_dev_list.lock */
1555         smcd_version = ini->smcd_version;
1556         for (i = 0; i < matches; i++) {
1557                 ini->smcd_version = SMC_V2;
1558                 ini->is_smcd = true;
1559                 ini->ism_selected = i;
1560                 if (smc_listen_ism_init(new_smc, ini))
1561                         /* try next active ISM device */
1562                         continue;
1563                 return; /* matching and usable V2 ISM device found */
1564         }
1565         /* no V2 ISM device could be initialized */
1566         ini->smcd_version = smcd_version;       /* restore original value */
1567
1568 not_found:
1569         ini->smcd_version &= ~SMC_V2;
1570         ini->ism_dev[0] = NULL;
1571         ini->is_smcd = false;
1572 }
1573
1574 static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
1575                                         struct smc_clc_msg_proposal *pclc,
1576                                         struct smc_init_info *ini)
1577 {
1578         struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
1579
1580         /* check if ISM V1 is available */
1581         if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
1582                 goto not_found;
1583         ini->is_smcd = true; /* prepare ISM check */
1584         ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
1585         if (smc_find_ism_device(new_smc, ini))
1586                 goto not_found;
1587         ini->ism_selected = 0;
1588         if (!smc_listen_ism_init(new_smc, ini))
1589                 return;         /* V1 ISM device found */
1590
1591 not_found:
1592         ini->ism_dev[0] = NULL;
1593         ini->is_smcd = false;
1594 }
1595
1596 /* listen worker: register buffers */
1597 static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
1598 {
1599         struct smc_connection *conn = &new_smc->conn;
1600
1601         if (!local_first) {
1602                 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
1603                         return SMC_CLC_DECL_ERR_REGRMB;
1604         }
1605         smc_rmb_sync_sg_for_device(&new_smc->conn);
1606
1607         return 0;
1608 }
1609
1610 static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
1611                                         struct smc_clc_msg_proposal *pclc,
1612                                         struct smc_init_info *ini)
1613 {
1614         int rc;
1615
1616         if (!smcr_indicated(ini->smc_type_v1))
1617                 return SMC_CLC_DECL_NOSMCDEV;
1618
1619         /* prepare RDMA check */
1620         ini->ib_lcl = &pclc->lcl;
1621         rc = smc_find_rdma_device(new_smc, ini);
1622         if (rc) {
1623                 /* no RDMA device found */
1624                 if (ini->smc_type_v1 == SMC_TYPE_B)
1625                         /* neither ISM nor RDMA device found */
1626                         rc = SMC_CLC_DECL_NOSMCDEV;
1627                 return rc;
1628         }
1629         rc = smc_listen_rdma_init(new_smc, ini);
1630         if (rc)
1631                 return rc;
1632         return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1633 }
1634
1635 /* determine the local device matching to proposal */
1636 static int smc_listen_find_device(struct smc_sock *new_smc,
1637                                   struct smc_clc_msg_proposal *pclc,
1638                                   struct smc_init_info *ini)
1639 {
1640         int rc;
1641
1642         /* check for ISM device matching V2 proposed device */
1643         smc_find_ism_v2_device_serv(new_smc, pclc, ini);
1644         if (ini->ism_dev[0])
1645                 return 0;
1646
1647         if (!(ini->smcd_version & SMC_V1))
1648                 return SMC_CLC_DECL_NOSMCDEV;
1649
1650         /* check for matching IP prefix and subnet length */
1651         rc = smc_listen_prfx_check(new_smc, pclc);
1652         if (rc)
1653                 return rc;
1654
1655         /* get vlan id from IP device */
1656         if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
1657                 return SMC_CLC_DECL_GETVLANERR;
1658
1659         /* check for ISM device matching V1 proposed device */
1660         smc_find_ism_v1_device_serv(new_smc, pclc, ini);
1661         if (ini->ism_dev[0])
1662                 return 0;
1663
1664         if (pclc->hdr.typev1 == SMC_TYPE_D)
1665                 return SMC_CLC_DECL_NOSMCDDEV; /* skip RDMA and decline */
1666
1667         /* check if RDMA is available */
1668         return smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
1669 }
1670
1671 /* listen worker: finish RDMA setup */
1672 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1673                                   struct smc_clc_msg_accept_confirm *cclc,
1674                                   bool local_first)
1675 {
1676         struct smc_link *link = new_smc->conn.lnk;
1677         int reason_code = 0;
1678
1679         if (local_first)
1680                 smc_link_save_peer_info(link, cclc);
1681
1682         if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
1683                 return SMC_CLC_DECL_ERR_RTOK;
1684
1685         if (local_first) {
1686                 if (smc_ib_ready_link(link))
1687                         return SMC_CLC_DECL_ERR_RDYLNK;
1688                 /* QP confirmation over RoCE fabric */
1689                 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1690                 reason_code = smcr_serv_conf_first_link(new_smc);
1691                 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1692         }
1693         return reason_code;
1694 }
1695
1696 /* setup for connection of server */
1697 static void smc_listen_work(struct work_struct *work)
1698 {
1699         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1700                                                 smc_listen_work);
1701         u8 version = smc_ism_v2_capable ? SMC_V2 : SMC_V1;
1702         struct socket *newclcsock = new_smc->clcsock;
1703         struct smc_clc_msg_accept_confirm *cclc;
1704         struct smc_clc_msg_proposal_area *buf;
1705         struct smc_clc_msg_proposal *pclc;
1706         struct smc_init_info *ini = NULL;
1707         int rc = 0;
1708
1709         if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1710                 return smc_listen_out_err(new_smc);
1711
1712         if (new_smc->use_fallback) {
1713                 smc_listen_out_connected(new_smc);
1714                 return;
1715         }
1716
1717         /* check if peer is smc capable */
1718         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1719                 smc_switch_to_fallback(new_smc);
1720                 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1721                 smc_listen_out_connected(new_smc);
1722                 return;
1723         }
1724
1725         /* do inband token exchange -
1726          * wait for and receive SMC Proposal CLC message
1727          */
1728         buf = kzalloc(sizeof(*buf), GFP_KERNEL);
1729         if (!buf) {
1730                 rc = SMC_CLC_DECL_MEM;
1731                 goto out_decl;
1732         }
1733         pclc = (struct smc_clc_msg_proposal *)buf;
1734         rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
1735                               SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1736         if (rc)
1737                 goto out_decl;
1738         version = pclc->hdr.version == SMC_V1 ? SMC_V1 : version;
1739
1740         /* IPSec connections opt out of SMC optimizations */
1741         if (using_ipsec(new_smc)) {
1742                 rc = SMC_CLC_DECL_IPSEC;
1743                 goto out_decl;
1744         }
1745
1746         ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1747         if (!ini) {
1748                 rc = SMC_CLC_DECL_MEM;
1749                 goto out_decl;
1750         }
1751
1752         /* initial version checking */
1753         rc = smc_listen_v2_check(new_smc, pclc, ini);
1754         if (rc)
1755                 goto out_decl;
1756
1757         mutex_lock(&smc_server_lgr_pending);
1758         smc_close_init(new_smc);
1759         smc_rx_init(new_smc);
1760         smc_tx_init(new_smc);
1761
1762         /* determine ISM or RoCE device used for connection */
1763         rc = smc_listen_find_device(new_smc, pclc, ini);
1764         if (rc)
1765                 goto out_unlock;
1766
1767         /* send SMC Accept CLC message */
1768         rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
1769                                  ini->smcd_version == SMC_V2 ? SMC_V2 : SMC_V1);
1770         if (rc)
1771                 goto out_unlock;
1772
1773         /* SMC-D does not need this lock any more */
1774         if (ini->is_smcd)
1775                 mutex_unlock(&smc_server_lgr_pending);
1776
1777         /* receive SMC Confirm CLC message */
1778         memset(buf, 0, sizeof(*buf));
1779         cclc = (struct smc_clc_msg_accept_confirm *)buf;
1780         rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
1781                               SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1782         if (rc) {
1783                 if (!ini->is_smcd)
1784                         goto out_unlock;
1785                 goto out_decl;
1786         }
1787
1788         /* finish worker */
1789         if (!ini->is_smcd) {
1790                 rc = smc_listen_rdma_finish(new_smc, cclc,
1791                                             ini->first_contact_local);
1792                 if (rc)
1793                         goto out_unlock;
1794                 mutex_unlock(&smc_server_lgr_pending);
1795         }
1796         smc_conn_save_peer_info(new_smc, cclc);
1797         smc_listen_out_connected(new_smc);
1798         goto out_free;
1799
1800 out_unlock:
1801         mutex_unlock(&smc_server_lgr_pending);
1802 out_decl:
1803         smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
1804                            version);
1805 out_free:
1806         kfree(ini);
1807         kfree(buf);
1808 }
1809
1810 static void smc_tcp_listen_work(struct work_struct *work)
1811 {
1812         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1813                                              tcp_listen_work);
1814         struct sock *lsk = &lsmc->sk;
1815         struct smc_sock *new_smc;
1816         int rc = 0;
1817
1818         lock_sock(lsk);
1819         while (lsk->sk_state == SMC_LISTEN) {
1820                 rc = smc_clcsock_accept(lsmc, &new_smc);
1821                 if (rc) /* clcsock accept queue empty or error */
1822                         goto out;
1823                 if (!new_smc)
1824                         continue;
1825
1826                 new_smc->listen_smc = lsmc;
1827                 new_smc->use_fallback = lsmc->use_fallback;
1828                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1829                 sock_hold(lsk); /* sock_put in smc_listen_work */
1830                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1831                 smc_copy_sock_settings_to_smc(new_smc);
1832                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1833                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1834                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1835                 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
1836                         sock_put(&new_smc->sk);
1837         }
1838
1839 out:
1840         release_sock(lsk);
1841         sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
1842 }
1843
1844 static void smc_clcsock_data_ready(struct sock *listen_clcsock)
1845 {
1846         struct smc_sock *lsmc;
1847
1848         lsmc = (struct smc_sock *)
1849                ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY);
1850         if (!lsmc)
1851                 return;
1852         lsmc->clcsk_data_ready(listen_clcsock);
1853         if (lsmc->sk.sk_state == SMC_LISTEN) {
1854                 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
1855                 if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
1856                         sock_put(&lsmc->sk);
1857         }
1858 }
1859
1860 static int smc_listen(struct socket *sock, int backlog)
1861 {
1862         struct sock *sk = sock->sk;
1863         struct smc_sock *smc;
1864         int rc;
1865
1866         smc = smc_sk(sk);
1867         lock_sock(sk);
1868
1869         rc = -EINVAL;
1870         if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1871             smc->connect_nonblock)
1872                 goto out;
1873
1874         rc = 0;
1875         if (sk->sk_state == SMC_LISTEN) {
1876                 sk->sk_max_ack_backlog = backlog;
1877                 goto out;
1878         }
1879         /* some socket options are handled in core, so we could not apply
1880          * them to the clc socket -- copy smc socket options to clc socket
1881          */
1882         smc_copy_sock_settings_to_clc(smc);
1883         if (!smc->use_fallback)
1884                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1885
1886         /* save original sk_data_ready function and establish
1887          * smc-specific sk_data_ready function
1888          */
1889         smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
1890         smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
1891         smc->clcsock->sk->sk_user_data =
1892                 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
1893         rc = kernel_listen(smc->clcsock, backlog);
1894         if (rc) {
1895                 smc->clcsock->sk->sk_data_ready = smc->clcsk_data_ready;
1896                 goto out;
1897         }
1898         sk->sk_max_ack_backlog = backlog;
1899         sk->sk_ack_backlog = 0;
1900         sk->sk_state = SMC_LISTEN;
1901
1902 out:
1903         release_sock(sk);
1904         return rc;
1905 }
1906
1907 static int smc_accept(struct socket *sock, struct socket *new_sock,
1908                       int flags, bool kern)
1909 {
1910         struct sock *sk = sock->sk, *nsk;
1911         DECLARE_WAITQUEUE(wait, current);
1912         struct smc_sock *lsmc;
1913         long timeo;
1914         int rc = 0;
1915
1916         lsmc = smc_sk(sk);
1917         sock_hold(sk); /* sock_put below */
1918         lock_sock(sk);
1919
1920         if (lsmc->sk.sk_state != SMC_LISTEN) {
1921                 rc = -EINVAL;
1922                 release_sock(sk);
1923                 goto out;
1924         }
1925
1926         /* Wait for an incoming connection */
1927         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1928         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1929         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1930                 set_current_state(TASK_INTERRUPTIBLE);
1931                 if (!timeo) {
1932                         rc = -EAGAIN;
1933                         break;
1934                 }
1935                 release_sock(sk);
1936                 timeo = schedule_timeout(timeo);
1937                 /* wakeup by sk_data_ready in smc_listen_work() */
1938                 sched_annotate_sleep();
1939                 lock_sock(sk);
1940                 if (signal_pending(current)) {
1941                         rc = sock_intr_errno(timeo);
1942                         break;
1943                 }
1944         }
1945         set_current_state(TASK_RUNNING);
1946         remove_wait_queue(sk_sleep(sk), &wait);
1947
1948         if (!rc)
1949                 rc = sock_error(nsk);
1950         release_sock(sk);
1951         if (rc)
1952                 goto out;
1953
1954         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1955                 /* wait till data arrives on the socket */
1956                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1957                                                                 MSEC_PER_SEC);
1958                 if (smc_sk(nsk)->use_fallback) {
1959                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1960
1961                         lock_sock(clcsk);
1962                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1963                                 sk_wait_data(clcsk, &timeo, NULL);
1964                         release_sock(clcsk);
1965                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1966                         lock_sock(nsk);
1967                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1968                         release_sock(nsk);
1969                 }
1970         }
1971
1972 out:
1973         sock_put(sk); /* sock_hold above */
1974         return rc;
1975 }
1976
1977 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1978                        int peer)
1979 {
1980         struct smc_sock *smc;
1981
1982         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1983             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1984                 return -ENOTCONN;
1985
1986         smc = smc_sk(sock->sk);
1987
1988         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1989 }
1990
1991 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1992 {
1993         struct sock *sk = sock->sk;
1994         struct smc_sock *smc;
1995         int rc;
1996
1997         smc = smc_sk(sk);
1998         lock_sock(sk);
1999
2000         /* SMC does not support connect with fastopen */
2001         if (msg->msg_flags & MSG_FASTOPEN) {
2002                 /* not connected yet, fallback */
2003                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2004                         smc_switch_to_fallback(smc);
2005                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
2006                 } else {
2007                         rc = -EINVAL;
2008                         goto out;
2009                 }
2010         } else if ((sk->sk_state != SMC_ACTIVE) &&
2011                    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2012                    (sk->sk_state != SMC_INIT)) {
2013                 rc = -EPIPE;
2014                 goto out;
2015         }
2016
2017         if (smc->use_fallback)
2018                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
2019         else
2020                 rc = smc_tx_sendmsg(smc, msg, len);
2021 out:
2022         release_sock(sk);
2023         return rc;
2024 }
2025
2026 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2027                        int flags)
2028 {
2029         struct sock *sk = sock->sk;
2030         struct smc_sock *smc;
2031         int rc = -ENOTCONN;
2032
2033         smc = smc_sk(sk);
2034         lock_sock(sk);
2035         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2036                 /* socket was connected before, no more data to read */
2037                 rc = 0;
2038                 goto out;
2039         }
2040         if ((sk->sk_state == SMC_INIT) ||
2041             (sk->sk_state == SMC_LISTEN) ||
2042             (sk->sk_state == SMC_CLOSED))
2043                 goto out;
2044
2045         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2046                 rc = 0;
2047                 goto out;
2048         }
2049
2050         if (smc->use_fallback) {
2051                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
2052         } else {
2053                 msg->msg_namelen = 0;
2054                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
2055         }
2056
2057 out:
2058         release_sock(sk);
2059         return rc;
2060 }
2061
2062 static __poll_t smc_accept_poll(struct sock *parent)
2063 {
2064         struct smc_sock *isk = smc_sk(parent);
2065         __poll_t mask = 0;
2066
2067         spin_lock(&isk->accept_q_lock);
2068         if (!list_empty(&isk->accept_q))
2069                 mask = EPOLLIN | EPOLLRDNORM;
2070         spin_unlock(&isk->accept_q_lock);
2071
2072         return mask;
2073 }
2074
2075 static __poll_t smc_poll(struct file *file, struct socket *sock,
2076                              poll_table *wait)
2077 {
2078         struct sock *sk = sock->sk;
2079         struct smc_sock *smc;
2080         __poll_t mask = 0;
2081
2082         if (!sk)
2083                 return EPOLLNVAL;
2084
2085         smc = smc_sk(sock->sk);
2086         if (smc->use_fallback) {
2087                 /* delegate to CLC child sock */
2088                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2089                 sk->sk_err = smc->clcsock->sk->sk_err;
2090         } else {
2091                 if (sk->sk_state != SMC_CLOSED)
2092                         sock_poll_wait(file, sock, wait);
2093                 if (sk->sk_err)
2094                         mask |= EPOLLERR;
2095                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2096                     (sk->sk_state == SMC_CLOSED))
2097                         mask |= EPOLLHUP;
2098                 if (sk->sk_state == SMC_LISTEN) {
2099                         /* woken up by sk_data_ready in smc_listen_work() */
2100                         mask |= smc_accept_poll(sk);
2101                 } else if (smc->use_fallback) { /* as result of connect_work()*/
2102                         mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2103                                                            wait);
2104                         sk->sk_err = smc->clcsock->sk->sk_err;
2105                 } else {
2106                         if ((sk->sk_state != SMC_INIT &&
2107                              atomic_read(&smc->conn.sndbuf_space)) ||
2108                             sk->sk_shutdown & SEND_SHUTDOWN) {
2109                                 mask |= EPOLLOUT | EPOLLWRNORM;
2110                         } else {
2111                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2112                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2113                         }
2114                         if (atomic_read(&smc->conn.bytes_to_rcv))
2115                                 mask |= EPOLLIN | EPOLLRDNORM;
2116                         if (sk->sk_shutdown & RCV_SHUTDOWN)
2117                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2118                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
2119                                 mask |= EPOLLIN;
2120                         if (smc->conn.urg_state == SMC_URG_VALID)
2121                                 mask |= EPOLLPRI;
2122                 }
2123         }
2124
2125         return mask;
2126 }
2127
2128 static int smc_shutdown(struct socket *sock, int how)
2129 {
2130         struct sock *sk = sock->sk;
2131         bool do_shutdown = true;
2132         struct smc_sock *smc;
2133         int rc = -EINVAL;
2134         int old_state;
2135         int rc1 = 0;
2136
2137         smc = smc_sk(sk);
2138
2139         if ((how < SHUT_RD) || (how > SHUT_RDWR))
2140                 return rc;
2141
2142         lock_sock(sk);
2143
2144         rc = -ENOTCONN;
2145         if ((sk->sk_state != SMC_ACTIVE) &&
2146             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2147             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2148             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2149             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2150             (sk->sk_state != SMC_APPFINCLOSEWAIT))
2151                 goto out;
2152         if (smc->use_fallback) {
2153                 rc = kernel_sock_shutdown(smc->clcsock, how);
2154                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2155                 if (sk->sk_shutdown == SHUTDOWN_MASK) {
2156                         sk->sk_state = SMC_CLOSED;
2157                         sock_put(sk);
2158                 }
2159                 goto out;
2160         }
2161         switch (how) {
2162         case SHUT_RDWR:         /* shutdown in both directions */
2163                 old_state = sk->sk_state;
2164                 rc = smc_close_active(smc);
2165                 if (old_state == SMC_ACTIVE &&
2166                     sk->sk_state == SMC_PEERCLOSEWAIT1)
2167                         do_shutdown = false;
2168                 break;
2169         case SHUT_WR:
2170                 rc = smc_close_shutdown_write(smc);
2171                 break;
2172         case SHUT_RD:
2173                 rc = 0;
2174                 /* nothing more to do because peer is not involved */
2175                 break;
2176         }
2177         if (do_shutdown && smc->clcsock)
2178                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
2179         /* map sock_shutdown_cmd constants to sk_shutdown value range */
2180         sk->sk_shutdown |= how + 1;
2181
2182 out:
2183         release_sock(sk);
2184         return rc ? rc : rc1;
2185 }
2186
2187 static int smc_setsockopt(struct socket *sock, int level, int optname,
2188                           sockptr_t optval, unsigned int optlen)
2189 {
2190         struct sock *sk = sock->sk;
2191         struct smc_sock *smc;
2192         int val, rc;
2193
2194         if (level == SOL_TCP && optname == TCP_ULP)
2195                 return -EOPNOTSUPP;
2196
2197         smc = smc_sk(sk);
2198
2199         /* generic setsockopts reaching us here always apply to the
2200          * CLC socket
2201          */
2202         if (unlikely(!smc->clcsock->ops->setsockopt))
2203                 rc = -EOPNOTSUPP;
2204         else
2205                 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2206                                                    optval, optlen);
2207         if (smc->clcsock->sk->sk_err) {
2208                 sk->sk_err = smc->clcsock->sk->sk_err;
2209                 sk->sk_error_report(sk);
2210         }
2211
2212         if (optlen < sizeof(int))
2213                 return -EINVAL;
2214         if (copy_from_sockptr(&val, optval, sizeof(int)))
2215                 return -EFAULT;
2216
2217         lock_sock(sk);
2218         if (rc || smc->use_fallback)
2219                 goto out;
2220         switch (optname) {
2221         case TCP_FASTOPEN:
2222         case TCP_FASTOPEN_CONNECT:
2223         case TCP_FASTOPEN_KEY:
2224         case TCP_FASTOPEN_NO_COOKIE:
2225                 /* option not supported by SMC */
2226                 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2227                         smc_switch_to_fallback(smc);
2228                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
2229                 } else {
2230                         rc = -EINVAL;
2231                 }
2232                 break;
2233         case TCP_NODELAY:
2234                 if (sk->sk_state != SMC_INIT &&
2235                     sk->sk_state != SMC_LISTEN &&
2236                     sk->sk_state != SMC_CLOSED) {
2237                         if (val)
2238                                 mod_delayed_work(smc->conn.lgr->tx_wq,
2239                                                  &smc->conn.tx_work, 0);
2240                 }
2241                 break;
2242         case TCP_CORK:
2243                 if (sk->sk_state != SMC_INIT &&
2244                     sk->sk_state != SMC_LISTEN &&
2245                     sk->sk_state != SMC_CLOSED) {
2246                         if (!val)
2247                                 mod_delayed_work(smc->conn.lgr->tx_wq,
2248                                                  &smc->conn.tx_work, 0);
2249                 }
2250                 break;
2251         case TCP_DEFER_ACCEPT:
2252                 smc->sockopt_defer_accept = val;
2253                 break;
2254         default:
2255                 break;
2256         }
2257 out:
2258         release_sock(sk);
2259
2260         return rc;
2261 }
2262
2263 static int smc_getsockopt(struct socket *sock, int level, int optname,
2264                           char __user *optval, int __user *optlen)
2265 {
2266         struct smc_sock *smc;
2267
2268         smc = smc_sk(sock->sk);
2269         /* socket options apply to the CLC socket */
2270         if (unlikely(!smc->clcsock->ops->getsockopt))
2271                 return -EOPNOTSUPP;
2272         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
2273                                              optval, optlen);
2274 }
2275
2276 static int smc_ioctl(struct socket *sock, unsigned int cmd,
2277                      unsigned long arg)
2278 {
2279         union smc_host_cursor cons, urg;
2280         struct smc_connection *conn;
2281         struct smc_sock *smc;
2282         int answ;
2283
2284         smc = smc_sk(sock->sk);
2285         conn = &smc->conn;
2286         lock_sock(&smc->sk);
2287         if (smc->use_fallback) {
2288                 if (!smc->clcsock) {
2289                         release_sock(&smc->sk);
2290                         return -EBADF;
2291                 }
2292                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
2293                 release_sock(&smc->sk);
2294                 return answ;
2295         }
2296         switch (cmd) {
2297         case SIOCINQ: /* same as FIONREAD */
2298                 if (smc->sk.sk_state == SMC_LISTEN) {
2299                         release_sock(&smc->sk);
2300                         return -EINVAL;
2301                 }
2302                 if (smc->sk.sk_state == SMC_INIT ||
2303                     smc->sk.sk_state == SMC_CLOSED)
2304                         answ = 0;
2305                 else
2306                         answ = atomic_read(&smc->conn.bytes_to_rcv);
2307                 break;
2308         case SIOCOUTQ:
2309                 /* output queue size (not send + not acked) */
2310                 if (smc->sk.sk_state == SMC_LISTEN) {
2311                         release_sock(&smc->sk);
2312                         return -EINVAL;
2313                 }
2314                 if (smc->sk.sk_state == SMC_INIT ||
2315                     smc->sk.sk_state == SMC_CLOSED)
2316                         answ = 0;
2317                 else
2318                         answ = smc->conn.sndbuf_desc->len -
2319                                         atomic_read(&smc->conn.sndbuf_space);
2320                 break;
2321         case SIOCOUTQNSD:
2322                 /* output queue size (not send only) */
2323                 if (smc->sk.sk_state == SMC_LISTEN) {
2324                         release_sock(&smc->sk);
2325                         return -EINVAL;
2326                 }
2327                 if (smc->sk.sk_state == SMC_INIT ||
2328                     smc->sk.sk_state == SMC_CLOSED)
2329                         answ = 0;
2330                 else
2331                         answ = smc_tx_prepared_sends(&smc->conn);
2332                 break;
2333         case SIOCATMARK:
2334                 if (smc->sk.sk_state == SMC_LISTEN) {
2335                         release_sock(&smc->sk);
2336                         return -EINVAL;
2337                 }
2338                 if (smc->sk.sk_state == SMC_INIT ||
2339                     smc->sk.sk_state == SMC_CLOSED) {
2340                         answ = 0;
2341                 } else {
2342                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
2343                         smc_curs_copy(&urg, &conn->urg_curs, conn);
2344                         answ = smc_curs_diff(conn->rmb_desc->len,
2345                                              &cons, &urg) == 1;
2346                 }
2347                 break;
2348         default:
2349                 release_sock(&smc->sk);
2350                 return -ENOIOCTLCMD;
2351         }
2352         release_sock(&smc->sk);
2353
2354         return put_user(answ, (int __user *)arg);
2355 }
2356
2357 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
2358                             int offset, size_t size, int flags)
2359 {
2360         struct sock *sk = sock->sk;
2361         struct smc_sock *smc;
2362         int rc = -EPIPE;
2363
2364         smc = smc_sk(sk);
2365         lock_sock(sk);
2366         if (sk->sk_state != SMC_ACTIVE) {
2367                 release_sock(sk);
2368                 goto out;
2369         }
2370         release_sock(sk);
2371         if (smc->use_fallback)
2372                 rc = kernel_sendpage(smc->clcsock, page, offset,
2373                                      size, flags);
2374         else
2375                 rc = sock_no_sendpage(sock, page, offset, size, flags);
2376
2377 out:
2378         return rc;
2379 }
2380
2381 /* Map the affected portions of the rmbe into an spd, note the number of bytes
2382  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2383  * updates till whenever a respective page has been fully processed.
2384  * Note that subsequent recv() calls have to wait till all splice() processing
2385  * completed.
2386  */
2387 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
2388                                struct pipe_inode_info *pipe, size_t len,
2389                                unsigned int flags)
2390 {
2391         struct sock *sk = sock->sk;
2392         struct smc_sock *smc;
2393         int rc = -ENOTCONN;
2394
2395         smc = smc_sk(sk);
2396         lock_sock(sk);
2397         if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2398                 /* socket was connected before, no more data to read */
2399                 rc = 0;
2400                 goto out;
2401         }
2402         if (sk->sk_state == SMC_INIT ||
2403             sk->sk_state == SMC_LISTEN ||
2404             sk->sk_state == SMC_CLOSED)
2405                 goto out;
2406
2407         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2408                 rc = 0;
2409                 goto out;
2410         }
2411
2412         if (smc->use_fallback) {
2413                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
2414                                                     pipe, len, flags);
2415         } else {
2416                 if (*ppos) {
2417                         rc = -ESPIPE;
2418                         goto out;
2419                 }
2420                 if (flags & SPLICE_F_NONBLOCK)
2421                         flags = MSG_DONTWAIT;
2422                 else
2423                         flags = 0;
2424                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
2425         }
2426 out:
2427         release_sock(sk);
2428
2429         return rc;
2430 }
2431
2432 /* must look like tcp */
2433 static const struct proto_ops smc_sock_ops = {
2434         .family         = PF_SMC,
2435         .owner          = THIS_MODULE,
2436         .release        = smc_release,
2437         .bind           = smc_bind,
2438         .connect        = smc_connect,
2439         .socketpair     = sock_no_socketpair,
2440         .accept         = smc_accept,
2441         .getname        = smc_getname,
2442         .poll           = smc_poll,
2443         .ioctl          = smc_ioctl,
2444         .listen         = smc_listen,
2445         .shutdown       = smc_shutdown,
2446         .setsockopt     = smc_setsockopt,
2447         .getsockopt     = smc_getsockopt,
2448         .sendmsg        = smc_sendmsg,
2449         .recvmsg        = smc_recvmsg,
2450         .mmap           = sock_no_mmap,
2451         .sendpage       = smc_sendpage,
2452         .splice_read    = smc_splice_read,
2453 };
2454
2455 static int smc_create(struct net *net, struct socket *sock, int protocol,
2456                       int kern)
2457 {
2458         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
2459         struct smc_sock *smc;
2460         struct sock *sk;
2461         int rc;
2462
2463         rc = -ESOCKTNOSUPPORT;
2464         if (sock->type != SOCK_STREAM)
2465                 goto out;
2466
2467         rc = -EPROTONOSUPPORT;
2468         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
2469                 goto out;
2470
2471         rc = -ENOBUFS;
2472         sock->ops = &smc_sock_ops;
2473         sk = smc_sock_alloc(net, sock, protocol);
2474         if (!sk)
2475                 goto out;
2476
2477         /* create internal TCP socket for CLC handshake and fallback */
2478         smc = smc_sk(sk);
2479         smc->use_fallback = false; /* assume rdma capability first */
2480         smc->fallback_rsn = 0;
2481         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2482                               &smc->clcsock);
2483         if (rc) {
2484                 sk_common_release(sk);
2485                 goto out;
2486         }
2487         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2488         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
2489
2490 out:
2491         return rc;
2492 }
2493
2494 static const struct net_proto_family smc_sock_family_ops = {
2495         .family = PF_SMC,
2496         .owner  = THIS_MODULE,
2497         .create = smc_create,
2498 };
2499
2500 unsigned int smc_net_id;
2501
2502 static __net_init int smc_net_init(struct net *net)
2503 {
2504         return smc_pnet_net_init(net);
2505 }
2506
2507 static void __net_exit smc_net_exit(struct net *net)
2508 {
2509         smc_pnet_net_exit(net);
2510 }
2511
2512 static struct pernet_operations smc_net_ops = {
2513         .init = smc_net_init,
2514         .exit = smc_net_exit,
2515         .id   = &smc_net_id,
2516         .size = sizeof(struct smc_net),
2517 };
2518
2519 static int __init smc_init(void)
2520 {
2521         int rc;
2522
2523         rc = register_pernet_subsys(&smc_net_ops);
2524         if (rc)
2525                 return rc;
2526
2527         smc_ism_init();
2528         smc_clc_init();
2529
2530         rc = smc_pnet_init();
2531         if (rc)
2532                 goto out_pernet_subsys;
2533
2534         rc = -ENOMEM;
2535         smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
2536         if (!smc_hs_wq)
2537                 goto out_pnet;
2538
2539         smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
2540         if (!smc_close_wq)
2541                 goto out_alloc_hs_wq;
2542
2543         rc = smc_core_init();
2544         if (rc) {
2545                 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2546                 goto out_alloc_wqs;
2547         }
2548
2549         rc = smc_llc_init();
2550         if (rc) {
2551                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2552                 goto out_core;
2553         }
2554
2555         rc = smc_cdc_init();
2556         if (rc) {
2557                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2558                 goto out_core;
2559         }
2560
2561         rc = proto_register(&smc_proto, 1);
2562         if (rc) {
2563                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2564                 goto out_core;
2565         }
2566
2567         rc = proto_register(&smc_proto6, 1);
2568         if (rc) {
2569                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2570                 goto out_proto;
2571         }
2572
2573         rc = sock_register(&smc_sock_family_ops);
2574         if (rc) {
2575                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
2576                 goto out_proto6;
2577         }
2578         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2579         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
2580
2581         rc = smc_ib_register_client();
2582         if (rc) {
2583                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2584                 goto out_sock;
2585         }
2586
2587         static_branch_enable(&tcp_have_smc);
2588         return 0;
2589
2590 out_sock:
2591         sock_unregister(PF_SMC);
2592 out_proto6:
2593         proto_unregister(&smc_proto6);
2594 out_proto:
2595         proto_unregister(&smc_proto);
2596 out_core:
2597         smc_core_exit();
2598 out_alloc_wqs:
2599         destroy_workqueue(smc_close_wq);
2600 out_alloc_hs_wq:
2601         destroy_workqueue(smc_hs_wq);
2602 out_pnet:
2603         smc_pnet_exit();
2604 out_pernet_subsys:
2605         unregister_pernet_subsys(&smc_net_ops);
2606
2607         return rc;
2608 }
2609
2610 static void __exit smc_exit(void)
2611 {
2612         static_branch_disable(&tcp_have_smc);
2613         sock_unregister(PF_SMC);
2614         smc_core_exit();
2615         smc_ib_unregister_client();
2616         destroy_workqueue(smc_close_wq);
2617         destroy_workqueue(smc_hs_wq);
2618         proto_unregister(&smc_proto6);
2619         proto_unregister(&smc_proto);
2620         smc_pnet_exit();
2621         unregister_pernet_subsys(&smc_net_ops);
2622         rcu_barrier();
2623 }
2624
2625 module_init(smc_init);
2626 module_exit(smc_exit);
2627
2628 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2629 MODULE_DESCRIPTION("smc socket address family");
2630 MODULE_LICENSE("GPL");
2631 MODULE_ALIAS_NETPROTO(PF_SMC);