net/smc/af_smc.c

   1 /*
   2  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   3  *
   4  *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
   5  *  applies to SOCK_STREAM sockets only
   6  *  offers an alternative communication option for TCP-protocol sockets
   7  *  applicable with RoCE-cards only
   8  *
   9  *  Initial restrictions:
  10  *    - support for alternate links postponed
  11  *
  12  *  Copyright IBM Corp. 2016, 2018
  13  *
  14  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  15  *              based on prototype from Frank Blaschka
  16  */
  17
  18 #define KMSG_COMPONENT "smc"
  19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  20
  21 #include <linux/module.h>
  22 #include <linux/socket.h>
  23 #include <linux/workqueue.h>
  24 #include <linux/in.h>
  25 #include <linux/sched/signal.h>
  26 #include <linux/if_vlan.h>
  27
  28 #include <net/sock.h>
  29 #include <net/tcp.h>
  30 #include <net/smc.h>
  31 #include <asm/ioctls.h>
  32
  33 #include "smc.h"
  34 #include "smc_clc.h"
  35 #include "smc_llc.h"
  36 #include "smc_cdc.h"
  37 #include "smc_core.h"
  38 #include "smc_ib.h"
  39 #include "smc_ism.h"
  40 #include "smc_pnet.h"
  41 #include "smc_tx.h"
  42 #include "smc_rx.h"
  43 #include "smc_close.h"
  44
  45 static DEFINE_MUTEX(smc_create_lgr_pending);    /* serialize link group
  46                                                  * creation
  47                                                  */
  48
  49 static void smc_tcp_listen_work(struct work_struct *);
  50 static void smc_connect_work(struct work_struct *);
  51
  52 static void smc_set_keepalive(struct sock *sk, int val)
  53 {
  54         struct smc_sock *smc = smc_sk(sk);
  55
  56         smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
  57 }
  58
  59 static struct smc_hashinfo smc_v4_hashinfo = {
  60         .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
  61 };
  62
  63 static struct smc_hashinfo smc_v6_hashinfo = {
  64         .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
  65 };
  66
  67 int smc_hash_sk(struct sock *sk)
  68 {
  69         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  70         struct hlist_head *head;
  71
  72         head = &h->ht;
  73
  74         write_lock_bh(&h->lock);
  75         sk_add_node(sk, head);
  76         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  77         write_unlock_bh(&h->lock);
  78
  79         return 0;
  80 }
  81 EXPORT_SYMBOL_GPL(smc_hash_sk);
  82
  83 void smc_unhash_sk(struct sock *sk)
  84 {
  85         struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
  86
  87         write_lock_bh(&h->lock);
  88         if (sk_del_node_init(sk))
  89                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
  90         write_unlock_bh(&h->lock);
  91 }
  92 EXPORT_SYMBOL_GPL(smc_unhash_sk);
  93
  94 struct proto smc_proto = {
  95         .name           = "SMC",
  96         .owner          = THIS_MODULE,
  97         .keepalive      = smc_set_keepalive,
  98         .hash           = smc_hash_sk,
  99         .unhash         = smc_unhash_sk,
 100         .obj_size       = sizeof(struct smc_sock),
 101         .h.smc_hash     = &smc_v4_hashinfo,
 102         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 103 };
 104 EXPORT_SYMBOL_GPL(smc_proto);
 105
 106 struct proto smc_proto6 = {
 107         .name           = "SMC6",
 108         .owner          = THIS_MODULE,
 109         .keepalive      = smc_set_keepalive,
 110         .hash           = smc_hash_sk,
 111         .unhash         = smc_unhash_sk,
 112         .obj_size       = sizeof(struct smc_sock),
 113         .h.smc_hash     = &smc_v6_hashinfo,
 114         .slab_flags     = SLAB_TYPESAFE_BY_RCU,
 115 };
 116 EXPORT_SYMBOL_GPL(smc_proto6);
 117
 118 static int smc_release(struct socket *sock)
 119 {
 120         struct sock *sk = sock->sk;
 121         struct smc_sock *smc;
 122         int rc = 0;
 123
 124         if (!sk)
 125                 goto out;
 126
 127         smc = smc_sk(sk);
 128
 129         /* cleanup for a dangling non-blocking connect */
 130         flush_work(&smc->connect_work);
 131         kfree(smc->connect_info);
 132         smc->connect_info = NULL;
 133
 134         if (sk->sk_state == SMC_LISTEN)
 135                 /* smc_close_non_accepted() is called and acquires
 136                  * sock lock for child sockets again
 137                  */
 138                 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 139         else
 140                 lock_sock(sk);
 141
 142         if (!smc->use_fallback) {
 143                 rc = smc_close_active(smc);
 144                 sock_set_flag(sk, SOCK_DEAD);
 145                 sk->sk_shutdown |= SHUTDOWN_MASK;
 146         }
 147
 148         sk->sk_prot->unhash(sk);
 149
 150         if (smc->clcsock) {
 151                 if (smc->use_fallback && sk->sk_state == SMC_LISTEN) {
 152                         /* wake up clcsock accept */
 153                         rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
 154                 }
 155                 mutex_lock(&smc->clcsock_release_lock);
 156                 sock_release(smc->clcsock);
 157                 smc->clcsock = NULL;
 158                 mutex_unlock(&smc->clcsock_release_lock);
 159         }
 160         if (smc->use_fallback) {
 161                 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
 162                         sock_put(sk); /* passive closing */
 163                 sk->sk_state = SMC_CLOSED;
 164                 sk->sk_state_change(sk);
 165         }
 166
 167         /* detach socket */
 168         sock_orphan(sk);
 169         sock->sk = NULL;
 170         if (!smc->use_fallback && sk->sk_state == SMC_CLOSED)
 171                 smc_conn_free(&smc->conn);
 172         release_sock(sk);
 173
 174         sock_put(sk); /* final sock_put */
 175 out:
 176         return rc;
 177 }
 178
 179 static void smc_destruct(struct sock *sk)
 180 {
 181         if (sk->sk_state != SMC_CLOSED)
 182                 return;
 183         if (!sock_flag(sk, SOCK_DEAD))
 184                 return;
 185
 186         sk_refcnt_debug_dec(sk);
 187 }
 188
 189 static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 190                                    int protocol)
 191 {
 192         struct smc_sock *smc;
 193         struct proto *prot;
 194         struct sock *sk;
 195
 196         prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
 197         sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
 198         if (!sk)
 199                 return NULL;
 200
 201         sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
 202         sk->sk_state = SMC_INIT;
 203         sk->sk_destruct = smc_destruct;
 204         sk->sk_protocol = protocol;
 205         smc = smc_sk(sk);
 206         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 207         INIT_WORK(&smc->connect_work, smc_connect_work);
 208         INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
 209         INIT_LIST_HEAD(&smc->accept_q);
 210         spin_lock_init(&smc->accept_q_lock);
 211         spin_lock_init(&smc->conn.send_lock);
 212         sk->sk_prot->hash(sk);
 213         sk_refcnt_debug_inc(sk);
 214         mutex_init(&smc->clcsock_release_lock);
 215
 216         return sk;
 217 }
 218
 219 static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 220                     int addr_len)
 221 {
 222         struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
 223         struct sock *sk = sock->sk;
 224         struct smc_sock *smc;
 225         int rc;
 226
 227         smc = smc_sk(sk);
 228
 229         /* replicate tests from inet_bind(), to be safe wrt. future changes */
 230         rc = -EINVAL;
 231         if (addr_len < sizeof(struct sockaddr_in))
 232                 goto out;
 233
 234         rc = -EAFNOSUPPORT;
 235         if (addr->sin_family != AF_INET &&
 236             addr->sin_family != AF_INET6 &&
 237             addr->sin_family != AF_UNSPEC)
 238                 goto out;
 239         /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
 240         if (addr->sin_family == AF_UNSPEC &&
 241             addr->sin_addr.s_addr != htonl(INADDR_ANY))
 242                 goto out;
 243
 244         lock_sock(sk);
 245
 246         /* Check if socket is already active */
 247         rc = -EINVAL;
 248         if (sk->sk_state != SMC_INIT)
 249                 goto out_rel;
 250
 251         smc->clcsock->sk->sk_reuse = sk->sk_reuse;
 252         rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 253
 254 out_rel:
 255         release_sock(sk);
 256 out:
 257         return rc;
 258 }
 259
 260 static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
 261                                    unsigned long mask)
 262 {
 263         /* options we don't get control via setsockopt for */
 264         nsk->sk_type = osk->sk_type;
 265         nsk->sk_sndbuf = osk->sk_sndbuf;
 266         nsk->sk_rcvbuf = osk->sk_rcvbuf;
 267         nsk->sk_sndtimeo = osk->sk_sndtimeo;
 268         nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
 269         nsk->sk_mark = osk->sk_mark;
 270         nsk->sk_priority = osk->sk_priority;
 271         nsk->sk_rcvlowat = osk->sk_rcvlowat;
 272         nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
 273         nsk->sk_err = osk->sk_err;
 274
 275         nsk->sk_flags &= ~mask;
 276         nsk->sk_flags |= osk->sk_flags & mask;
 277 }
 278
 279 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
 280                              (1UL << SOCK_KEEPOPEN) | \
 281                              (1UL << SOCK_LINGER) | \
 282                              (1UL << SOCK_BROADCAST) | \
 283                              (1UL << SOCK_TIMESTAMP) | \
 284                              (1UL << SOCK_DBG) | \
 285                              (1UL << SOCK_RCVTSTAMP) | \
 286                              (1UL << SOCK_RCVTSTAMPNS) | \
 287                              (1UL << SOCK_LOCALROUTE) | \
 288                              (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
 289                              (1UL << SOCK_RXQ_OVFL) | \
 290                              (1UL << SOCK_WIFI_STATUS) | \
 291                              (1UL << SOCK_NOFCS) | \
 292                              (1UL << SOCK_FILTER_LOCKED))
 293 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
 294  * clc socket (since smc is not called for these options from net/core)
 295  */
 296 static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
 297 {
 298         smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
 299 }
 300
 301 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
 302                              (1UL << SOCK_KEEPOPEN) | \
 303                              (1UL << SOCK_LINGER) | \
 304                              (1UL << SOCK_DBG))
 305 /* copy only settings and flags relevant for smc from clc to smc socket */
 306 static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
 307 {
 308         smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
 309 }
 310
 311 /* register a new rmb, optionally send confirm_rkey msg to register with peer */
 312 static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
 313                        bool conf_rkey)
 314 {
 315         /* register memory region for new rmb */
 316         if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
 317                 rmb_desc->regerr = 1;
 318                 return -EFAULT;
 319         }
 320         if (!conf_rkey)
 321                 return 0;
 322         /* exchange confirm_rkey msg with peer */
 323         if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
 324                 rmb_desc->regerr = 1;
 325                 return -EFAULT;
 326         }
 327         return 0;
 328 }
 329
 330 static int smc_clnt_conf_first_link(struct smc_sock *smc)
 331 {
 332         struct net *net = sock_net(smc->clcsock->sk);
 333         struct smc_link_group *lgr = smc->conn.lgr;
 334         struct smc_link *link;
 335         int rest;
 336         int rc;
 337
 338         link = &lgr->lnk[SMC_SINGLE_LINK];
 339         /* receive CONFIRM LINK request from server over RoCE fabric */
 340         rest = wait_for_completion_interruptible_timeout(
 341                 &link->llc_confirm,
 342                 SMC_LLC_WAIT_FIRST_TIME);
 343         if (rest <= 0) {
 344                 struct smc_clc_msg_decline dclc;
 345
 346                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 347                                       SMC_CLC_DECLINE);
 348                 return rc;
 349         }
 350
 351         if (link->llc_confirm_rc)
 352                 return SMC_CLC_DECL_RMBE_EC;
 353
 354         rc = smc_ib_modify_qp_rts(link);
 355         if (rc)
 356                 return SMC_CLC_DECL_ERR_RDYLNK;
 357
 358         smc_wr_remember_qp_attr(link);
 359
 360         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
 361                 return SMC_CLC_DECL_ERR_REGRMB;
 362
 363         /* send CONFIRM LINK response over RoCE fabric */
 364         rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
 365         if (rc < 0)
 366                 return SMC_CLC_DECL_TIMEOUT_CL;
 367
 368         /* receive ADD LINK request from server over RoCE fabric */
 369         rest = wait_for_completion_interruptible_timeout(&link->llc_add,
 370                                                          SMC_LLC_WAIT_TIME);
 371         if (rest <= 0) {
 372                 struct smc_clc_msg_decline dclc;
 373
 374                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 375                                       SMC_CLC_DECLINE);
 376                 return rc;
 377         }
 378
 379         /* send add link reject message, only one link supported for now */
 380         rc = smc_llc_send_add_link(link,
 381                                    link->smcibdev->mac[link->ibport - 1],
 382                                    link->gid, SMC_LLC_RESP);
 383         if (rc < 0)
 384                 return SMC_CLC_DECL_TIMEOUT_AL;
 385
 386         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
 387
 388         return 0;
 389 }
 390
 391 static void smcr_conn_save_peer_info(struct smc_sock *smc,
 392                                      struct smc_clc_msg_accept_confirm *clc)
 393 {
 394         int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
 395
 396         smc->conn.peer_rmbe_idx = clc->rmbe_idx;
 397         smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
 398         smc->conn.peer_rmbe_size = bufsize;
 399         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 400         smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
 401 }
 402
 403 static void smcd_conn_save_peer_info(struct smc_sock *smc,
 404                                      struct smc_clc_msg_accept_confirm *clc)
 405 {
 406         int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
 407
 408         smc->conn.peer_rmbe_idx = clc->dmbe_idx;
 409         smc->conn.peer_token = clc->token;
 410         /* msg header takes up space in the buffer */
 411         smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
 412         atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
 413         smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
 414 }
 415
 416 static void smc_conn_save_peer_info(struct smc_sock *smc,
 417                                     struct smc_clc_msg_accept_confirm *clc)
 418 {
 419         if (smc->conn.lgr->is_smcd)
 420                 smcd_conn_save_peer_info(smc, clc);
 421         else
 422                 smcr_conn_save_peer_info(smc, clc);
 423 }
 424
 425 static void smc_link_save_peer_info(struct smc_link *link,
 426                                     struct smc_clc_msg_accept_confirm *clc)
 427 {
 428         link->peer_qpn = ntoh24(clc->qpn);
 429         memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
 430         memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
 431         link->peer_psn = ntoh24(clc->psn);
 432         link->peer_mtu = clc->qp_mtu;
 433 }
 434
 435 /* fall back during connect */
 436 static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
 437 {
 438         smc->use_fallback = true;
 439         smc->fallback_rsn = reason_code;
 440         smc_copy_sock_settings_to_clc(smc);
 441         if (smc->sk.sk_state == SMC_INIT)
 442                 smc->sk.sk_state = SMC_ACTIVE;
 443         return 0;
 444 }
 445
 446 /* decline and fall back during connect */
 447 static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
 448 {
 449         int rc;
 450
 451         if (reason_code < 0) { /* error, fallback is not possible */
 452                 if (smc->sk.sk_state == SMC_INIT)
 453                         sock_put(&smc->sk); /* passive closing */
 454                 return reason_code;
 455         }
 456         if (reason_code != SMC_CLC_DECL_PEERDECL) {
 457                 rc = smc_clc_send_decline(smc, reason_code);
 458                 if (rc < 0) {
 459                         if (smc->sk.sk_state == SMC_INIT)
 460                                 sock_put(&smc->sk); /* passive closing */
 461                         return rc;
 462                 }
 463         }
 464         return smc_connect_fallback(smc, reason_code);
 465 }
 466
 467 /* abort connecting */
 468 static int smc_connect_abort(struct smc_sock *smc, int reason_code,
 469                              int local_contact)
 470 {
 471         if (local_contact == SMC_FIRST_CONTACT)
 472                 smc_lgr_forget(smc->conn.lgr);
 473         mutex_unlock(&smc_create_lgr_pending);
 474         smc_conn_free(&smc->conn);
 475         return reason_code;
 476 }
 477
 478 /* check if there is a rdma device available for this connection. */
 479 /* called for connect and listen */
 480 static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
 481                           u8 *ibport, unsigned short vlan_id, u8 gid[])
 482 {
 483         int reason_code = 0;
 484
 485         /* PNET table look up: search active ib_device and port
 486          * within same PNETID that also contains the ethernet device
 487          * used for the internal TCP socket
 488          */
 489         smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport, vlan_id,
 490                                     gid);
 491         if (!(*ibdev))
 492                 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
 493
 494         return reason_code;
 495 }
 496
 497 /* check if there is an ISM device available for this connection. */
 498 /* called for connect and listen */
 499 static int smc_check_ism(struct smc_sock *smc, struct smcd_dev **ismdev)
 500 {
 501         /* Find ISM device with same PNETID as connecting interface  */
 502         smc_pnet_find_ism_resource(smc->clcsock->sk, ismdev);
 503         if (!(*ismdev))
 504                 return SMC_CLC_DECL_CNFERR; /* configuration error */
 505         return 0;
 506 }
 507
 508 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
 509 static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
 510                                       struct smcd_dev *ismdev,
 511                                       unsigned short vlan_id)
 512 {
 513         if (vlan_id && smc_ism_get_vlan(ismdev, vlan_id))
 514                 return SMC_CLC_DECL_CNFERR;
 515         return 0;
 516 }
 517
 518 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 519  * used, the VLAN ID will be registered again during the connection setup.
 520  */
 521 static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
 522                                         struct smcd_dev *ismdev,
 523                                         unsigned short vlan_id)
 524 {
 525         if (!is_smcd)
 526                 return 0;
 527         if (vlan_id && smc_ism_put_vlan(ismdev, vlan_id))
 528                 return SMC_CLC_DECL_CNFERR;
 529         return 0;
 530 }
 531
 532 /* CLC handshake during connect */
 533 static int smc_connect_clc(struct smc_sock *smc, int smc_type,
 534                            struct smc_clc_msg_accept_confirm *aclc,
 535                            struct smc_ib_device *ibdev, u8 ibport,
 536                            u8 gid[], struct smcd_dev *ismdev)
 537 {
 538         int rc = 0;
 539
 540         /* do inband token exchange */
 541         rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, gid, ismdev);
 542         if (rc)
 543                 return rc;
 544         /* receive SMC Accept CLC message */
 545         return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT);
 546 }
 547
 548 /* setup for RDMA connection of client */
 549 static int smc_connect_rdma(struct smc_sock *smc,
 550                             struct smc_clc_msg_accept_confirm *aclc,
 551                             struct smc_ib_device *ibdev, u8 ibport)
 552 {
 553         int local_contact = SMC_FIRST_CONTACT;
 554         struct smc_link *link;
 555         int reason_code = 0;
 556
 557         mutex_lock(&smc_create_lgr_pending);
 558         local_contact = smc_conn_create(smc, false, aclc->hdr.flag, ibdev,
 559                                         ibport, &aclc->lcl, NULL, 0);
 560         if (local_contact < 0) {
 561                 if (local_contact == -ENOMEM)
 562                         reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
 563                 else if (local_contact == -ENOLINK)
 564                         reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
 565                 else
 566                         reason_code = SMC_CLC_DECL_INTERR; /* other error */
 567                 return smc_connect_abort(smc, reason_code, 0);
 568         }
 569         link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
 570
 571         smc_conn_save_peer_info(smc, aclc);
 572
 573         /* create send buffer and rmb */
 574         if (smc_buf_create(smc, false))
 575                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
 576
 577         if (local_contact == SMC_FIRST_CONTACT)
 578                 smc_link_save_peer_info(link, aclc);
 579
 580         if (smc_rmb_rtoken_handling(&smc->conn, aclc))
 581                 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
 582                                          local_contact);
 583
 584         smc_close_init(smc);
 585         smc_rx_init(smc);
 586
 587         if (local_contact == SMC_FIRST_CONTACT) {
 588                 if (smc_ib_ready_link(link))
 589                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
 590                                                  local_contact);
 591         } else {
 592                 if (!smc->conn.rmb_desc->reused &&
 593                     smc_reg_rmb(link, smc->conn.rmb_desc, true))
 594                         return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
 595                                                  local_contact);
 596         }
 597         smc_rmb_sync_sg_for_device(&smc->conn);
 598
 599         reason_code = smc_clc_send_confirm(smc);
 600         if (reason_code)
 601                 return smc_connect_abort(smc, reason_code, local_contact);
 602
 603         smc_tx_init(smc);
 604
 605         if (local_contact == SMC_FIRST_CONTACT) {
 606                 /* QP confirmation over RoCE fabric */
 607                 reason_code = smc_clnt_conf_first_link(smc);
 608                 if (reason_code)
 609                         return smc_connect_abort(smc, reason_code,
 610                                                  local_contact);
 611         }
 612         mutex_unlock(&smc_create_lgr_pending);
 613
 614         smc_copy_sock_settings_to_clc(smc);
 615         if (smc->sk.sk_state == SMC_INIT)
 616                 smc->sk.sk_state = SMC_ACTIVE;
 617
 618         return 0;
 619 }
 620
 621 /* setup for ISM connection of client */
 622 static int smc_connect_ism(struct smc_sock *smc,
 623                            struct smc_clc_msg_accept_confirm *aclc,
 624                            struct smcd_dev *ismdev)
 625 {
 626         int local_contact = SMC_FIRST_CONTACT;
 627         int rc = 0;
 628
 629         mutex_lock(&smc_create_lgr_pending);
 630         local_contact = smc_conn_create(smc, true, aclc->hdr.flag, NULL, 0,
 631                                         NULL, ismdev, aclc->gid);
 632         if (local_contact < 0)
 633                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, 0);
 634
 635         /* Create send and receive buffers */
 636         if (smc_buf_create(smc, true))
 637                 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
 638
 639         smc_conn_save_peer_info(smc, aclc);
 640         smc_close_init(smc);
 641         smc_rx_init(smc);
 642         smc_tx_init(smc);
 643
 644         rc = smc_clc_send_confirm(smc);
 645         if (rc)
 646                 return smc_connect_abort(smc, rc, local_contact);
 647         mutex_unlock(&smc_create_lgr_pending);
 648
 649         smc_copy_sock_settings_to_clc(smc);
 650         if (smc->sk.sk_state == SMC_INIT)
 651                 smc->sk.sk_state = SMC_ACTIVE;
 652
 653         return 0;
 654 }
 655
 656 /* perform steps before actually connecting */
 657 static int __smc_connect(struct smc_sock *smc)
 658 {
 659         bool ism_supported = false, rdma_supported = false;
 660         struct smc_clc_msg_accept_confirm aclc;
 661         struct smc_ib_device *ibdev;
 662         struct smcd_dev *ismdev;
 663         u8 gid[SMC_GID_SIZE];
 664         unsigned short vlan;
 665         int smc_type;
 666         int rc = 0;
 667         u8 ibport;
 668
 669         sock_hold(&smc->sk); /* sock put in passive closing */
 670
 671         if (smc->use_fallback)
 672                 return smc_connect_fallback(smc, smc->fallback_rsn);
 673
 674         /* if peer has not signalled SMC-capability, fall back */
 675         if (!tcp_sk(smc->clcsock->sk)->syn_smc)
 676                 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
 677
 678         /* IPSec connections opt out of SMC-R optimizations */
 679         if (using_ipsec(smc))
 680                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
 681
 682         /* check for VLAN ID */
 683         if (smc_vlan_by_tcpsk(smc->clcsock, &vlan))
 684                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);
 685
 686         /* check if there is an ism device available */
 687         if (!smc_check_ism(smc, &ismdev) &&
 688             !smc_connect_ism_vlan_setup(smc, ismdev, vlan)) {
 689                 /* ISM is supported for this connection */
 690                 ism_supported = true;
 691                 smc_type = SMC_TYPE_D;
 692         }
 693
 694         /* check if there is a rdma device available */
 695         if (!smc_check_rdma(smc, &ibdev, &ibport, vlan, gid)) {
 696                 /* RDMA is supported for this connection */
 697                 rdma_supported = true;
 698                 if (ism_supported)
 699                         smc_type = SMC_TYPE_B; /* both */
 700                 else
 701                         smc_type = SMC_TYPE_R; /* only RDMA */
 702         }
 703
 704         /* if neither ISM nor RDMA are supported, fallback */
 705         if (!rdma_supported && !ism_supported)
 706                 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
 707
 708         /* perform CLC handshake */
 709         rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, gid, ismdev);
 710         if (rc) {
 711                 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
 712                 return smc_connect_decline_fallback(smc, rc);
 713         }
 714
 715         /* depending on previous steps, connect using rdma or ism */
 716         if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
 717                 rc = smc_connect_rdma(smc, &aclc, ibdev, ibport);
 718         else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
 719                 rc = smc_connect_ism(smc, &aclc, ismdev);
 720         else
 721                 rc = SMC_CLC_DECL_MODEUNSUPP;
 722         if (rc) {
 723                 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
 724                 return smc_connect_decline_fallback(smc, rc);
 725         }
 726
 727         smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
 728         return 0;
 729 }
 730
 731 static void smc_connect_work(struct work_struct *work)
 732 {
 733         struct smc_sock *smc = container_of(work, struct smc_sock,
 734                                             connect_work);
 735         int rc;
 736
 737         lock_sock(&smc->sk);
 738         rc = kernel_connect(smc->clcsock, &smc->connect_info->addr,
 739                             smc->connect_info->alen, smc->connect_info->flags);
 740         if (smc->clcsock->sk->sk_err) {
 741                 smc->sk.sk_err = smc->clcsock->sk->sk_err;
 742                 goto out;
 743         }
 744         if (rc < 0) {
 745                 smc->sk.sk_err = -rc;
 746                 goto out;
 747         }
 748
 749         rc = __smc_connect(smc);
 750         if (rc < 0)
 751                 smc->sk.sk_err = -rc;
 752
 753 out:
 754         if (smc->sk.sk_err)
 755                 smc->sk.sk_state_change(&smc->sk);
 756         else
 757                 smc->sk.sk_write_space(&smc->sk);
 758         kfree(smc->connect_info);
 759         smc->connect_info = NULL;
 760         release_sock(&smc->sk);
 761 }
 762
 763 static int smc_connect(struct socket *sock, struct sockaddr *addr,
 764                        int alen, int flags)
 765 {
 766         struct sock *sk = sock->sk;
 767         struct smc_sock *smc;
 768         int rc = -EINVAL;
 769
 770         smc = smc_sk(sk);
 771
 772         /* separate smc parameter checking to be safe */
 773         if (alen < sizeof(addr->sa_family))
 774                 goto out_err;
 775         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
 776                 goto out_err;
 777
 778         lock_sock(sk);
 779         switch (sk->sk_state) {
 780         default:
 781                 goto out;
 782         case SMC_ACTIVE:
 783                 rc = -EISCONN;
 784                 goto out;
 785         case SMC_INIT:
 786                 rc = 0;
 787                 break;
 788         }
 789
 790         smc_copy_sock_settings_to_clc(smc);
 791         tcp_sk(smc->clcsock->sk)->syn_smc = 1;
 792         if (flags & O_NONBLOCK) {
 793                 if (smc->connect_info) {
 794                         rc = -EALREADY;
 795                         goto out;
 796                 }
 797                 smc->connect_info = kzalloc(alen + 2 * sizeof(int), GFP_KERNEL);
 798                 if (!smc->connect_info) {
 799                         rc = -ENOMEM;
 800                         goto out;
 801                 }
 802                 smc->connect_info->alen = alen;
 803                 smc->connect_info->flags = flags ^ O_NONBLOCK;
 804                 memcpy(&smc->connect_info->addr, addr, alen);
 805                 schedule_work(&smc->connect_work);
 806                 rc = -EINPROGRESS;
 807         } else {
 808                 rc = kernel_connect(smc->clcsock, addr, alen, flags);
 809                 if (rc)
 810                         goto out;
 811
 812                 rc = __smc_connect(smc);
 813                 if (rc < 0)
 814                         goto out;
 815                 else
 816                         rc = 0; /* success cases including fallback */
 817         }
 818
 819 out:
 820         release_sock(sk);
 821 out_err:
 822         return rc;
 823 }
 824
 825 static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
 826 {
 827         struct socket *new_clcsock = NULL;
 828         struct sock *lsk = &lsmc->sk;
 829         struct sock *new_sk;
 830         int rc = -EINVAL;
 831
 832         release_sock(lsk);
 833         new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
 834         if (!new_sk) {
 835                 rc = -ENOMEM;
 836                 lsk->sk_err = ENOMEM;
 837                 *new_smc = NULL;
 838                 lock_sock(lsk);
 839                 goto out;
 840         }
 841         *new_smc = smc_sk(new_sk);
 842
 843         mutex_lock(&lsmc->clcsock_release_lock);
 844         if (lsmc->clcsock)
 845                 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
 846         mutex_unlock(&lsmc->clcsock_release_lock);
 847         lock_sock(lsk);
 848         if  (rc < 0)
 849                 lsk->sk_err = -rc;
 850         if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
 851                 new_sk->sk_prot->unhash(new_sk);
 852                 if (new_clcsock)
 853                         sock_release(new_clcsock);
 854                 new_sk->sk_state = SMC_CLOSED;
 855                 sock_set_flag(new_sk, SOCK_DEAD);
 856                 sock_put(new_sk); /* final */
 857                 *new_smc = NULL;
 858                 goto out;
 859         }
 860
 861         (*new_smc)->clcsock = new_clcsock;
 862 out:
 863         return rc;
 864 }
 865
 866 /* add a just created sock to the accept queue of the listen sock as
 867  * candidate for a following socket accept call from user space
 868  */
 869 static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
 870 {
 871         struct smc_sock *par = smc_sk(parent);
 872
 873         sock_hold(sk); /* sock_put in smc_accept_unlink () */
 874         spin_lock(&par->accept_q_lock);
 875         list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
 876         spin_unlock(&par->accept_q_lock);
 877         sk_acceptq_added(parent);
 878 }
 879
 880 /* remove a socket from the accept queue of its parental listening socket */
 881 static void smc_accept_unlink(struct sock *sk)
 882 {
 883         struct smc_sock *par = smc_sk(sk)->listen_smc;
 884
 885         spin_lock(&par->accept_q_lock);
 886         list_del_init(&smc_sk(sk)->accept_q);
 887         spin_unlock(&par->accept_q_lock);
 888         sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
 889         sock_put(sk); /* sock_hold in smc_accept_enqueue */
 890 }
 891
 892 /* remove a sock from the accept queue to bind it to a new socket created
 893  * for a socket accept call from user space
 894  */
 895 struct sock *smc_accept_dequeue(struct sock *parent,
 896                                 struct socket *new_sock)
 897 {
 898         struct smc_sock *isk, *n;
 899         struct sock *new_sk;
 900
 901         list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
 902                 new_sk = (struct sock *)isk;
 903
 904                 smc_accept_unlink(new_sk);
 905                 if (new_sk->sk_state == SMC_CLOSED) {
 906                         new_sk->sk_prot->unhash(new_sk);
 907                         if (isk->clcsock) {
 908                                 sock_release(isk->clcsock);
 909                                 isk->clcsock = NULL;
 910                         }
 911                         sock_put(new_sk); /* final */
 912                         continue;
 913                 }
 914                 if (new_sock)
 915                         sock_graft(new_sk, new_sock);
 916                 return new_sk;
 917         }
 918         return NULL;
 919 }
 920
 921 /* clean up for a created but never accepted sock */
 922 void smc_close_non_accepted(struct sock *sk)
 923 {
 924         struct smc_sock *smc = smc_sk(sk);
 925
 926         lock_sock(sk);
 927         if (!sk->sk_lingertime)
 928                 /* wait for peer closing */
 929                 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
 930         if (!smc->use_fallback) {
 931                 smc_close_active(smc);
 932                 sock_set_flag(sk, SOCK_DEAD);
 933                 sk->sk_shutdown |= SHUTDOWN_MASK;
 934         }
 935         sk->sk_prot->unhash(sk);
 936         if (smc->clcsock) {
 937                 struct socket *tcp;
 938
 939                 tcp = smc->clcsock;
 940                 smc->clcsock = NULL;
 941                 sock_release(tcp);
 942         }
 943         if (smc->use_fallback) {
 944                 sock_put(sk); /* passive closing */
 945                 sk->sk_state = SMC_CLOSED;
 946         } else {
 947                 if (sk->sk_state == SMC_CLOSED)
 948                         smc_conn_free(&smc->conn);
 949         }
 950         release_sock(sk);
 951         sock_put(sk); /* final sock_put */
 952 }
 953
 954 static int smc_serv_conf_first_link(struct smc_sock *smc)
 955 {
 956         struct net *net = sock_net(smc->clcsock->sk);
 957         struct smc_link_group *lgr = smc->conn.lgr;
 958         struct smc_link *link;
 959         int rest;
 960         int rc;
 961
 962         link = &lgr->lnk[SMC_SINGLE_LINK];
 963
 964         if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
 965                 return SMC_CLC_DECL_ERR_REGRMB;
 966
 967         /* send CONFIRM LINK request to client over the RoCE fabric */
 968         rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
 969         if (rc < 0)
 970                 return SMC_CLC_DECL_TIMEOUT_CL;
 971
 972         /* receive CONFIRM LINK response from client over the RoCE fabric */
 973         rest = wait_for_completion_interruptible_timeout(
 974                 &link->llc_confirm_resp,
 975                 SMC_LLC_WAIT_FIRST_TIME);
 976         if (rest <= 0) {
 977                 struct smc_clc_msg_decline dclc;
 978
 979                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
 980                                       SMC_CLC_DECLINE);
 981                 return rc;
 982         }
 983
 984         if (link->llc_confirm_resp_rc)
 985                 return SMC_CLC_DECL_RMBE_EC;
 986
 987         /* send ADD LINK request to client over the RoCE fabric */
 988         rc = smc_llc_send_add_link(link,
 989                                    link->smcibdev->mac[link->ibport - 1],
 990                                    link->gid, SMC_LLC_REQ);
 991         if (rc < 0)
 992                 return SMC_CLC_DECL_TIMEOUT_AL;
 993
 994         /* receive ADD LINK response from client over the RoCE fabric */
 995         rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
 996                                                          SMC_LLC_WAIT_TIME);
 997         if (rest <= 0) {
 998                 struct smc_clc_msg_decline dclc;
 999
1000                 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1001                                       SMC_CLC_DECLINE);
1002                 return rc;
1003         }
1004
1005         smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
1006
1007         return 0;
1008 }
1009
1010 /* listen worker: finish */
1011 static void smc_listen_out(struct smc_sock *new_smc)
1012 {
1013         struct smc_sock *lsmc = new_smc->listen_smc;
1014         struct sock *newsmcsk = &new_smc->sk;
1015
1016         lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1017         if (lsmc->sk.sk_state == SMC_LISTEN) {
1018                 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1019         } else { /* no longer listening */
1020                 smc_close_non_accepted(newsmcsk);
1021         }
1022         release_sock(&lsmc->sk);
1023
1024         /* Wake up accept */
1025         lsmc->sk.sk_data_ready(&lsmc->sk);
1026         sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1027 }
1028
1029 /* listen worker: finish in state connected */
1030 static void smc_listen_out_connected(struct smc_sock *new_smc)
1031 {
1032         struct sock *newsmcsk = &new_smc->sk;
1033
1034         sk_refcnt_debug_inc(newsmcsk);
1035         if (newsmcsk->sk_state == SMC_INIT)
1036                 newsmcsk->sk_state = SMC_ACTIVE;
1037
1038         smc_listen_out(new_smc);
1039 }
1040
1041 /* listen worker: finish in error state */
1042 static void smc_listen_out_err(struct smc_sock *new_smc)
1043 {
1044         struct sock *newsmcsk = &new_smc->sk;
1045
1046         if (newsmcsk->sk_state == SMC_INIT)
1047                 sock_put(&new_smc->sk); /* passive closing */
1048         newsmcsk->sk_state = SMC_CLOSED;
1049         smc_conn_free(&new_smc->conn);
1050
1051         smc_listen_out(new_smc);
1052 }
1053
1054 /* listen worker: decline and fall back if possible */
1055 static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1056                                int local_contact)
1057 {
1058         /* RDMA setup failed, switch back to TCP */
1059         if (local_contact == SMC_FIRST_CONTACT)
1060                 smc_lgr_forget(new_smc->conn.lgr);
1061         if (reason_code < 0) { /* error, no fallback possible */
1062                 smc_listen_out_err(new_smc);
1063                 return;
1064         }
1065         smc_conn_free(&new_smc->conn);
1066         new_smc->use_fallback = true;
1067         new_smc->fallback_rsn = reason_code;
1068         if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1069                 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1070                         smc_listen_out_err(new_smc);
1071                         return;
1072                 }
1073         }
1074         smc_listen_out_connected(new_smc);
1075 }
1076
1077 /* listen worker: check prefixes */
1078 static int smc_listen_rdma_check(struct smc_sock *new_smc,
1079                                  struct smc_clc_msg_proposal *pclc)
1080 {
1081         struct smc_clc_msg_proposal_prefix *pclc_prfx;
1082         struct socket *newclcsock = new_smc->clcsock;
1083
1084         pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1085         if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1086                 return SMC_CLC_DECL_CNFERR;
1087
1088         return 0;
1089 }
1090
1091 /* listen worker: initialize connection and buffers */
1092 static int smc_listen_rdma_init(struct smc_sock *new_smc,
1093                                 struct smc_clc_msg_proposal *pclc,
1094                                 struct smc_ib_device *ibdev, u8 ibport,
1095                                 int *local_contact)
1096 {
1097         /* allocate connection / link group */
1098         *local_contact = smc_conn_create(new_smc, false, 0, ibdev, ibport,
1099                                          &pclc->lcl, NULL, 0);
1100         if (*local_contact < 0) {
1101                 if (*local_contact == -ENOMEM)
1102                         return SMC_CLC_DECL_MEM;/* insufficient memory*/
1103                 return SMC_CLC_DECL_INTERR; /* other error */
1104         }
1105
1106         /* create send buffer and rmb */
1107         if (smc_buf_create(new_smc, false))
1108                 return SMC_CLC_DECL_MEM;
1109
1110         return 0;
1111 }
1112
1113 /* listen worker: initialize connection and buffers for SMC-D */
1114 static int smc_listen_ism_init(struct smc_sock *new_smc,
1115                                struct smc_clc_msg_proposal *pclc,
1116                                struct smcd_dev *ismdev,
1117                                int *local_contact)
1118 {
1119         struct smc_clc_msg_smcd *pclc_smcd;
1120
1121         pclc_smcd = smc_get_clc_msg_smcd(pclc);
1122         *local_contact = smc_conn_create(new_smc, true, 0, NULL, 0, NULL,
1123                                          ismdev, pclc_smcd->gid);
1124         if (*local_contact < 0) {
1125                 if (*local_contact == -ENOMEM)
1126                         return SMC_CLC_DECL_MEM;/* insufficient memory*/
1127                 return SMC_CLC_DECL_INTERR; /* other error */
1128         }
1129
1130         /* Check if peer can be reached via ISM device */
1131         if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1132                             new_smc->conn.lgr->vlan_id,
1133                             new_smc->conn.lgr->smcd)) {
1134                 if (*local_contact == SMC_FIRST_CONTACT)
1135                         smc_lgr_forget(new_smc->conn.lgr);
1136                 smc_conn_free(&new_smc->conn);
1137                 return SMC_CLC_DECL_CNFERR;
1138         }
1139
1140         /* Create send and receive buffers */
1141         if (smc_buf_create(new_smc, true)) {
1142                 if (*local_contact == SMC_FIRST_CONTACT)
1143                         smc_lgr_forget(new_smc->conn.lgr);
1144                 smc_conn_free(&new_smc->conn);
1145                 return SMC_CLC_DECL_MEM;
1146         }
1147
1148         return 0;
1149 }
1150
1151 /* listen worker: register buffers */
1152 static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1153 {
1154         struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1155
1156         if (local_contact != SMC_FIRST_CONTACT) {
1157                 if (!new_smc->conn.rmb_desc->reused) {
1158                         if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
1159                                 return SMC_CLC_DECL_ERR_REGRMB;
1160                 }
1161         }
1162         smc_rmb_sync_sg_for_device(&new_smc->conn);
1163
1164         return 0;
1165 }
1166
1167 /* listen worker: finish RDMA setup */
1168 static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1169                                   struct smc_clc_msg_accept_confirm *cclc,
1170                                   int local_contact)
1171 {
1172         struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1173         int reason_code = 0;
1174
1175         if (local_contact == SMC_FIRST_CONTACT)
1176                 smc_link_save_peer_info(link, cclc);
1177
1178         if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
1179                 reason_code = SMC_CLC_DECL_ERR_RTOK;
1180                 goto decline;
1181         }
1182
1183         if (local_contact == SMC_FIRST_CONTACT) {
1184                 if (smc_ib_ready_link(link)) {
1185                         reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1186                         goto decline;
1187                 }
1188                 /* QP confirmation over RoCE fabric */
1189                 reason_code = smc_serv_conf_first_link(new_smc);
1190                 if (reason_code)
1191                         goto decline;
1192         }
1193         return 0;
1194
1195 decline:
1196         mutex_unlock(&smc_create_lgr_pending);
1197         smc_listen_decline(new_smc, reason_code, local_contact);
1198         return reason_code;
1199 }
1200
1201 /* setup for RDMA connection of server */
1202 static void smc_listen_work(struct work_struct *work)
1203 {
1204         struct smc_sock *new_smc = container_of(work, struct smc_sock,
1205                                                 smc_listen_work);
1206         struct socket *newclcsock = new_smc->clcsock;
1207         struct smc_clc_msg_accept_confirm cclc;
1208         struct smc_clc_msg_proposal *pclc;
1209         struct smc_ib_device *ibdev;
1210         bool ism_supported = false;
1211         struct smcd_dev *ismdev;
1212         u8 buf[SMC_CLC_MAX_LEN];
1213         int local_contact = 0;
1214         unsigned short vlan;
1215         int reason_code = 0;
1216         int rc = 0;
1217         u8 ibport;
1218
1219         if (new_smc->use_fallback) {
1220                 smc_listen_out_connected(new_smc);
1221                 return;
1222         }
1223
1224         /* check if peer is smc capable */
1225         if (!tcp_sk(newclcsock->sk)->syn_smc) {
1226                 new_smc->use_fallback = true;
1227                 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1228                 smc_listen_out_connected(new_smc);
1229                 return;
1230         }
1231
1232         /* do inband token exchange -
1233          * wait for and receive SMC Proposal CLC message
1234          */
1235         pclc = (struct smc_clc_msg_proposal *)&buf;
1236         reason_code = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1237                                        SMC_CLC_PROPOSAL);
1238         if (reason_code) {
1239                 smc_listen_decline(new_smc, reason_code, 0);
1240                 return;
1241         }
1242
1243         /* IPSec connections opt out of SMC-R optimizations */
1244         if (using_ipsec(new_smc)) {
1245                 smc_listen_decline(new_smc, SMC_CLC_DECL_IPSEC, 0);
1246                 return;
1247         }
1248
1249         mutex_lock(&smc_create_lgr_pending);
1250         smc_close_init(new_smc);
1251         smc_rx_init(new_smc);
1252         smc_tx_init(new_smc);
1253
1254         /* check if ISM is available */
1255         if ((pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) &&
1256             !smc_check_ism(new_smc, &ismdev) &&
1257             !smc_listen_ism_init(new_smc, pclc, ismdev, &local_contact)) {
1258                 ism_supported = true;
1259         }
1260
1261         /* check if RDMA is available */
1262         if (!ism_supported &&
1263             ((pclc->hdr.path != SMC_TYPE_R && pclc->hdr.path != SMC_TYPE_B) ||
1264              smc_vlan_by_tcpsk(new_smc->clcsock, &vlan) ||
1265              smc_check_rdma(new_smc, &ibdev, &ibport, vlan, NULL) ||
1266              smc_listen_rdma_check(new_smc, pclc) ||
1267              smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
1268                                   &local_contact) ||
1269              smc_listen_rdma_reg(new_smc, local_contact))) {
1270                 /* SMC not supported, decline */
1271                 mutex_unlock(&smc_create_lgr_pending);
1272                 smc_listen_decline(new_smc, SMC_CLC_DECL_MODEUNSUPP,
1273                                    local_contact);
1274                 return;
1275         }
1276
1277         /* send SMC Accept CLC message */
1278         rc = smc_clc_send_accept(new_smc, local_contact);
1279         if (rc) {
1280                 mutex_unlock(&smc_create_lgr_pending);
1281                 smc_listen_decline(new_smc, rc, local_contact);
1282                 return;
1283         }
1284
1285         /* receive SMC Confirm CLC message */
1286         reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1287                                        SMC_CLC_CONFIRM);
1288         if (reason_code) {
1289                 mutex_unlock(&smc_create_lgr_pending);
1290                 smc_listen_decline(new_smc, reason_code, local_contact);
1291                 return;
1292         }
1293
1294         /* finish worker */
1295         if (!ism_supported) {
1296                 if (smc_listen_rdma_finish(new_smc, &cclc, local_contact))
1297                         return;
1298         }
1299         smc_conn_save_peer_info(new_smc, &cclc);
1300         mutex_unlock(&smc_create_lgr_pending);
1301         smc_listen_out_connected(new_smc);
1302 }
1303
1304 static void smc_tcp_listen_work(struct work_struct *work)
1305 {
1306         struct smc_sock *lsmc = container_of(work, struct smc_sock,
1307                                              tcp_listen_work);
1308         struct sock *lsk = &lsmc->sk;
1309         struct smc_sock *new_smc;
1310         int rc = 0;
1311
1312         lock_sock(lsk);
1313         while (lsk->sk_state == SMC_LISTEN) {
1314                 rc = smc_clcsock_accept(lsmc, &new_smc);
1315                 if (rc)
1316                         goto out;
1317                 if (!new_smc)
1318                         continue;
1319
1320                 new_smc->listen_smc = lsmc;
1321                 new_smc->use_fallback = lsmc->use_fallback;
1322                 new_smc->fallback_rsn = lsmc->fallback_rsn;
1323                 sock_hold(lsk); /* sock_put in smc_listen_work */
1324                 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1325                 smc_copy_sock_settings_to_smc(new_smc);
1326                 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1327                 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1328                 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1329                 if (!schedule_work(&new_smc->smc_listen_work))
1330                         sock_put(&new_smc->sk);
1331         }
1332
1333 out:
1334         release_sock(lsk);
1335         sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1336 }
1337
1338 static int smc_listen(struct socket *sock, int backlog)
1339 {
1340         struct sock *sk = sock->sk;
1341         struct smc_sock *smc;
1342         int rc;
1343
1344         smc = smc_sk(sk);
1345         lock_sock(sk);
1346
1347         rc = -EINVAL;
1348         if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
1349                 goto out;
1350
1351         rc = 0;
1352         if (sk->sk_state == SMC_LISTEN) {
1353                 sk->sk_max_ack_backlog = backlog;
1354                 goto out;
1355         }
1356         /* some socket options are handled in core, so we could not apply
1357          * them to the clc socket -- copy smc socket options to clc socket
1358          */
1359         smc_copy_sock_settings_to_clc(smc);
1360         if (!smc->use_fallback)
1361                 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1362
1363         rc = kernel_listen(smc->clcsock, backlog);
1364         if (rc)
1365                 goto out;
1366         sk->sk_max_ack_backlog = backlog;
1367         sk->sk_ack_backlog = 0;
1368         sk->sk_state = SMC_LISTEN;
1369         INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
1370         sock_hold(sk); /* sock_hold in tcp_listen_worker */
1371         if (!schedule_work(&smc->tcp_listen_work))
1372                 sock_put(sk);
1373
1374 out:
1375         release_sock(sk);
1376         return rc;
1377 }
1378
1379 static int smc_accept(struct socket *sock, struct socket *new_sock,
1380                       int flags, bool kern)
1381 {
1382         struct sock *sk = sock->sk, *nsk;
1383         DECLARE_WAITQUEUE(wait, current);
1384         struct smc_sock *lsmc;
1385         long timeo;
1386         int rc = 0;
1387
1388         lsmc = smc_sk(sk);
1389         sock_hold(sk); /* sock_put below */
1390         lock_sock(sk);
1391
1392         if (lsmc->sk.sk_state != SMC_LISTEN) {
1393                 rc = -EINVAL;
1394                 release_sock(sk);
1395                 goto out;
1396         }
1397
1398         /* Wait for an incoming connection */
1399         timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1400         add_wait_queue_exclusive(sk_sleep(sk), &wait);
1401         while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1402                 set_current_state(TASK_INTERRUPTIBLE);
1403                 if (!timeo) {
1404                         rc = -EAGAIN;
1405                         break;
1406                 }
1407                 release_sock(sk);
1408                 timeo = schedule_timeout(timeo);
1409                 /* wakeup by sk_data_ready in smc_listen_work() */
1410                 sched_annotate_sleep();
1411                 lock_sock(sk);
1412                 if (signal_pending(current)) {
1413                         rc = sock_intr_errno(timeo);
1414                         break;
1415                 }
1416         }
1417         set_current_state(TASK_RUNNING);
1418         remove_wait_queue(sk_sleep(sk), &wait);
1419
1420         if (!rc)
1421                 rc = sock_error(nsk);
1422         release_sock(sk);
1423         if (rc)
1424                 goto out;
1425
1426         if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1427                 /* wait till data arrives on the socket */
1428                 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1429                                                                 MSEC_PER_SEC);
1430                 if (smc_sk(nsk)->use_fallback) {
1431                         struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1432
1433                         lock_sock(clcsk);
1434                         if (skb_queue_empty(&clcsk->sk_receive_queue))
1435                                 sk_wait_data(clcsk, &timeo, NULL);
1436                         release_sock(clcsk);
1437                 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1438                         lock_sock(nsk);
1439                         smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1440                         release_sock(nsk);
1441                 }
1442         }
1443
1444 out:
1445         sock_put(sk); /* sock_hold above */
1446         return rc;
1447 }
1448
1449 static int smc_getname(struct socket *sock, struct sockaddr *addr,
1450                        int peer)
1451 {
1452         struct smc_sock *smc;
1453
1454         if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1455             (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
1456                 return -ENOTCONN;
1457
1458         smc = smc_sk(sock->sk);
1459
1460         return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
1461 }
1462
1463 static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1464 {
1465         struct sock *sk = sock->sk;
1466         struct smc_sock *smc;
1467         int rc = -EPIPE;
1468
1469         smc = smc_sk(sk);
1470         lock_sock(sk);
1471         if ((sk->sk_state != SMC_ACTIVE) &&
1472             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1473             (sk->sk_state != SMC_INIT))
1474                 goto out;
1475
1476         if (msg->msg_flags & MSG_FASTOPEN) {
1477                 if (sk->sk_state == SMC_INIT) {
1478                         smc->use_fallback = true;
1479                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1480                 } else {
1481                         rc = -EINVAL;
1482                         goto out;
1483                 }
1484         }
1485
1486         if (smc->use_fallback)
1487                 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1488         else
1489                 rc = smc_tx_sendmsg(smc, msg, len);
1490 out:
1491         release_sock(sk);
1492         return rc;
1493 }
1494
1495 static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1496                        int flags)
1497 {
1498         struct sock *sk = sock->sk;
1499         struct smc_sock *smc;
1500         int rc = -ENOTCONN;
1501
1502         smc = smc_sk(sk);
1503         lock_sock(sk);
1504         if ((sk->sk_state == SMC_INIT) ||
1505             (sk->sk_state == SMC_LISTEN) ||
1506             (sk->sk_state == SMC_CLOSED))
1507                 goto out;
1508
1509         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1510                 rc = 0;
1511                 goto out;
1512         }
1513
1514         if (smc->use_fallback) {
1515                 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
1516         } else {
1517                 msg->msg_namelen = 0;
1518                 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1519         }
1520
1521 out:
1522         release_sock(sk);
1523         return rc;
1524 }
1525
1526 static __poll_t smc_accept_poll(struct sock *parent)
1527 {
1528         struct smc_sock *isk = smc_sk(parent);
1529         __poll_t mask = 0;
1530
1531         spin_lock(&isk->accept_q_lock);
1532         if (!list_empty(&isk->accept_q))
1533                 mask = EPOLLIN | EPOLLRDNORM;
1534         spin_unlock(&isk->accept_q_lock);
1535
1536         return mask;
1537 }
1538
1539 static __poll_t smc_poll(struct file *file, struct socket *sock,
1540                              poll_table *wait)
1541 {
1542         struct sock *sk = sock->sk;
1543         __poll_t mask = 0;
1544         struct smc_sock *smc;
1545
1546         if (!sk)
1547                 return EPOLLNVAL;
1548
1549         smc = smc_sk(sock->sk);
1550         if (smc->use_fallback) {
1551                 /* delegate to CLC child sock */
1552                 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
1553                 sk->sk_err = smc->clcsock->sk->sk_err;
1554                 if (sk->sk_err)
1555                         mask |= EPOLLERR;
1556         } else {
1557                 if (sk->sk_state != SMC_CLOSED)
1558                         sock_poll_wait(file, sock, wait);
1559                 if (sk->sk_err)
1560                         mask |= EPOLLERR;
1561                 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1562                     (sk->sk_state == SMC_CLOSED))
1563                         mask |= EPOLLHUP;
1564                 if (sk->sk_state == SMC_LISTEN) {
1565                         /* woken up by sk_data_ready in smc_listen_work() */
1566                         mask = smc_accept_poll(sk);
1567                 } else {
1568                         if (atomic_read(&smc->conn.sndbuf_space) ||
1569                             sk->sk_shutdown & SEND_SHUTDOWN) {
1570                                 mask |= EPOLLOUT | EPOLLWRNORM;
1571                         } else {
1572                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1573                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1574                         }
1575                         if (atomic_read(&smc->conn.bytes_to_rcv))
1576                                 mask |= EPOLLIN | EPOLLRDNORM;
1577                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1578                                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1579                         if (sk->sk_state == SMC_APPCLOSEWAIT1)
1580                                 mask |= EPOLLIN;
1581                         if (smc->conn.urg_state == SMC_URG_VALID)
1582                                 mask |= EPOLLPRI;
1583                 }
1584         }
1585
1586         return mask;
1587 }
1588
1589 static int smc_shutdown(struct socket *sock, int how)
1590 {
1591         struct sock *sk = sock->sk;
1592         bool do_shutdown = true;
1593         struct smc_sock *smc;
1594         int rc = -EINVAL;
1595         int old_state;
1596         int rc1 = 0;
1597
1598         smc = smc_sk(sk);
1599
1600         if ((how < SHUT_RD) || (how > SHUT_RDWR))
1601                 return rc;
1602
1603         lock_sock(sk);
1604
1605         rc = -ENOTCONN;
1606         if ((sk->sk_state != SMC_ACTIVE) &&
1607             (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1608             (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1609             (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1610             (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1611             (sk->sk_state != SMC_APPFINCLOSEWAIT))
1612                 goto out;
1613         if (smc->use_fallback) {
1614                 rc = kernel_sock_shutdown(smc->clcsock, how);
1615                 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1616                 if (sk->sk_shutdown == SHUTDOWN_MASK)
1617                         sk->sk_state = SMC_CLOSED;
1618                 goto out;
1619         }
1620         switch (how) {
1621         case SHUT_RDWR:         /* shutdown in both directions */
1622                 old_state = sk->sk_state;
1623                 rc = smc_close_active(smc);
1624                 if (old_state == SMC_ACTIVE &&
1625                     sk->sk_state == SMC_PEERCLOSEWAIT1)
1626                         do_shutdown = false;
1627                 break;
1628         case SHUT_WR:
1629                 rc = smc_close_shutdown_write(smc);
1630                 break;
1631         case SHUT_RD:
1632                 rc = 0;
1633                 /* nothing more to do because peer is not involved */
1634                 break;
1635         }
1636         if (do_shutdown && smc->clcsock)
1637                 rc1 = kernel_sock_shutdown(smc->clcsock, how);
1638         /* map sock_shutdown_cmd constants to sk_shutdown value range */
1639         sk->sk_shutdown |= how + 1;
1640
1641 out:
1642         release_sock(sk);
1643         return rc ? rc : rc1;
1644 }
1645
1646 static int smc_setsockopt(struct socket *sock, int level, int optname,
1647                           char __user *optval, unsigned int optlen)
1648 {
1649         struct sock *sk = sock->sk;
1650         struct smc_sock *smc;
1651         int val, rc;
1652
1653         if (level == SOL_TCP && optname == TCP_ULP)
1654                 return -EOPNOTSUPP;
1655
1656         smc = smc_sk(sk);
1657
1658         /* generic setsockopts reaching us here always apply to the
1659          * CLC socket
1660          */
1661         rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1662                                            optval, optlen);
1663         if (smc->clcsock->sk->sk_err) {
1664                 sk->sk_err = smc->clcsock->sk->sk_err;
1665                 sk->sk_error_report(sk);
1666         }
1667         if (rc)
1668                 return rc;
1669
1670         if (optlen < sizeof(int))
1671                 return -EINVAL;
1672         if (get_user(val, (int __user *)optval))
1673                 return -EFAULT;
1674
1675         lock_sock(sk);
1676         switch (optname) {
1677         case TCP_FASTOPEN:
1678         case TCP_FASTOPEN_CONNECT:
1679         case TCP_FASTOPEN_KEY:
1680         case TCP_FASTOPEN_NO_COOKIE:
1681                 /* option not supported by SMC */
1682                 if (sk->sk_state == SMC_INIT) {
1683                         smc->use_fallback = true;
1684                         smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1685                 } else {
1686                         if (!smc->use_fallback)
1687                                 rc = -EINVAL;
1688                 }
1689                 break;
1690         case TCP_NODELAY:
1691                 if (sk->sk_state != SMC_INIT &&
1692                     sk->sk_state != SMC_LISTEN &&
1693                     sk->sk_state != SMC_CLOSED) {
1694                         if (val && !smc->use_fallback)
1695                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1696                                                  0);
1697                 }
1698                 break;
1699         case TCP_CORK:
1700                 if (sk->sk_state != SMC_INIT &&
1701                     sk->sk_state != SMC_LISTEN &&
1702                     sk->sk_state != SMC_CLOSED) {
1703                         if (!val && !smc->use_fallback)
1704                                 mod_delayed_work(system_wq, &smc->conn.tx_work,
1705                                                  0);
1706                 }
1707                 break;
1708         case TCP_DEFER_ACCEPT:
1709                 smc->sockopt_defer_accept = val;
1710                 break;
1711         default:
1712                 break;
1713         }
1714         release_sock(sk);
1715
1716         return rc;
1717 }
1718
1719 static int smc_getsockopt(struct socket *sock, int level, int optname,
1720                           char __user *optval, int __user *optlen)
1721 {
1722         struct smc_sock *smc;
1723
1724         smc = smc_sk(sock->sk);
1725         /* socket options apply to the CLC socket */
1726         return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1727                                              optval, optlen);
1728 }
1729
1730 static int smc_ioctl(struct socket *sock, unsigned int cmd,
1731                      unsigned long arg)
1732 {
1733         union smc_host_cursor cons, urg;
1734         struct smc_connection *conn;
1735         struct smc_sock *smc;
1736         int answ;
1737
1738         smc = smc_sk(sock->sk);
1739         conn = &smc->conn;
1740         lock_sock(&smc->sk);
1741         if (smc->use_fallback) {
1742                 if (!smc->clcsock) {
1743                         release_sock(&smc->sk);
1744                         return -EBADF;
1745                 }
1746                 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1747                 release_sock(&smc->sk);
1748                 return answ;
1749         }
1750         switch (cmd) {
1751         case SIOCINQ: /* same as FIONREAD */
1752                 if (smc->sk.sk_state == SMC_LISTEN) {
1753                         release_sock(&smc->sk);
1754                         return -EINVAL;
1755                 }
1756                 if (smc->sk.sk_state == SMC_INIT ||
1757                     smc->sk.sk_state == SMC_CLOSED)
1758                         answ = 0;
1759                 else
1760                         answ = atomic_read(&smc->conn.bytes_to_rcv);
1761                 break;
1762         case SIOCOUTQ:
1763                 /* output queue size (not send + not acked) */
1764                 if (smc->sk.sk_state == SMC_LISTEN) {
1765                         release_sock(&smc->sk);
1766                         return -EINVAL;
1767                 }
1768                 if (smc->sk.sk_state == SMC_INIT ||
1769                     smc->sk.sk_state == SMC_CLOSED)
1770                         answ = 0;
1771                 else
1772                         answ = smc->conn.sndbuf_desc->len -
1773                                         atomic_read(&smc->conn.sndbuf_space);
1774                 break;
1775         case SIOCOUTQNSD:
1776                 /* output queue size (not send only) */
1777                 if (smc->sk.sk_state == SMC_LISTEN) {
1778                         release_sock(&smc->sk);
1779                         return -EINVAL;
1780                 }
1781                 if (smc->sk.sk_state == SMC_INIT ||
1782                     smc->sk.sk_state == SMC_CLOSED)
1783                         answ = 0;
1784                 else
1785                         answ = smc_tx_prepared_sends(&smc->conn);
1786                 break;
1787         case SIOCATMARK:
1788                 if (smc->sk.sk_state == SMC_LISTEN) {
1789                         release_sock(&smc->sk);
1790                         return -EINVAL;
1791                 }
1792                 if (smc->sk.sk_state == SMC_INIT ||
1793                     smc->sk.sk_state == SMC_CLOSED) {
1794                         answ = 0;
1795                 } else {
1796                         smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1797                         smc_curs_copy(&urg, &conn->urg_curs, conn);
1798                         answ = smc_curs_diff(conn->rmb_desc->len,
1799                                              &cons, &urg) == 1;
1800                 }
1801                 break;
1802         default:
1803                 release_sock(&smc->sk);
1804                 return -ENOIOCTLCMD;
1805         }
1806         release_sock(&smc->sk);
1807
1808         return put_user(answ, (int __user *)arg);
1809 }
1810
1811 static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1812                             int offset, size_t size, int flags)
1813 {
1814         struct sock *sk = sock->sk;
1815         struct smc_sock *smc;
1816         int rc = -EPIPE;
1817
1818         smc = smc_sk(sk);
1819         lock_sock(sk);
1820         if (sk->sk_state != SMC_ACTIVE) {
1821                 release_sock(sk);
1822                 goto out;
1823         }
1824         release_sock(sk);
1825         if (smc->use_fallback)
1826                 rc = kernel_sendpage(smc->clcsock, page, offset,
1827                                      size, flags);
1828         else
1829                 rc = sock_no_sendpage(sock, page, offset, size, flags);
1830
1831 out:
1832         return rc;
1833 }
1834
1835 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1836  * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1837  * updates till whenever a respective page has been fully processed.
1838  * Note that subsequent recv() calls have to wait till all splice() processing
1839  * completed.
1840  */
1841 static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1842                                struct pipe_inode_info *pipe, size_t len,
1843                                unsigned int flags)
1844 {
1845         struct sock *sk = sock->sk;
1846         struct smc_sock *smc;
1847         int rc = -ENOTCONN;
1848
1849         smc = smc_sk(sk);
1850         lock_sock(sk);
1851
1852         if (sk->sk_state == SMC_INIT ||
1853             sk->sk_state == SMC_LISTEN ||
1854             sk->sk_state == SMC_CLOSED)
1855                 goto out;
1856
1857         if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1858                 rc = 0;
1859                 goto out;
1860         }
1861
1862         if (smc->use_fallback) {
1863                 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1864                                                     pipe, len, flags);
1865         } else {
1866                 if (*ppos) {
1867                         rc = -ESPIPE;
1868                         goto out;
1869                 }
1870                 if (flags & SPLICE_F_NONBLOCK)
1871                         flags = MSG_DONTWAIT;
1872                 else
1873                         flags = 0;
1874                 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
1875         }
1876 out:
1877         release_sock(sk);
1878
1879         return rc;
1880 }
1881
1882 /* must look like tcp */
1883 static const struct proto_ops smc_sock_ops = {
1884         .family         = PF_SMC,
1885         .owner          = THIS_MODULE,
1886         .release        = smc_release,
1887         .bind           = smc_bind,
1888         .connect        = smc_connect,
1889         .socketpair     = sock_no_socketpair,
1890         .accept         = smc_accept,
1891         .getname        = smc_getname,
1892         .poll           = smc_poll,
1893         .ioctl          = smc_ioctl,
1894         .listen         = smc_listen,
1895         .shutdown       = smc_shutdown,
1896         .setsockopt     = smc_setsockopt,
1897         .getsockopt     = smc_getsockopt,
1898         .sendmsg        = smc_sendmsg,
1899         .recvmsg        = smc_recvmsg,
1900         .mmap           = sock_no_mmap,
1901         .sendpage       = smc_sendpage,
1902         .splice_read    = smc_splice_read,
1903 };
1904
1905 static int smc_create(struct net *net, struct socket *sock, int protocol,
1906                       int kern)
1907 {
1908         int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
1909         struct smc_sock *smc;
1910         struct sock *sk;
1911         int rc;
1912
1913         rc = -ESOCKTNOSUPPORT;
1914         if (sock->type != SOCK_STREAM)
1915                 goto out;
1916
1917         rc = -EPROTONOSUPPORT;
1918         if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
1919                 goto out;
1920
1921         rc = -ENOBUFS;
1922         sock->ops = &smc_sock_ops;
1923         sk = smc_sock_alloc(net, sock, protocol);
1924         if (!sk)
1925                 goto out;
1926
1927         /* create internal TCP socket for CLC handshake and fallback */
1928         smc = smc_sk(sk);
1929         smc->use_fallback = false; /* assume rdma capability first */
1930         smc->fallback_rsn = 0;
1931         rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1932                               &smc->clcsock);
1933         if (rc) {
1934                 sk_common_release(sk);
1935                 goto out;
1936         }
1937         smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1938         smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
1939
1940 out:
1941         return rc;
1942 }
1943
1944 static const struct net_proto_family smc_sock_family_ops = {
1945         .family = PF_SMC,
1946         .owner  = THIS_MODULE,
1947         .create = smc_create,
1948 };
1949
1950 static int __init smc_init(void)
1951 {
1952         int rc;
1953
1954         rc = smc_pnet_init();
1955         if (rc)
1956                 return rc;
1957
1958         rc = smc_llc_init();
1959         if (rc) {
1960                 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
1961                 goto out_pnet;
1962         }
1963
1964         rc = smc_cdc_init();
1965         if (rc) {
1966                 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
1967                 goto out_pnet;
1968         }
1969
1970         rc = proto_register(&smc_proto, 1);
1971         if (rc) {
1972                 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
1973                 goto out_pnet;
1974         }
1975
1976         rc = proto_register(&smc_proto6, 1);
1977         if (rc) {
1978                 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
1979                 goto out_proto;
1980         }
1981
1982         rc = sock_register(&smc_sock_family_ops);
1983         if (rc) {
1984                 pr_err("%s: sock_register fails with %d\n", __func__, rc);
1985                 goto out_proto6;
1986         }
1987         INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
1988         INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
1989
1990         rc = smc_ib_register_client();
1991         if (rc) {
1992                 pr_err("%s: ib_register fails with %d\n", __func__, rc);
1993                 goto out_sock;
1994         }
1995
1996         static_branch_enable(&tcp_have_smc);
1997         return 0;
1998
1999 out_sock:
2000         sock_unregister(PF_SMC);
2001 out_proto6:
2002         proto_unregister(&smc_proto6);
2003 out_proto:
2004         proto_unregister(&smc_proto);
2005 out_pnet:
2006         smc_pnet_exit();
2007         return rc;
2008 }
2009
2010 static void __exit smc_exit(void)
2011 {
2012         smc_core_exit();
2013         static_branch_disable(&tcp_have_smc);
2014         smc_ib_unregister_client();
2015         sock_unregister(PF_SMC);
2016         proto_unregister(&smc_proto6);
2017         proto_unregister(&smc_proto);
2018         smc_pnet_exit();
2019 }
2020
2021 module_init(smc_init);
2022 module_exit(smc_exit);
2023
2024 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2025 MODULE_DESCRIPTION("smc socket address family");
2026 MODULE_LICENSE("GPL");
2027 MODULE_ALIAS_NETPROTO(PF_SMC);