drivers/crypto/chelsio/chtls/chtls_cm.c

   1 /*
   2  * Copyright (c) 2018 Chelsio Communications, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * Written by: Atul Gupta (atul.gupta@chelsio.com)
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/list.h>
  13 #include <linux/workqueue.h>
  14 #include <linux/skbuff.h>
  15 #include <linux/timer.h>
  16 #include <linux/notifier.h>
  17 #include <linux/inetdevice.h>
  18 #include <linux/ip.h>
  19 #include <linux/tcp.h>
  20 #include <linux/sched/signal.h>
  21 #include <linux/kallsyms.h>
  22 #include <linux/kprobes.h>
  23 #include <linux/if_vlan.h>
  24 #include <net/tcp.h>
  25 #include <net/dst.h>
  26
  27 #include "chtls.h"
  28 #include "chtls_cm.h"
  29
  30 /*
  31  * State transitions and actions for close.  Note that if we are in SYN_SENT
  32  * we remain in that state as we cannot control a connection while it's in
  33  * SYN_SENT; such connections are allowed to establish and are then aborted.
  34  */
  35 static unsigned char new_state[16] = {
  36         /* current state:     new state:      action: */
  37         /* (Invalid)       */ TCP_CLOSE,
  38         /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
  39         /* TCP_SYN_SENT    */ TCP_SYN_SENT,
  40         /* TCP_SYN_RECV    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
  41         /* TCP_FIN_WAIT1   */ TCP_FIN_WAIT1,
  42         /* TCP_FIN_WAIT2   */ TCP_FIN_WAIT2,
  43         /* TCP_TIME_WAIT   */ TCP_CLOSE,
  44         /* TCP_CLOSE       */ TCP_CLOSE,
  45         /* TCP_CLOSE_WAIT  */ TCP_LAST_ACK | TCP_ACTION_FIN,
  46         /* TCP_LAST_ACK    */ TCP_LAST_ACK,
  47         /* TCP_LISTEN      */ TCP_CLOSE,
  48         /* TCP_CLOSING     */ TCP_CLOSING,
  49 };
  50
  51 static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
  52 {
  53         struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
  54
  55         if (!csk)
  56                 return NULL;
  57
  58         csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
  59         if (!csk->txdata_skb_cache) {
  60                 kfree(csk);
  61                 return NULL;
  62         }
  63
  64         kref_init(&csk->kref);
  65         csk->cdev = cdev;
  66         skb_queue_head_init(&csk->txq);
  67         csk->wr_skb_head = NULL;
  68         csk->wr_skb_tail = NULL;
  69         csk->mss = MAX_MSS;
  70         csk->tlshws.ofld = 1;
  71         csk->tlshws.txkey = -1;
  72         csk->tlshws.rxkey = -1;
  73         csk->tlshws.mfs = TLS_MFS;
  74         skb_queue_head_init(&csk->tlshws.sk_recv_queue);
  75         return csk;
  76 }
  77
  78 static void chtls_sock_release(struct kref *ref)
  79 {
  80         struct chtls_sock *csk =
  81                 container_of(ref, struct chtls_sock, kref);
  82
  83         kfree(csk);
  84 }
  85
  86 static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
  87                                             struct sock *sk)
  88 {
  89         struct net_device *ndev = cdev->ports[0];
  90
  91         if (likely(!inet_sk(sk)->inet_rcv_saddr))
  92                 return ndev;
  93
  94         ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
  95         if (!ndev)
  96                 return NULL;
  97
  98         if (is_vlan_dev(ndev))
  99                 return vlan_dev_real_dev(ndev);
 100         return ndev;
 101 }
 102
 103 static void assign_rxopt(struct sock *sk, unsigned int opt)
 104 {
 105         const struct chtls_dev *cdev;
 106         struct chtls_sock *csk;
 107         struct tcp_sock *tp;
 108
 109         csk = rcu_dereference_sk_user_data(sk);
 110         tp = tcp_sk(sk);
 111
 112         cdev = csk->cdev;
 113         tp->tcp_header_len           = sizeof(struct tcphdr);
 114         tp->rx_opt.mss_clamp         = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
 115         tp->mss_cache                = tp->rx_opt.mss_clamp;
 116         tp->rx_opt.tstamp_ok         = TCPOPT_TSTAMP_G(opt);
 117         tp->rx_opt.snd_wscale        = TCPOPT_SACK_G(opt);
 118         tp->rx_opt.wscale_ok         = TCPOPT_WSCALE_OK_G(opt);
 119         SND_WSCALE(tp)               = TCPOPT_SND_WSCALE_G(opt);
 120         if (!tp->rx_opt.wscale_ok)
 121                 tp->rx_opt.rcv_wscale = 0;
 122         if (tp->rx_opt.tstamp_ok) {
 123                 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
 124                 tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
 125         } else if (csk->opt2 & TSTAMPS_EN_F) {
 126                 csk->opt2 &= ~TSTAMPS_EN_F;
 127                 csk->mtu_idx = TCPOPT_MSS_G(opt);
 128         }
 129 }
 130
 131 static void chtls_purge_receive_queue(struct sock *sk)
 132 {
 133         struct sk_buff *skb;
 134
 135         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
 136                 skb_dst_set(skb, (void *)NULL);
 137                 kfree_skb(skb);
 138         }
 139 }
 140
 141 static void chtls_purge_write_queue(struct sock *sk)
 142 {
 143         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 144         struct sk_buff *skb;
 145
 146         while ((skb = __skb_dequeue(&csk->txq))) {
 147                 sk->sk_wmem_queued -= skb->truesize;
 148                 __kfree_skb(skb);
 149         }
 150 }
 151
 152 static void chtls_purge_recv_queue(struct sock *sk)
 153 {
 154         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 155         struct chtls_hws *tlsk = &csk->tlshws;
 156         struct sk_buff *skb;
 157
 158         while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
 159                 skb_dst_set(skb, NULL);
 160                 kfree_skb(skb);
 161         }
 162 }
 163
 164 static void abort_arp_failure(void *handle, struct sk_buff *skb)
 165 {
 166         struct cpl_abort_req *req = cplhdr(skb);
 167         struct chtls_dev *cdev;
 168
 169         cdev = (struct chtls_dev *)handle;
 170         req->cmd = CPL_ABORT_NO_RST;
 171         cxgb4_ofld_send(cdev->lldi->ports[0], skb);
 172 }
 173
 174 static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
 175 {
 176         if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
 177                 __skb_trim(skb, 0);
 178                 refcount_inc(&skb->users);
 179         } else {
 180                 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
 181         }
 182         return skb;
 183 }
 184
 185 static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
 186 {
 187         struct cpl_abort_req *req;
 188         struct chtls_sock *csk;
 189         struct tcp_sock *tp;
 190
 191         csk = rcu_dereference_sk_user_data(sk);
 192         tp = tcp_sk(sk);
 193
 194         if (!skb)
 195                 skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
 196
 197         req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
 198         INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
 199         skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
 200         req->rsvd0 = htonl(tp->snd_nxt);
 201         req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
 202         req->cmd = mode;
 203         t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
 204         send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
 205 }
 206
 207 static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
 208 {
 209         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 210
 211         if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
 212                      !csk->cdev)) {
 213                 if (sk->sk_state == TCP_SYN_RECV)
 214                         csk_set_flag(csk, CSK_RST_ABORTED);
 215                 goto out;
 216         }
 217
 218         if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
 219                 struct tcp_sock *tp = tcp_sk(sk);
 220
 221                 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
 222                         WARN_ONCE(1, "send tx flowc error");
 223                 csk_set_flag(csk, CSK_TX_DATA_SENT);
 224         }
 225
 226         csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
 227         chtls_purge_write_queue(sk);
 228
 229         csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
 230         if (sk->sk_state != TCP_SYN_RECV)
 231                 chtls_send_abort(sk, mode, skb);
 232         else
 233                 goto out;
 234
 235         return;
 236 out:
 237         if (skb)
 238                 kfree_skb(skb);
 239 }
 240
 241 static void release_tcp_port(struct sock *sk)
 242 {
 243         if (inet_csk(sk)->icsk_bind_hash)
 244                 inet_put_port(sk);
 245 }
 246
 247 static void tcp_uncork(struct sock *sk)
 248 {
 249         struct tcp_sock *tp = tcp_sk(sk);
 250
 251         if (tp->nonagle & TCP_NAGLE_CORK) {
 252                 tp->nonagle &= ~TCP_NAGLE_CORK;
 253                 chtls_tcp_push(sk, 0);
 254         }
 255 }
 256
 257 static void chtls_close_conn(struct sock *sk)
 258 {
 259         struct cpl_close_con_req *req;
 260         struct chtls_sock *csk;
 261         struct sk_buff *skb;
 262         unsigned int tid;
 263         unsigned int len;
 264
 265         len = roundup(sizeof(struct cpl_close_con_req), 16);
 266         csk = rcu_dereference_sk_user_data(sk);
 267         tid = csk->tid;
 268
 269         skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
 270         req = (struct cpl_close_con_req *)__skb_put(skb, len);
 271         memset(req, 0, len);
 272         req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
 273                               FW_WR_IMMDLEN_V(sizeof(*req) -
 274                                               sizeof(req->wr)));
 275         req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
 276                                FW_WR_FLOWID_V(tid));
 277
 278         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 279
 280         tcp_uncork(sk);
 281         skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
 282         if (sk->sk_state != TCP_SYN_SENT)
 283                 chtls_push_frames(csk, 1);
 284 }
 285
 286 /*
 287  * Perform a state transition during close and return the actions indicated
 288  * for the transition.  Do not make this function inline, the main reason
 289  * it exists at all is to avoid multiple inlining of tcp_set_state.
 290  */
 291 static int make_close_transition(struct sock *sk)
 292 {
 293         int next = (int)new_state[sk->sk_state];
 294
 295         tcp_set_state(sk, next & TCP_STATE_MASK);
 296         return next & TCP_ACTION_FIN;
 297 }
 298
 299 void chtls_close(struct sock *sk, long timeout)
 300 {
 301         int data_lost, prev_state;
 302         struct chtls_sock *csk;
 303
 304         csk = rcu_dereference_sk_user_data(sk);
 305
 306         lock_sock(sk);
 307         sk->sk_shutdown |= SHUTDOWN_MASK;
 308
 309         data_lost = skb_queue_len(&sk->sk_receive_queue);
 310         data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
 311         chtls_purge_recv_queue(sk);
 312         chtls_purge_receive_queue(sk);
 313
 314         if (sk->sk_state == TCP_CLOSE) {
 315                 goto wait;
 316         } else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
 317                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
 318                 release_tcp_port(sk);
 319                 goto unlock;
 320         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
 321                 sk->sk_prot->disconnect(sk, 0);
 322         } else if (make_close_transition(sk)) {
 323                 chtls_close_conn(sk);
 324         }
 325 wait:
 326         if (timeout)
 327                 sk_stream_wait_close(sk, timeout);
 328
 329 unlock:
 330         prev_state = sk->sk_state;
 331         sock_hold(sk);
 332         sock_orphan(sk);
 333
 334         release_sock(sk);
 335
 336         local_bh_disable();
 337         bh_lock_sock(sk);
 338
 339         if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
 340                 goto out;
 341
 342         if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
 343             !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
 344                 struct sk_buff *skb;
 345
 346                 skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
 347                 if (skb)
 348                         chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
 349         }
 350
 351         if (sk->sk_state == TCP_CLOSE)
 352                 inet_csk_destroy_sock(sk);
 353
 354 out:
 355         bh_unlock_sock(sk);
 356         local_bh_enable();
 357         sock_put(sk);
 358 }
 359
 360 /*
 361  * Wait until a socket enters on of the given states.
 362  */
 363 static int wait_for_states(struct sock *sk, unsigned int states)
 364 {
 365         DECLARE_WAITQUEUE(wait, current);
 366         struct socket_wq _sk_wq;
 367         long current_timeo;
 368         int err = 0;
 369
 370         current_timeo = 200;
 371
 372         /*
 373          * We want this to work even when there's no associated struct socket.
 374          * In that case we provide a temporary wait_queue_head_t.
 375          */
 376         if (!sk->sk_wq) {
 377                 init_waitqueue_head(&_sk_wq.wait);
 378                 _sk_wq.fasync_list = NULL;
 379                 init_rcu_head_on_stack(&_sk_wq.rcu);
 380                 RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
 381         }
 382
 383         add_wait_queue(sk_sleep(sk), &wait);
 384         while (!sk_in_state(sk, states)) {
 385                 if (!current_timeo) {
 386                         err = -EBUSY;
 387                         break;
 388                 }
 389                 if (signal_pending(current)) {
 390                         err = sock_intr_errno(current_timeo);
 391                         break;
 392                 }
 393                 set_current_state(TASK_UNINTERRUPTIBLE);
 394                 release_sock(sk);
 395                 if (!sk_in_state(sk, states))
 396                         current_timeo = schedule_timeout(current_timeo);
 397                 __set_current_state(TASK_RUNNING);
 398                 lock_sock(sk);
 399         }
 400         remove_wait_queue(sk_sleep(sk), &wait);
 401
 402         if (rcu_dereference(sk->sk_wq) == &_sk_wq)
 403                 sk->sk_wq = NULL;
 404         return err;
 405 }
 406
 407 int chtls_disconnect(struct sock *sk, int flags)
 408 {
 409         struct chtls_sock *csk;
 410         struct tcp_sock *tp;
 411         int err;
 412
 413         tp = tcp_sk(sk);
 414         csk = rcu_dereference_sk_user_data(sk);
 415         chtls_purge_recv_queue(sk);
 416         chtls_purge_receive_queue(sk);
 417         chtls_purge_write_queue(sk);
 418
 419         if (sk->sk_state != TCP_CLOSE) {
 420                 sk->sk_err = ECONNRESET;
 421                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
 422                 err = wait_for_states(sk, TCPF_CLOSE);
 423                 if (err)
 424                         return err;
 425         }
 426         chtls_purge_recv_queue(sk);
 427         chtls_purge_receive_queue(sk);
 428         tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
 429         return tcp_disconnect(sk, flags);
 430 }
 431
 432 #define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
 433                                  TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
 434 void chtls_shutdown(struct sock *sk, int how)
 435 {
 436         if ((how & SEND_SHUTDOWN) &&
 437             sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
 438             make_close_transition(sk))
 439                 chtls_close_conn(sk);
 440 }
 441
 442 void chtls_destroy_sock(struct sock *sk)
 443 {
 444         struct chtls_sock *csk;
 445
 446         csk = rcu_dereference_sk_user_data(sk);
 447         chtls_purge_recv_queue(sk);
 448         csk->ulp_mode = ULP_MODE_NONE;
 449         chtls_purge_write_queue(sk);
 450         free_tls_keyid(sk);
 451         kref_put(&csk->kref, chtls_sock_release);
 452         sk->sk_prot = &tcp_prot;
 453         sk->sk_prot->destroy(sk);
 454 }
 455
 456 static void reset_listen_child(struct sock *child)
 457 {
 458         struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
 459         struct sk_buff *skb;
 460
 461         skb = alloc_ctrl_skb(csk->txdata_skb_cache,
 462                              sizeof(struct cpl_abort_req));
 463
 464         chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
 465         sock_orphan(child);
 466         INC_ORPHAN_COUNT(child);
 467         if (child->sk_state == TCP_CLOSE)
 468                 inet_csk_destroy_sock(child);
 469 }
 470
 471 static void chtls_disconnect_acceptq(struct sock *listen_sk)
 472 {
 473         struct request_sock **pprev;
 474
 475         pprev = ACCEPT_QUEUE(listen_sk);
 476         while (*pprev) {
 477                 struct request_sock *req = *pprev;
 478
 479                 if (req->rsk_ops == &chtls_rsk_ops) {
 480                         struct sock *child = req->sk;
 481
 482                         *pprev = req->dl_next;
 483                         sk_acceptq_removed(listen_sk);
 484                         reqsk_put(req);
 485                         sock_hold(child);
 486                         local_bh_disable();
 487                         bh_lock_sock(child);
 488                         release_tcp_port(child);
 489                         reset_listen_child(child);
 490                         bh_unlock_sock(child);
 491                         local_bh_enable();
 492                         sock_put(child);
 493                 } else {
 494                         pprev = &req->dl_next;
 495                 }
 496         }
 497 }
 498
 499 static int listen_hashfn(const struct sock *sk)
 500 {
 501         return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
 502 }
 503
 504 static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
 505                                            struct sock *sk,
 506                                            unsigned int stid)
 507 {
 508         struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL);
 509
 510         if (p) {
 511                 int key = listen_hashfn(sk);
 512
 513                 p->sk = sk;
 514                 p->stid = stid;
 515                 spin_lock(&cdev->listen_lock);
 516                 p->next = cdev->listen_hash_tab[key];
 517                 cdev->listen_hash_tab[key] = p;
 518                 spin_unlock(&cdev->listen_lock);
 519         }
 520         return p;
 521 }
 522
 523 static int listen_hash_find(struct chtls_dev *cdev,
 524                             struct sock *sk)
 525 {
 526         struct listen_info *p;
 527         int stid = -1;
 528         int key;
 529
 530         key = listen_hashfn(sk);
 531
 532         spin_lock(&cdev->listen_lock);
 533         for (p = cdev->listen_hash_tab[key]; p; p = p->next)
 534                 if (p->sk == sk) {
 535                         stid = p->stid;
 536                         break;
 537                 }
 538         spin_unlock(&cdev->listen_lock);
 539         return stid;
 540 }
 541
 542 static int listen_hash_del(struct chtls_dev *cdev,
 543                            struct sock *sk)
 544 {
 545         struct listen_info *p, **prev;
 546         int stid = -1;
 547         int key;
 548
 549         key = listen_hashfn(sk);
 550         prev = &cdev->listen_hash_tab[key];
 551
 552         spin_lock(&cdev->listen_lock);
 553         for (p = *prev; p; prev = &p->next, p = p->next)
 554                 if (p->sk == sk) {
 555                         stid = p->stid;
 556                         *prev = p->next;
 557                         kfree(p);
 558                         break;
 559                 }
 560         spin_unlock(&cdev->listen_lock);
 561         return stid;
 562 }
 563
 564 static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
 565 {
 566         struct request_sock *req;
 567         struct chtls_sock *csk;
 568
 569         csk = rcu_dereference_sk_user_data(child);
 570         req = csk->passive_reap_next;
 571
 572         reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
 573         __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
 574         chtls_reqsk_free(req);
 575         csk->passive_reap_next = NULL;
 576 }
 577
 578 static void chtls_reset_synq(struct listen_ctx *listen_ctx)
 579 {
 580         struct sock *listen_sk = listen_ctx->lsk;
 581
 582         while (!skb_queue_empty(&listen_ctx->synq)) {
 583                 struct chtls_sock *csk =
 584                         container_of((struct synq *)skb_peek
 585                                 (&listen_ctx->synq), struct chtls_sock, synq);
 586                 struct sock *child = csk->sk;
 587
 588                 cleanup_syn_rcv_conn(child, listen_sk);
 589                 sock_hold(child);
 590                 local_bh_disable();
 591                 bh_lock_sock(child);
 592                 release_tcp_port(child);
 593                 reset_listen_child(child);
 594                 bh_unlock_sock(child);
 595                 local_bh_enable();
 596                 sock_put(child);
 597         }
 598 }
 599
 600 int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
 601 {
 602         struct net_device *ndev;
 603         struct listen_ctx *ctx;
 604         struct adapter *adap;
 605         struct port_info *pi;
 606         int stid;
 607         int ret;
 608
 609         if (sk->sk_family != PF_INET)
 610                 return -EAGAIN;
 611
 612         rcu_read_lock();
 613         ndev = chtls_ipv4_netdev(cdev, sk);
 614         rcu_read_unlock();
 615         if (!ndev)
 616                 return -EBADF;
 617
 618         pi = netdev_priv(ndev);
 619         adap = pi->adapter;
 620         if (!(adap->flags & FULL_INIT_DONE))
 621                 return -EBADF;
 622
 623         if (listen_hash_find(cdev, sk) >= 0)   /* already have it */
 624                 return -EADDRINUSE;
 625
 626         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 627         if (!ctx)
 628                 return -ENOMEM;
 629
 630         __module_get(THIS_MODULE);
 631         ctx->lsk = sk;
 632         ctx->cdev = cdev;
 633         ctx->state = T4_LISTEN_START_PENDING;
 634         skb_queue_head_init(&ctx->synq);
 635
 636         stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
 637         if (stid < 0)
 638                 goto free_ctx;
 639
 640         sock_hold(sk);
 641         if (!listen_hash_add(cdev, sk, stid))
 642                 goto free_stid;
 643
 644         ret = cxgb4_create_server(ndev, stid,
 645                                   inet_sk(sk)->inet_rcv_saddr,
 646                                   inet_sk(sk)->inet_sport, 0,
 647                                   cdev->lldi->rxq_ids[0]);
 648         if (ret > 0)
 649                 ret = net_xmit_errno(ret);
 650         if (ret)
 651                 goto del_hash;
 652         return 0;
 653 del_hash:
 654         listen_hash_del(cdev, sk);
 655 free_stid:
 656         cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
 657         sock_put(sk);
 658 free_ctx:
 659         kfree(ctx);
 660         module_put(THIS_MODULE);
 661         return -EBADF;
 662 }
 663
 664 void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
 665 {
 666         struct listen_ctx *listen_ctx;
 667         int stid;
 668
 669         stid = listen_hash_del(cdev, sk);
 670         if (stid < 0)
 671                 return;
 672
 673         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
 674         chtls_reset_synq(listen_ctx);
 675
 676         cxgb4_remove_server(cdev->lldi->ports[0], stid,
 677                             cdev->lldi->rxq_ids[0], 0);
 678         chtls_disconnect_acceptq(sk);
 679 }
 680
 681 static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
 682 {
 683         struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
 684         unsigned int stid = GET_TID(rpl);
 685         struct listen_ctx *listen_ctx;
 686
 687         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
 688         if (!listen_ctx)
 689                 return CPL_RET_BUF_DONE;
 690
 691         if (listen_ctx->state == T4_LISTEN_START_PENDING) {
 692                 listen_ctx->state = T4_LISTEN_STARTED;
 693                 return CPL_RET_BUF_DONE;
 694         }
 695
 696         if (rpl->status != CPL_ERR_NONE) {
 697                 pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
 698                         rpl->status, stid);
 699         } else {
 700                 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
 701                 sock_put(listen_ctx->lsk);
 702                 kfree(listen_ctx);
 703                 module_put(THIS_MODULE);
 704         }
 705         return CPL_RET_BUF_DONE;
 706 }
 707
 708 static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
 709 {
 710         struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
 711         struct listen_ctx *listen_ctx;
 712         unsigned int stid;
 713         void *data;
 714
 715         stid = GET_TID(rpl);
 716         data = lookup_stid(cdev->tids, stid);
 717         listen_ctx = (struct listen_ctx *)data;
 718
 719         if (rpl->status != CPL_ERR_NONE) {
 720                 pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
 721                         rpl->status, stid);
 722         } else {
 723                 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
 724                 sock_put(listen_ctx->lsk);
 725                 kfree(listen_ctx);
 726                 module_put(THIS_MODULE);
 727         }
 728         return CPL_RET_BUF_DONE;
 729 }
 730
 731 static void chtls_purge_wr_queue(struct sock *sk)
 732 {
 733         struct sk_buff *skb;
 734
 735         while ((skb = dequeue_wr(sk)) != NULL)
 736                 kfree_skb(skb);
 737 }
 738
 739 static void chtls_release_resources(struct sock *sk)
 740 {
 741         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 742         struct chtls_dev *cdev = csk->cdev;
 743         unsigned int tid = csk->tid;
 744         struct tid_info *tids;
 745
 746         if (!cdev)
 747                 return;
 748
 749         tids = cdev->tids;
 750         kfree_skb(csk->txdata_skb_cache);
 751         csk->txdata_skb_cache = NULL;
 752
 753         if (csk->wr_credits != csk->wr_max_credits) {
 754                 chtls_purge_wr_queue(sk);
 755                 chtls_reset_wr_list(csk);
 756         }
 757
 758         if (csk->l2t_entry) {
 759                 cxgb4_l2t_release(csk->l2t_entry);
 760                 csk->l2t_entry = NULL;
 761         }
 762
 763         cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
 764         sock_put(sk);
 765 }
 766
 767 static void chtls_conn_done(struct sock *sk)
 768 {
 769         if (sock_flag(sk, SOCK_DEAD))
 770                 chtls_purge_receive_queue(sk);
 771         sk_wakeup_sleepers(sk, 0);
 772         tcp_done(sk);
 773 }
 774
 775 static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
 776 {
 777         /*
 778          * If the server is still open we clean up the child connection,
 779          * otherwise the server already did the clean up as it was purging
 780          * its SYN queue and the skb was just sitting in its backlog.
 781          */
 782         if (likely(parent->sk_state == TCP_LISTEN)) {
 783                 cleanup_syn_rcv_conn(child, parent);
 784                 /* Without the below call to sock_orphan,
 785                  * we leak the socket resource with syn_flood test
 786                  * as inet_csk_destroy_sock will not be called
 787                  * in tcp_done since SOCK_DEAD flag is not set.
 788                  * Kernel handles this differently where new socket is
 789                  * created only after 3 way handshake is done.
 790                  */
 791                 sock_orphan(child);
 792                 percpu_counter_inc((child)->sk_prot->orphan_count);
 793                 chtls_release_resources(child);
 794                 chtls_conn_done(child);
 795         } else {
 796                 if (csk_flag(child, CSK_RST_ABORTED)) {
 797                         chtls_release_resources(child);
 798                         chtls_conn_done(child);
 799                 }
 800         }
 801 }
 802
 803 static void pass_open_abort(struct sock *child, struct sock *parent,
 804                             struct sk_buff *skb)
 805 {
 806         do_abort_syn_rcv(child, parent);
 807         kfree_skb(skb);
 808 }
 809
 810 static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
 811 {
 812         pass_open_abort(skb->sk, lsk, skb);
 813 }
 814
 815 static void chtls_pass_open_arp_failure(struct sock *sk,
 816                                         struct sk_buff *skb)
 817 {
 818         const struct request_sock *oreq;
 819         struct chtls_sock *csk;
 820         struct chtls_dev *cdev;
 821         struct sock *parent;
 822         void *data;
 823
 824         csk = rcu_dereference_sk_user_data(sk);
 825         cdev = csk->cdev;
 826
 827         /*
 828          * If the connection is being aborted due to the parent listening
 829          * socket going away there's nothing to do, the ABORT_REQ will close
 830          * the connection.
 831          */
 832         if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
 833                 kfree_skb(skb);
 834                 return;
 835         }
 836
 837         oreq = csk->passive_reap_next;
 838         data = lookup_stid(cdev->tids, oreq->ts_recent);
 839         parent = ((struct listen_ctx *)data)->lsk;
 840
 841         bh_lock_sock(parent);
 842         if (!sock_owned_by_user(parent)) {
 843                 pass_open_abort(sk, parent, skb);
 844         } else {
 845                 BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
 846                 __sk_add_backlog(parent, skb);
 847         }
 848         bh_unlock_sock(parent);
 849 }
 850
 851 static void chtls_accept_rpl_arp_failure(void *handle,
 852                                          struct sk_buff *skb)
 853 {
 854         struct sock *sk = (struct sock *)handle;
 855
 856         sock_hold(sk);
 857         process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
 858         sock_put(sk);
 859 }
 860
 861 static unsigned int chtls_select_mss(const struct chtls_sock *csk,
 862                                      unsigned int pmtu,
 863                                      struct cpl_pass_accept_req *req)
 864 {
 865         struct chtls_dev *cdev;
 866         struct dst_entry *dst;
 867         unsigned int tcpoptsz;
 868         unsigned int iphdrsz;
 869         unsigned int mtu_idx;
 870         struct tcp_sock *tp;
 871         unsigned int mss;
 872         struct sock *sk;
 873
 874         mss = ntohs(req->tcpopt.mss);
 875         sk = csk->sk;
 876         dst = __sk_dst_get(sk);
 877         cdev = csk->cdev;
 878         tp = tcp_sk(sk);
 879         tcpoptsz = 0;
 880
 881         iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
 882         if (req->tcpopt.tstamp)
 883                 tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
 884
 885         tp->advmss = dst_metric_advmss(dst);
 886         if (USER_MSS(tp) && tp->advmss > USER_MSS(tp))
 887                 tp->advmss = USER_MSS(tp);
 888         if (tp->advmss > pmtu - iphdrsz)
 889                 tp->advmss = pmtu - iphdrsz;
 890         if (mss && tp->advmss > mss)
 891                 tp->advmss = mss;
 892
 893         tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
 894                                             iphdrsz + tcpoptsz,
 895                                             tp->advmss - tcpoptsz,
 896                                             8, &mtu_idx);
 897         tp->advmss -= iphdrsz;
 898
 899         inet_csk(sk)->icsk_pmtu_cookie = pmtu;
 900         return mtu_idx;
 901 }
 902
 903 static unsigned int select_rcv_wnd(struct chtls_sock *csk)
 904 {
 905         unsigned int rcvwnd;
 906         unsigned int wnd;
 907         struct sock *sk;
 908
 909         sk = csk->sk;
 910         wnd = tcp_full_space(sk);
 911
 912         if (wnd < MIN_RCV_WND)
 913                 wnd = MIN_RCV_WND;
 914
 915         rcvwnd = MAX_RCV_WND;
 916
 917         csk_set_flag(csk, CSK_UPDATE_RCV_WND);
 918         return min(wnd, rcvwnd);
 919 }
 920
 921 static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
 922 {
 923         int wscale = 0;
 924
 925         if (space > MAX_RCV_WND)
 926                 space = MAX_RCV_WND;
 927         if (win_clamp && win_clamp < space)
 928                 space = win_clamp;
 929
 930         if (wscale_ok) {
 931                 while (wscale < 14 && (65535 << wscale) < space)
 932                         wscale++;
 933         }
 934         return wscale;
 935 }
 936
 937 static void chtls_pass_accept_rpl(struct sk_buff *skb,
 938                                   struct cpl_pass_accept_req *req,
 939                                   unsigned int tid)
 940
 941 {
 942         struct cpl_t5_pass_accept_rpl *rpl5;
 943         struct cxgb4_lld_info *lldi;
 944         const struct tcphdr *tcph;
 945         const struct tcp_sock *tp;
 946         struct chtls_sock *csk;
 947         unsigned int len;
 948         struct sock *sk;
 949         u32 opt2, hlen;
 950         u64 opt0;
 951
 952         sk = skb->sk;
 953         tp = tcp_sk(sk);
 954         csk = sk->sk_user_data;
 955         csk->tid = tid;
 956         lldi = csk->cdev->lldi;
 957         len = roundup(sizeof(*rpl5), 16);
 958
 959         rpl5 = __skb_put_zero(skb, len);
 960         INIT_TP_WR(rpl5, tid);
 961
 962         OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
 963                                                      csk->tid));
 964         csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
 965                                         req);
 966         opt0 = TCAM_BYPASS_F |
 967                WND_SCALE_V((tp)->rx_opt.rcv_wscale) |
 968                MSS_IDX_V(csk->mtu_idx) |
 969                L2T_IDX_V(csk->l2t_entry->idx) |
 970                NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
 971                TX_CHAN_V(csk->tx_chan) |
 972                SMAC_SEL_V(csk->smac_idx) |
 973                DSCP_V(csk->tos >> 2) |
 974                ULP_MODE_V(ULP_MODE_TLS) |
 975                RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
 976
 977         opt2 = RX_CHANNEL_V(0) |
 978                 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
 979
 980         if (!is_t5(lldi->adapter_type))
 981                 opt2 |= RX_FC_DISABLE_F;
 982         if (req->tcpopt.tstamp)
 983                 opt2 |= TSTAMPS_EN_F;
 984         if (req->tcpopt.sack)
 985                 opt2 |= SACK_EN_F;
 986         hlen = ntohl(req->hdr_len);
 987
 988         tcph = (struct tcphdr *)((u8 *)(req + 1) +
 989                         T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
 990         if (tcph->ece && tcph->cwr)
 991                 opt2 |= CCTRL_ECN_V(1);
 992         opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
 993         opt2 |= T5_ISS_F;
 994         opt2 |= T5_OPT_2_VALID_F;
 995         rpl5->opt0 = cpu_to_be64(opt0);
 996         rpl5->opt2 = cpu_to_be32(opt2);
 997         rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
 998         set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
 999         t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
1000         cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
1001 }
1002
1003 static void inet_inherit_port(struct inet_hashinfo *hash_info,
1004                               struct sock *lsk, struct sock *newsk)
1005 {
1006         local_bh_disable();
1007         __inet_inherit_port(lsk, newsk);
1008         local_bh_enable();
1009 }
1010
1011 static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1012 {
1013         if (skb->protocol) {
1014                 kfree_skb(skb);
1015                 return 0;
1016         }
1017         BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
1018         return 0;
1019 }
1020
1021 static struct sock *chtls_recv_sock(struct sock *lsk,
1022                                     struct request_sock *oreq,
1023                                     void *network_hdr,
1024                                     const struct cpl_pass_accept_req *req,
1025                                     struct chtls_dev *cdev)
1026 {
1027         struct adapter *adap = pci_get_drvdata(cdev->pdev);
1028         const struct tcphdr *tcph;
1029         struct inet_sock *newinet;
1030         const struct iphdr *iph;
1031         struct net_device *ndev;
1032         struct chtls_sock *csk;
1033         struct dst_entry *dst;
1034         struct neighbour *n;
1035         struct tcp_sock *tp;
1036         struct sock *newsk;
1037         bool found = false;
1038         u16 port_id;
1039         int rxq_idx;
1040         int step, i;
1041
1042         iph = (const struct iphdr *)network_hdr;
1043         newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
1044         if (!newsk)
1045                 goto free_oreq;
1046
1047         dst = inet_csk_route_child_sock(lsk, newsk, oreq);
1048         if (!dst)
1049                 goto free_sk;
1050
1051         tcph = (struct tcphdr *)(iph + 1);
1052         n = dst_neigh_lookup(dst, &iph->saddr);
1053         if (!n || !n->dev)
1054                 goto free_dst;
1055
1056         ndev = n->dev;
1057         if (is_vlan_dev(ndev))
1058                 ndev = vlan_dev_real_dev(ndev);
1059
1060         for_each_port(adap, i)
1061                 if (cdev->ports[i] == ndev)
1062                         found = true;
1063
1064         if (!found)
1065                 goto free_dst;
1066
1067         port_id = cxgb4_port_idx(ndev);
1068
1069         csk = chtls_sock_create(cdev);
1070         if (!csk)
1071                 goto free_dst;
1072
1073         csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
1074         if (!csk->l2t_entry)
1075                 goto free_csk;
1076
1077         newsk->sk_user_data = csk;
1078         newsk->sk_backlog_rcv = chtls_backlog_rcv;
1079
1080         tp = tcp_sk(newsk);
1081         newinet = inet_sk(newsk);
1082
1083         newinet->inet_daddr = iph->saddr;
1084         newinet->inet_rcv_saddr = iph->daddr;
1085         newinet->inet_saddr = iph->daddr;
1086
1087         oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1088         sk_setup_caps(newsk, dst);
1089         newsk->sk_prot_creator = lsk->sk_prot_creator;
1090         csk->sk = newsk;
1091         csk->passive_reap_next = oreq;
1092         csk->tx_chan = cxgb4_port_chan(ndev);
1093         csk->port_id = port_id;
1094         csk->egress_dev = ndev;
1095         csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
1096         csk->ulp_mode = ULP_MODE_TLS;
1097         step = cdev->lldi->nrxq / cdev->lldi->nchan;
1098         csk->rss_qid = cdev->lldi->rxq_ids[port_id * step];
1099         rxq_idx = port_id * step;
1100         csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
1101                         port_id * step;
1102         csk->sndbuf = newsk->sk_sndbuf;
1103         csk->smac_idx = cxgb4_tp_smt_idx(cdev->lldi->adapter_type,
1104                                          cxgb4_port_viid(ndev));
1105         tp->rcv_wnd = select_rcv_wnd(csk);
1106         RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
1107                                            WSCALE_OK(tp),
1108                                            tp->window_clamp);
1109         neigh_release(n);
1110         inet_inherit_port(&tcp_hashinfo, lsk, newsk);
1111         csk_set_flag(csk, CSK_CONN_INLINE);
1112         bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
1113
1114         return newsk;
1115 free_csk:
1116         chtls_sock_release(&csk->kref);
1117 free_dst:
1118         if (n)
1119                 neigh_release(n);
1120         dst_release(dst);
1121 free_sk:
1122         inet_csk_prepare_forced_close(newsk);
1123         tcp_done(newsk);
1124 free_oreq:
1125         chtls_reqsk_free(oreq);
1126         return NULL;
1127 }
1128
1129 /*
1130  * Populate a TID_RELEASE WR.  The skb must be already propely sized.
1131  */
1132 static  void mk_tid_release(struct sk_buff *skb,
1133                             unsigned int chan, unsigned int tid)
1134 {
1135         struct cpl_tid_release *req;
1136         unsigned int len;
1137
1138         len = roundup(sizeof(struct cpl_tid_release), 16);
1139         req = (struct cpl_tid_release *)__skb_put(skb, len);
1140         memset(req, 0, len);
1141         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1142         INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
1143 }
1144
1145 static int chtls_get_module(struct sock *sk)
1146 {
1147         struct inet_connection_sock *icsk = inet_csk(sk);
1148
1149         if (!try_module_get(icsk->icsk_ulp_ops->owner))
1150                 return -1;
1151
1152         return 0;
1153 }
1154
1155 static void chtls_pass_accept_request(struct sock *sk,
1156                                       struct sk_buff *skb)
1157 {
1158         struct cpl_t5_pass_accept_rpl *rpl;
1159         struct cpl_pass_accept_req *req;
1160         struct listen_ctx *listen_ctx;
1161         struct request_sock *oreq;
1162         struct sk_buff *reply_skb;
1163         struct chtls_sock *csk;
1164         struct chtls_dev *cdev;
1165         struct tcphdr *tcph;
1166         struct sock *newsk;
1167         struct ethhdr *eh;
1168         struct iphdr *iph;
1169         void *network_hdr;
1170         unsigned int stid;
1171         unsigned int len;
1172         unsigned int tid;
1173
1174         req = cplhdr(skb) + RSS_HDR;
1175         tid = GET_TID(req);
1176         cdev = BLOG_SKB_CB(skb)->cdev;
1177         newsk = lookup_tid(cdev->tids, tid);
1178         stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1179         if (newsk) {
1180                 pr_info("tid (%d) already in use\n", tid);
1181                 return;
1182         }
1183
1184         len = roundup(sizeof(*rpl), 16);
1185         reply_skb = alloc_skb(len, GFP_ATOMIC);
1186         if (!reply_skb) {
1187                 cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
1188                 kfree_skb(skb);
1189                 return;
1190         }
1191
1192         if (sk->sk_state != TCP_LISTEN)
1193                 goto reject;
1194
1195         if (inet_csk_reqsk_queue_is_full(sk))
1196                 goto reject;
1197
1198         if (sk_acceptq_is_full(sk))
1199                 goto reject;
1200
1201         oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
1202         if (!oreq)
1203                 goto reject;
1204
1205         oreq->rsk_rcv_wnd = 0;
1206         oreq->rsk_window_clamp = 0;
1207         oreq->cookie_ts = 0;
1208         oreq->mss = 0;
1209         oreq->ts_recent = 0;
1210
1211         eh = (struct ethhdr *)(req + 1);
1212         iph = (struct iphdr *)(eh + 1);
1213         if (iph->version != 0x4)
1214                 goto free_oreq;
1215
1216         network_hdr = (void *)(eh + 1);
1217         tcph = (struct tcphdr *)(iph + 1);
1218
1219         tcp_rsk(oreq)->tfo_listener = false;
1220         tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
1221         chtls_set_req_port(oreq, tcph->source, tcph->dest);
1222         inet_rsk(oreq)->ecn_ok = 0;
1223         chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
1224         if (req->tcpopt.wsf <= 14) {
1225                 inet_rsk(oreq)->wscale_ok = 1;
1226                 inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
1227         }
1228         inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
1229
1230         newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
1231         if (!newsk)
1232                 goto reject;
1233
1234         if (chtls_get_module(newsk))
1235                 goto reject;
1236         inet_csk_reqsk_queue_added(sk);
1237         reply_skb->sk = newsk;
1238         chtls_install_cpl_ops(newsk);
1239         cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
1240         csk = rcu_dereference_sk_user_data(newsk);
1241         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
1242         csk->listen_ctx = listen_ctx;
1243         __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
1244         chtls_pass_accept_rpl(reply_skb, req, tid);
1245         kfree_skb(skb);
1246         return;
1247
1248 free_oreq:
1249         chtls_reqsk_free(oreq);
1250 reject:
1251         mk_tid_release(reply_skb, 0, tid);
1252         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1253         kfree_skb(skb);
1254 }
1255
1256 /*
1257  * Handle a CPL_PASS_ACCEPT_REQ message.
1258  */
1259 static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
1260 {
1261         struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
1262         struct listen_ctx *ctx;
1263         unsigned int stid;
1264         unsigned int tid;
1265         struct sock *lsk;
1266         void *data;
1267
1268         stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1269         tid = GET_TID(req);
1270
1271         data = lookup_stid(cdev->tids, stid);
1272         if (!data)
1273                 return 1;
1274
1275         ctx = (struct listen_ctx *)data;
1276         lsk = ctx->lsk;
1277
1278         if (unlikely(tid >= cdev->tids->ntids)) {
1279                 pr_info("passive open TID %u too large\n", tid);
1280                 return 1;
1281         }
1282
1283         BLOG_SKB_CB(skb)->cdev = cdev;
1284         process_cpl_msg(chtls_pass_accept_request, lsk, skb);
1285         return 0;
1286 }
1287
1288 /*
1289  * Completes some final bits of initialization for just established connections
1290  * and changes their state to TCP_ESTABLISHED.
1291  *
1292  * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
1293  */
1294 static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
1295 {
1296         struct tcp_sock *tp = tcp_sk(sk);
1297
1298         tp->pushed_seq = snd_isn;
1299         tp->write_seq = snd_isn;
1300         tp->snd_nxt = snd_isn;
1301         tp->snd_una = snd_isn;
1302         inet_sk(sk)->inet_id = prandom_u32();
1303         assign_rxopt(sk, opt);
1304
1305         if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
1306                 tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
1307
1308         smp_mb();
1309         tcp_set_state(sk, TCP_ESTABLISHED);
1310 }
1311
1312 static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
1313 {
1314         struct sk_buff *abort_skb;
1315
1316         abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
1317         if (abort_skb)
1318                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
1319 }
1320
1321 static struct sock *reap_list;
1322 static DEFINE_SPINLOCK(reap_list_lock);
1323
1324 /*
1325  * Process the reap list.
1326  */
1327 DECLARE_TASK_FUNC(process_reap_list, task_param)
1328 {
1329         spin_lock_bh(&reap_list_lock);
1330         while (reap_list) {
1331                 struct sock *sk = reap_list;
1332                 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1333
1334                 reap_list = csk->passive_reap_next;
1335                 csk->passive_reap_next = NULL;
1336                 spin_unlock(&reap_list_lock);
1337                 sock_hold(sk);
1338
1339                 bh_lock_sock(sk);
1340                 chtls_abort_conn(sk, NULL);
1341                 sock_orphan(sk);
1342                 if (sk->sk_state == TCP_CLOSE)
1343                         inet_csk_destroy_sock(sk);
1344                 bh_unlock_sock(sk);
1345                 sock_put(sk);
1346                 spin_lock(&reap_list_lock);
1347         }
1348         spin_unlock_bh(&reap_list_lock);
1349 }
1350
1351 static DECLARE_WORK(reap_task, process_reap_list);
1352
1353 static void add_to_reap_list(struct sock *sk)
1354 {
1355         struct chtls_sock *csk = sk->sk_user_data;
1356
1357         local_bh_disable();
1358         release_tcp_port(sk); /* release the port immediately */
1359
1360         spin_lock(&reap_list_lock);
1361         csk->passive_reap_next = reap_list;
1362         reap_list = sk;
1363         if (!csk->passive_reap_next)
1364                 schedule_work(&reap_task);
1365         spin_unlock(&reap_list_lock);
1366         local_bh_enable();
1367 }
1368
1369 static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
1370                                     struct chtls_dev *cdev)
1371 {
1372         struct request_sock *oreq;
1373         struct chtls_sock *csk;
1374
1375         if (lsk->sk_state != TCP_LISTEN)
1376                 return;
1377
1378         csk = child->sk_user_data;
1379         oreq = csk->passive_reap_next;
1380         csk->passive_reap_next = NULL;
1381
1382         reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
1383         __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
1384
1385         if (sk_acceptq_is_full(lsk)) {
1386                 chtls_reqsk_free(oreq);
1387                 add_to_reap_list(child);
1388         } else {
1389                 refcount_set(&oreq->rsk_refcnt, 1);
1390                 inet_csk_reqsk_queue_add(lsk, oreq, child);
1391                 lsk->sk_data_ready(lsk);
1392         }
1393 }
1394
1395 static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
1396 {
1397         struct sock *child = skb->sk;
1398
1399         skb->sk = NULL;
1400         add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
1401         kfree_skb(skb);
1402 }
1403
1404 static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
1405 {
1406         struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
1407         struct chtls_sock *csk;
1408         struct sock *lsk, *sk;
1409         unsigned int hwtid;
1410
1411         hwtid = GET_TID(req);
1412         sk = lookup_tid(cdev->tids, hwtid);
1413         if (!sk)
1414                 return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
1415
1416         bh_lock_sock(sk);
1417         if (unlikely(sock_owned_by_user(sk))) {
1418                 kfree_skb(skb);
1419         } else {
1420                 unsigned int stid;
1421                 void *data;
1422
1423                 csk = sk->sk_user_data;
1424                 csk->wr_max_credits = 64;
1425                 csk->wr_credits = 64;
1426                 csk->wr_unacked = 0;
1427                 make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
1428                 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1429                 sk->sk_state_change(sk);
1430                 if (unlikely(sk->sk_socket))
1431                         sk_wake_async(sk, 0, POLL_OUT);
1432
1433                 data = lookup_stid(cdev->tids, stid);
1434                 if (!data) {
1435                         /* listening server close */
1436                         kfree_skb(skb);
1437                         goto unlock;
1438                 }
1439                 lsk = ((struct listen_ctx *)data)->lsk;
1440
1441                 bh_lock_sock(lsk);
1442                 if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
1443                         /* removed from synq */
1444                         bh_unlock_sock(lsk);
1445                         kfree_skb(skb);
1446                         goto unlock;
1447                 }
1448
1449                 if (likely(!sock_owned_by_user(lsk))) {
1450                         kfree_skb(skb);
1451                         add_pass_open_to_parent(sk, lsk, cdev);
1452                 } else {
1453                         skb->sk = sk;
1454                         BLOG_SKB_CB(skb)->cdev = cdev;
1455                         BLOG_SKB_CB(skb)->backlog_rcv =
1456                                 bl_add_pass_open_to_parent;
1457                         __sk_add_backlog(lsk, skb);
1458                 }
1459                 bh_unlock_sock(lsk);
1460         }
1461 unlock:
1462         bh_unlock_sock(sk);
1463         return 0;
1464 }
1465
1466 /*
1467  * Handle receipt of an urgent pointer.
1468  */
1469 static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
1470 {
1471         struct tcp_sock *tp = tcp_sk(sk);
1472
1473         urg_seq--;
1474         if (tp->urg_data && !after(urg_seq, tp->urg_seq))
1475                 return; /* duplicate pointer */
1476
1477         sk_send_sigurg(sk);
1478         if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
1479             !sock_flag(sk, SOCK_URGINLINE) &&
1480             tp->copied_seq != tp->rcv_nxt) {
1481                 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1482
1483                 tp->copied_seq++;
1484                 if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
1485                         chtls_free_skb(sk, skb);
1486         }
1487
1488         tp->urg_data = TCP_URG_NOTYET;
1489         tp->urg_seq = urg_seq;
1490 }
1491
1492 static void check_sk_callbacks(struct chtls_sock *csk)
1493 {
1494         struct sock *sk = csk->sk;
1495
1496         if (unlikely(sk->sk_user_data &&
1497                      !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
1498                 csk_set_flag(csk, CSK_CALLBACKS_CHKD);
1499 }
1500
1501 /*
1502  * Handles Rx data that arrives in a state where the socket isn't accepting
1503  * new data.
1504  */
1505 static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
1506 {
1507         if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
1508                 chtls_abort_conn(sk, skb);
1509
1510         kfree_skb(skb);
1511 }
1512
1513 static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
1514 {
1515         struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
1516         struct chtls_sock *csk;
1517         struct tcp_sock *tp;
1518
1519         csk = rcu_dereference_sk_user_data(sk);
1520         tp = tcp_sk(sk);
1521
1522         if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1523                 handle_excess_rx(sk, skb);
1524                 return;
1525         }
1526
1527         ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1528         ULP_SKB_CB(skb)->psh = hdr->psh;
1529         skb_ulp_mode(skb) = ULP_MODE_NONE;
1530
1531         skb_reset_transport_header(skb);
1532         __skb_pull(skb, sizeof(*hdr) + RSS_HDR);
1533         if (!skb->data_len)
1534                 __skb_trim(skb, ntohs(hdr->len));
1535
1536         if (unlikely(hdr->urg))
1537                 handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
1538         if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
1539                      tp->urg_seq - tp->rcv_nxt < skb->len))
1540                 tp->urg_data = TCP_URG_VALID |
1541                                skb->data[tp->urg_seq - tp->rcv_nxt];
1542
1543         if (unlikely(hdr->dack_mode != csk->delack_mode)) {
1544                 csk->delack_mode = hdr->dack_mode;
1545                 csk->delack_seq = tp->rcv_nxt;
1546         }
1547
1548         tcp_hdr(skb)->fin = 0;
1549         tp->rcv_nxt += skb->len;
1550
1551         __skb_queue_tail(&sk->sk_receive_queue, skb);
1552
1553         if (!sock_flag(sk, SOCK_DEAD)) {
1554                 check_sk_callbacks(csk);
1555                 sk->sk_data_ready(sk);
1556         }
1557 }
1558
1559 static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
1560 {
1561         struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
1562         unsigned int hwtid = GET_TID(req);
1563         struct sock *sk;
1564
1565         sk = lookup_tid(cdev->tids, hwtid);
1566         if (unlikely(!sk)) {
1567                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1568                 return -EINVAL;
1569         }
1570         skb_dst_set(skb, NULL);
1571         process_cpl_msg(chtls_recv_data, sk, skb);
1572         return 0;
1573 }
1574
1575 static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
1576 {
1577         struct cpl_tls_data *hdr = cplhdr(skb);
1578         struct chtls_sock *csk;
1579         struct chtls_hws *tlsk;
1580         struct tcp_sock *tp;
1581
1582         csk = rcu_dereference_sk_user_data(sk);
1583         tlsk = &csk->tlshws;
1584         tp = tcp_sk(sk);
1585
1586         if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1587                 handle_excess_rx(sk, skb);
1588                 return;
1589         }
1590
1591         ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1592         ULP_SKB_CB(skb)->flags = 0;
1593         skb_ulp_mode(skb) = ULP_MODE_TLS;
1594
1595         skb_reset_transport_header(skb);
1596         __skb_pull(skb, sizeof(*hdr));
1597         if (!skb->data_len)
1598                 __skb_trim(skb,
1599                            CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
1600
1601         if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
1602                      tp->rcv_nxt < skb->len))
1603                 tp->urg_data = TCP_URG_VALID |
1604                                skb->data[tp->urg_seq - tp->rcv_nxt];
1605
1606         tcp_hdr(skb)->fin = 0;
1607         tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
1608         __skb_queue_tail(&tlsk->sk_recv_queue, skb);
1609 }
1610
1611 static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
1612 {
1613         struct cpl_tls_data *req = cplhdr(skb);
1614         unsigned int hwtid = GET_TID(req);
1615         struct sock *sk;
1616
1617         sk = lookup_tid(cdev->tids, hwtid);
1618         if (unlikely(!sk)) {
1619                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1620                 return -EINVAL;
1621         }
1622         skb_dst_set(skb, NULL);
1623         process_cpl_msg(chtls_recv_pdu, sk, skb);
1624         return 0;
1625 }
1626
1627 static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
1628 {
1629         struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
1630
1631         skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
1632         tls_cmp_hdr->length = ntohs((__force __be16)nlen);
1633 }
1634
1635 static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
1636 {
1637         struct tlsrx_cmp_hdr *tls_hdr_pkt;
1638         struct cpl_rx_tls_cmp *cmp_cpl;
1639         struct sk_buff *skb_rec;
1640         struct chtls_sock *csk;
1641         struct chtls_hws *tlsk;
1642         struct tcp_sock *tp;
1643
1644         cmp_cpl = cplhdr(skb);
1645         csk = rcu_dereference_sk_user_data(sk);
1646         tlsk = &csk->tlshws;
1647         tp = tcp_sk(sk);
1648
1649         ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
1650         ULP_SKB_CB(skb)->flags = 0;
1651
1652         skb_reset_transport_header(skb);
1653         __skb_pull(skb, sizeof(*cmp_cpl));
1654         tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
1655         if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
1656                 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
1657         if (!skb->data_len)
1658                 __skb_trim(skb, TLS_HEADER_LENGTH);
1659
1660         tp->rcv_nxt +=
1661                 CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
1662
1663         ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
1664         skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
1665         if (!skb_rec) {
1666                 __skb_queue_tail(&sk->sk_receive_queue, skb);
1667         } else {
1668                 chtls_set_hdrlen(skb, tlsk->pldlen);
1669                 tlsk->pldlen = 0;
1670                 __skb_queue_tail(&sk->sk_receive_queue, skb);
1671                 __skb_queue_tail(&sk->sk_receive_queue, skb_rec);
1672         }
1673
1674         if (!sock_flag(sk, SOCK_DEAD)) {
1675                 check_sk_callbacks(csk);
1676                 sk->sk_data_ready(sk);
1677         }
1678 }
1679
1680 static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
1681 {
1682         struct cpl_rx_tls_cmp *req = cplhdr(skb);
1683         unsigned int hwtid = GET_TID(req);
1684         struct sock *sk;
1685
1686         sk = lookup_tid(cdev->tids, hwtid);
1687         if (unlikely(!sk)) {
1688                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1689                 return -EINVAL;
1690         }
1691         skb_dst_set(skb, NULL);
1692         process_cpl_msg(chtls_rx_hdr, sk, skb);
1693
1694         return 0;
1695 }
1696
1697 static void chtls_timewait(struct sock *sk)
1698 {
1699         struct tcp_sock *tp = tcp_sk(sk);
1700
1701         tp->rcv_nxt++;
1702         tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
1703         tp->srtt_us = 0;
1704         tcp_time_wait(sk, TCP_TIME_WAIT, 0);
1705 }
1706
1707 static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
1708 {
1709         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1710
1711         sk->sk_shutdown |= RCV_SHUTDOWN;
1712         sock_set_flag(sk, SOCK_DONE);
1713
1714         switch (sk->sk_state) {
1715         case TCP_SYN_RECV:
1716         case TCP_ESTABLISHED:
1717                 tcp_set_state(sk, TCP_CLOSE_WAIT);
1718                 break;
1719         case TCP_FIN_WAIT1:
1720                 tcp_set_state(sk, TCP_CLOSING);
1721                 break;
1722         case TCP_FIN_WAIT2:
1723                 chtls_release_resources(sk);
1724                 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1725                         chtls_conn_done(sk);
1726                 else
1727                         chtls_timewait(sk);
1728                 break;
1729         default:
1730                 pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
1731         }
1732
1733         if (!sock_flag(sk, SOCK_DEAD)) {
1734                 sk->sk_state_change(sk);
1735                 /* Do not send POLL_HUP for half duplex close. */
1736
1737                 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1738                     sk->sk_state == TCP_CLOSE)
1739                         sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
1740                 else
1741                         sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
1742         }
1743         kfree_skb(skb);
1744 }
1745
1746 static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
1747 {
1748         struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
1749         struct chtls_sock *csk;
1750         struct tcp_sock *tp;
1751
1752         csk = rcu_dereference_sk_user_data(sk);
1753         tp = tcp_sk(sk);
1754
1755         tp->snd_una = ntohl(rpl->snd_nxt) - 1;  /* exclude FIN */
1756
1757         switch (sk->sk_state) {
1758         case TCP_CLOSING:
1759                 chtls_release_resources(sk);
1760                 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1761                         chtls_conn_done(sk);
1762                 else
1763                         chtls_timewait(sk);
1764                 break;
1765         case TCP_LAST_ACK:
1766                 chtls_release_resources(sk);
1767                 chtls_conn_done(sk);
1768                 break;
1769         case TCP_FIN_WAIT1:
1770                 tcp_set_state(sk, TCP_FIN_WAIT2);
1771                 sk->sk_shutdown |= SEND_SHUTDOWN;
1772
1773                 if (!sock_flag(sk, SOCK_DEAD))
1774                         sk->sk_state_change(sk);
1775                 else if (tcp_sk(sk)->linger2 < 0 &&
1776                          !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
1777                         chtls_abort_conn(sk, skb);
1778                 break;
1779         default:
1780                 pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
1781         }
1782         kfree_skb(skb);
1783 }
1784
1785 static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
1786                                    size_t len, gfp_t gfp)
1787 {
1788         if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
1789                 WARN_ONCE(skb->len < len, "skb alloc error");
1790                 __skb_trim(skb, len);
1791                 skb_get(skb);
1792         } else {
1793                 skb = alloc_skb(len, gfp);
1794                 if (skb)
1795                         __skb_put(skb, len);
1796         }
1797         return skb;
1798 }
1799
1800 static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
1801                              int cmd)
1802 {
1803         struct cpl_abort_rpl *rpl = cplhdr(skb);
1804
1805         INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
1806         rpl->cmd = cmd;
1807 }
1808
1809 static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
1810 {
1811         struct cpl_abort_req_rss *req = cplhdr(skb);
1812         struct sk_buff *reply_skb;
1813
1814         reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1815                               GFP_KERNEL | __GFP_NOFAIL);
1816         __skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
1817         set_abort_rpl_wr(reply_skb, GET_TID(req),
1818                          (req->status & CPL_ABORT_NO_RST));
1819         set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
1820         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1821         kfree_skb(skb);
1822 }
1823
1824 /*
1825  * Add an skb to the deferred skb queue for processing from process context.
1826  */
1827 static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
1828                            defer_handler_t handler)
1829 {
1830         DEFERRED_SKB_CB(skb)->handler = handler;
1831         spin_lock_bh(&cdev->deferq.lock);
1832         __skb_queue_tail(&cdev->deferq, skb);
1833         if (skb_queue_len(&cdev->deferq) == 1)
1834                 schedule_work(&cdev->deferq_task);
1835         spin_unlock_bh(&cdev->deferq.lock);
1836 }
1837
1838 static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
1839                                  struct chtls_dev *cdev,
1840                                  int status, int queue)
1841 {
1842         struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
1843         struct sk_buff *reply_skb;
1844         struct chtls_sock *csk;
1845         unsigned int tid;
1846
1847         csk = rcu_dereference_sk_user_data(sk);
1848         tid = GET_TID(req);
1849
1850         reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
1851         if (!reply_skb) {
1852                 req->status = (queue << 1) | status;
1853                 t4_defer_reply(skb, cdev, send_defer_abort_rpl);
1854                 return;
1855         }
1856
1857         set_abort_rpl_wr(reply_skb, tid, status);
1858         set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
1859         if (csk_conn_inline(csk)) {
1860                 struct l2t_entry *e = csk->l2t_entry;
1861
1862                 if (e && sk->sk_state != TCP_SYN_RECV) {
1863                         cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
1864                         return;
1865                 }
1866         }
1867         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1868         kfree_skb(skb);
1869 }
1870
1871 /*
1872  * This is run from a listener's backlog to abort a child connection in
1873  * SYN_RCV state (i.e., one on the listener's SYN queue).
1874  */
1875 static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
1876 {
1877         struct chtls_sock *csk;
1878         struct sock *child;
1879         int queue;
1880
1881         child = skb->sk;
1882         csk = rcu_dereference_sk_user_data(child);
1883         queue = csk->txq_idx;
1884
1885         skb->sk = NULL;
1886         chtls_send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
1887                              CPL_ABORT_NO_RST, queue);
1888         do_abort_syn_rcv(child, lsk);
1889 }
1890
1891 static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
1892 {
1893         const struct request_sock *oreq;
1894         struct listen_ctx *listen_ctx;
1895         struct chtls_sock *csk;
1896         struct chtls_dev *cdev;
1897         struct sock *psk;
1898         void *ctx;
1899
1900         csk = sk->sk_user_data;
1901         oreq = csk->passive_reap_next;
1902         cdev = csk->cdev;
1903
1904         if (!oreq)
1905                 return -1;
1906
1907         ctx = lookup_stid(cdev->tids, oreq->ts_recent);
1908         if (!ctx)
1909                 return -1;
1910
1911         listen_ctx = (struct listen_ctx *)ctx;
1912         psk = listen_ctx->lsk;
1913
1914         bh_lock_sock(psk);
1915         if (!sock_owned_by_user(psk)) {
1916                 int queue = csk->txq_idx;
1917
1918                 chtls_send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
1919                 do_abort_syn_rcv(sk, psk);
1920         } else {
1921                 skb->sk = sk;
1922                 BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
1923                 __sk_add_backlog(psk, skb);
1924         }
1925         bh_unlock_sock(psk);
1926         return 0;
1927 }
1928
1929 static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
1930 {
1931         const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
1932         struct chtls_sock *csk = sk->sk_user_data;
1933         int rst_status = CPL_ABORT_NO_RST;
1934         int queue = csk->txq_idx;
1935
1936         if (is_neg_adv(req->status)) {
1937                 kfree_skb(skb);
1938                 return;
1939         }
1940
1941         csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
1942
1943         if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
1944             !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
1945                 struct tcp_sock *tp = tcp_sk(sk);
1946
1947                 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
1948                         WARN_ONCE(1, "send_tx_flowc error");
1949                 csk_set_flag(csk, CSK_TX_DATA_SENT);
1950         }
1951
1952         csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
1953
1954         if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
1955                 sk->sk_err = ETIMEDOUT;
1956
1957                 if (!sock_flag(sk, SOCK_DEAD))
1958                         sk->sk_error_report(sk);
1959
1960                 if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
1961                         return;
1962         }
1963
1964         chtls_send_abort_rpl(sk, skb, csk->cdev, rst_status, queue);
1965         chtls_release_resources(sk);
1966         chtls_conn_done(sk);
1967 }
1968
1969 static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
1970 {
1971         struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
1972         struct chtls_sock *csk;
1973         struct chtls_dev *cdev;
1974
1975         csk = rcu_dereference_sk_user_data(sk);
1976         cdev = csk->cdev;
1977
1978         if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
1979                 csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
1980                 if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
1981                         if (sk->sk_state == TCP_SYN_SENT) {
1982                                 cxgb4_remove_tid(cdev->tids,
1983                                                  csk->port_id,
1984                                                  GET_TID(rpl),
1985                                                  sk->sk_family);
1986                                 sock_put(sk);
1987                         }
1988                         chtls_release_resources(sk);
1989                         chtls_conn_done(sk);
1990                 }
1991         }
1992         kfree_skb(skb);
1993 }
1994
1995 static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
1996 {
1997         struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
1998         void (*fn)(struct sock *sk, struct sk_buff *skb);
1999         unsigned int hwtid = GET_TID(req);
2000         struct sock *sk;
2001         u8 opcode;
2002
2003         opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
2004
2005         sk = lookup_tid(cdev->tids, hwtid);
2006         if (!sk)
2007                 goto rel_skb;
2008
2009         switch (opcode) {
2010         case CPL_PEER_CLOSE:
2011                 fn = chtls_peer_close;
2012                 break;
2013         case CPL_CLOSE_CON_RPL:
2014                 fn = chtls_close_con_rpl;
2015                 break;
2016         case CPL_ABORT_REQ_RSS:
2017                 fn = chtls_abort_req_rss;
2018                 break;
2019         case CPL_ABORT_RPL_RSS:
2020                 fn = chtls_abort_rpl_rss;
2021                 break;
2022         default:
2023                 goto rel_skb;
2024         }
2025
2026         process_cpl_msg(fn, sk, skb);
2027         return 0;
2028
2029 rel_skb:
2030         kfree_skb(skb);
2031         return 0;
2032 }
2033
2034 static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
2035 {
2036         struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
2037         struct chtls_sock *csk = sk->sk_user_data;
2038         struct tcp_sock *tp = tcp_sk(sk);
2039         u32 credits = hdr->credits;
2040         u32 snd_una;
2041
2042         snd_una = ntohl(hdr->snd_una);
2043         csk->wr_credits += credits;
2044
2045         if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
2046                 csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
2047
2048         while (credits) {
2049                 struct sk_buff *pskb = csk->wr_skb_head;
2050                 u32 csum;
2051
2052                 if (unlikely(!pskb)) {
2053                         if (csk->wr_nondata)
2054                                 csk->wr_nondata -= credits;
2055                         break;
2056                 }
2057                 csum = (__force u32)pskb->csum;
2058                 if (unlikely(credits < csum)) {
2059                         pskb->csum = (__force __wsum)(csum - credits);
2060                         break;
2061                 }
2062                 dequeue_wr(sk);
2063                 credits -= csum;
2064                 kfree_skb(pskb);
2065         }
2066         if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
2067                 if (unlikely(before(snd_una, tp->snd_una))) {
2068                         kfree_skb(skb);
2069                         return;
2070                 }
2071
2072                 if (tp->snd_una != snd_una) {
2073                         tp->snd_una = snd_una;
2074                         tp->rcv_tstamp = tcp_time_stamp(tp);
2075                         if (tp->snd_una == tp->snd_nxt &&
2076                             !csk_flag_nochk(csk, CSK_TX_FAILOVER))
2077                                 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2078                 }
2079         }
2080
2081         if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
2082                 unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
2083
2084                 csk->wr_credits -= fclen16;
2085                 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2086                 csk_reset_flag(csk, CSK_TX_FAILOVER);
2087         }
2088         if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
2089                 sk->sk_write_space(sk);
2090
2091         kfree_skb(skb);
2092 }
2093
2094 static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
2095 {
2096         struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
2097         unsigned int hwtid = GET_TID(rpl);
2098         struct sock *sk;
2099
2100         sk = lookup_tid(cdev->tids, hwtid);
2101         if (unlikely(!sk)) {
2102                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
2103                 return -EINVAL;
2104         }
2105         process_cpl_msg(chtls_rx_ack, sk, skb);
2106
2107         return 0;
2108 }
2109
2110 chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
2111         [CPL_PASS_OPEN_RPL]     = chtls_pass_open_rpl,
2112         [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
2113         [CPL_PASS_ACCEPT_REQ]   = chtls_pass_accept_req,
2114         [CPL_PASS_ESTABLISH]    = chtls_pass_establish,
2115         [CPL_RX_DATA]           = chtls_rx_data,
2116         [CPL_TLS_DATA]          = chtls_rx_pdu,
2117         [CPL_RX_TLS_CMP]        = chtls_rx_cmp,
2118         [CPL_PEER_CLOSE]        = chtls_conn_cpl,
2119         [CPL_CLOSE_CON_RPL]     = chtls_conn_cpl,
2120         [CPL_ABORT_REQ_RSS]     = chtls_conn_cpl,
2121         [CPL_ABORT_RPL_RSS]     = chtls_conn_cpl,
2122         [CPL_FW4_ACK]           = chtls_wr_ack,
2123 };