drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2018 Chelsio Communications, Inc.
   4  *
   5  * Written by: Atul Gupta (atul.gupta@chelsio.com)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/list.h>
  10 #include <linux/workqueue.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/timer.h>
  13 #include <linux/notifier.h>
  14 #include <linux/inetdevice.h>
  15 #include <linux/ip.h>
  16 #include <linux/tcp.h>
  17 #include <linux/tls.h>
  18 #include <net/tls.h>
  19
  20 #include "chtls.h"
  21 #include "chtls_cm.h"
  22
  23 static void __set_tcb_field_direct(struct chtls_sock *csk,
  24                                    struct cpl_set_tcb_field *req, u16 word,
  25                                    u64 mask, u64 val, u8 cookie, int no_reply)
  26 {
  27         struct ulptx_idata *sc;
  28
  29         INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
  30         req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
  31         req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
  32                                 QUEUENO_V(csk->rss_qid));
  33         req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
  34         req->mask = cpu_to_be64(mask);
  35         req->val = cpu_to_be64(val);
  36         sc = (struct ulptx_idata *)(req + 1);
  37         sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
  38         sc->len = htonl(0);
  39 }
  40
  41 static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
  42                             u64 mask, u64 val, u8 cookie, int no_reply)
  43 {
  44         struct cpl_set_tcb_field *req;
  45         struct chtls_sock *csk;
  46         struct ulptx_idata *sc;
  47         unsigned int wrlen;
  48
  49         wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
  50         csk = rcu_dereference_sk_user_data(sk);
  51
  52         req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
  53         __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
  54         set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
  55 }
  56
  57 /*
  58  * Send control message to HW, message go as immediate data and packet
  59  * is freed immediately.
  60  */
  61 static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
  62 {
  63         struct cpl_set_tcb_field *req;
  64         unsigned int credits_needed;
  65         struct chtls_sock *csk;
  66         struct ulptx_idata *sc;
  67         struct sk_buff *skb;
  68         unsigned int wrlen;
  69         int ret;
  70
  71         wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
  72
  73         skb = alloc_skb(wrlen, GFP_ATOMIC);
  74         if (!skb)
  75                 return -ENOMEM;
  76
  77         credits_needed = DIV_ROUND_UP(wrlen, 16);
  78         csk = rcu_dereference_sk_user_data(sk);
  79
  80         __set_tcb_field(sk, skb, word, mask, val, 0, 1);
  81         skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
  82         csk->wr_credits -= credits_needed;
  83         csk->wr_unacked += credits_needed;
  84         enqueue_wr(csk, skb);
  85         ret = cxgb4_ofld_send(csk->egress_dev, skb);
  86         if (ret < 0)
  87                 kfree_skb(skb);
  88         return ret < 0 ? ret : 0;
  89 }
  90
  91 void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word,
  92                                  u64 mask, u64 val, u8 cookie,
  93                                  int through_l2t)
  94 {
  95         struct sk_buff *skb;
  96         unsigned int wrlen;
  97
  98         wrlen = sizeof(struct cpl_set_tcb_field) + sizeof(struct ulptx_idata);
  99         wrlen = roundup(wrlen, 16);
 100
 101         skb = alloc_skb(wrlen, GFP_KERNEL | __GFP_NOFAIL);
 102         if (!skb)
 103                 return;
 104
 105         __set_tcb_field(sk, skb, word, mask, val, cookie, 0);
 106         send_or_defer(sk, tcp_sk(sk), skb, through_l2t);
 107 }
 108
 109 /*
 110  * Set one of the t_flags bits in the TCB.
 111  */
 112 int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
 113 {
 114         return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
 115                                    (u64)val << bit_pos);
 116 }
 117
 118 static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
 119 {
 120         return chtls_set_tcb_field(sk, 31, 0xFFFFFFFFULL, keyid);
 121 }
 122
 123 static int chtls_set_tcb_seqno(struct sock *sk)
 124 {
 125         return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
 126 }
 127
 128 static int chtls_set_tcb_quiesce(struct sock *sk, int val)
 129 {
 130         return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
 131                                    TF_RX_QUIESCE_V(val));
 132 }
 133
 134 void chtls_set_quiesce_ctrl(struct sock *sk, int val)
 135 {
 136         struct chtls_sock *csk;
 137         struct sk_buff *skb;
 138         unsigned int wrlen;
 139         int ret;
 140
 141         wrlen = sizeof(struct cpl_set_tcb_field) + sizeof(struct ulptx_idata);
 142         wrlen = roundup(wrlen, 16);
 143
 144         skb = alloc_skb(wrlen, GFP_ATOMIC);
 145         if (!skb)
 146                 return;
 147
 148         csk = rcu_dereference_sk_user_data(sk);
 149
 150         __set_tcb_field(sk, skb, 1, TF_RX_QUIESCE_V(1), 0, 0, 1);
 151         set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
 152         ret = cxgb4_ofld_send(csk->egress_dev, skb);
 153         if (ret < 0)
 154                 kfree_skb(skb);
 155 }
 156
 157 /* TLS Key bitmap processing */
 158 int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
 159 {
 160         unsigned int num_key_ctx, bsize;
 161         int ksize;
 162
 163         num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
 164         bsize = BITS_TO_LONGS(num_key_ctx);
 165
 166         cdev->kmap.size = num_key_ctx;
 167         cdev->kmap.available = bsize;
 168         ksize = sizeof(*cdev->kmap.addr) * bsize;
 169         cdev->kmap.addr = kvzalloc(ksize, GFP_KERNEL);
 170         if (!cdev->kmap.addr)
 171                 return -ENOMEM;
 172
 173         cdev->kmap.start = lldi->vr->key.start;
 174         spin_lock_init(&cdev->kmap.lock);
 175         return 0;
 176 }
 177
 178 static int get_new_keyid(struct chtls_sock *csk, u32 optname)
 179 {
 180         struct net_device *dev = csk->egress_dev;
 181         struct chtls_dev *cdev = csk->cdev;
 182         struct chtls_hws *hws;
 183         struct adapter *adap;
 184         int keyid;
 185
 186         adap = netdev2adap(dev);
 187         hws = &csk->tlshws;
 188
 189         spin_lock_bh(&cdev->kmap.lock);
 190         keyid = find_first_zero_bit(cdev->kmap.addr, cdev->kmap.size);
 191         if (keyid < cdev->kmap.size) {
 192                 __set_bit(keyid, cdev->kmap.addr);
 193                 if (optname == TLS_RX)
 194                         hws->rxkey = keyid;
 195                 else
 196                         hws->txkey = keyid;
 197                 atomic_inc(&adap->chcr_stats.tls_key);
 198         } else {
 199                 keyid = -1;
 200         }
 201         spin_unlock_bh(&cdev->kmap.lock);
 202         return keyid;
 203 }
 204
 205 void free_tls_keyid(struct sock *sk)
 206 {
 207         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 208         struct net_device *dev = csk->egress_dev;
 209         struct chtls_dev *cdev = csk->cdev;
 210         struct chtls_hws *hws;
 211         struct adapter *adap;
 212
 213         if (!cdev->kmap.addr)
 214                 return;
 215
 216         adap = netdev2adap(dev);
 217         hws = &csk->tlshws;
 218
 219         spin_lock_bh(&cdev->kmap.lock);
 220         if (hws->rxkey >= 0) {
 221                 __clear_bit(hws->rxkey, cdev->kmap.addr);
 222                 atomic_dec(&adap->chcr_stats.tls_key);
 223                 hws->rxkey = -1;
 224         }
 225         if (hws->txkey >= 0) {
 226                 __clear_bit(hws->txkey, cdev->kmap.addr);
 227                 atomic_dec(&adap->chcr_stats.tls_key);
 228                 hws->txkey = -1;
 229         }
 230         spin_unlock_bh(&cdev->kmap.lock);
 231 }
 232
 233 unsigned int keyid_to_addr(int start_addr, int keyid)
 234 {
 235         return (start_addr + (keyid * TLS_KEY_CONTEXT_SZ)) >> 5;
 236 }
 237
 238 static void chtls_rxkey_ivauth(struct _key_ctx *kctx)
 239 {
 240         kctx->iv_to_auth = cpu_to_be64(KEYCTX_TX_WR_IV_V(6ULL) |
 241                                   KEYCTX_TX_WR_AAD_V(1ULL) |
 242                                   KEYCTX_TX_WR_AADST_V(5ULL) |
 243                                   KEYCTX_TX_WR_CIPHER_V(14ULL) |
 244                                   KEYCTX_TX_WR_CIPHERST_V(0ULL) |
 245                                   KEYCTX_TX_WR_AUTH_V(14ULL) |
 246                                   KEYCTX_TX_WR_AUTHST_V(16ULL) |
 247                                   KEYCTX_TX_WR_AUTHIN_V(16ULL));
 248 }
 249
 250 static int chtls_key_info(struct chtls_sock *csk,
 251                           struct _key_ctx *kctx,
 252                           u32 keylen, u32 optname,
 253                           int cipher_type)
 254 {
 255         unsigned char key[AES_MAX_KEY_SIZE];
 256         unsigned char *key_p, *salt;
 257         unsigned char ghash_h[AEAD_H_SIZE];
 258         int ck_size, key_ctx_size, kctx_mackey_size, salt_size;
 259         struct crypto_aes_ctx aes;
 260         int ret;
 261
 262         key_ctx_size = sizeof(struct _key_ctx) +
 263                        roundup(keylen, 16) + AEAD_H_SIZE;
 264
 265         /* GCM mode of AES supports 128 and 256 bit encryption, so
 266          * prepare key context base on GCM cipher type
 267          */
 268         switch (cipher_type) {
 269         case TLS_CIPHER_AES_GCM_128: {
 270                 struct tls12_crypto_info_aes_gcm_128 *gcm_ctx_128 =
 271                         (struct tls12_crypto_info_aes_gcm_128 *)
 272                                         &csk->tlshws.crypto_info;
 273                 memcpy(key, gcm_ctx_128->key, keylen);
 274
 275                 key_p            = gcm_ctx_128->key;
 276                 salt             = gcm_ctx_128->salt;
 277                 ck_size          = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
 278                 salt_size        = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
 279                 kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
 280                 break;
 281         }
 282         case TLS_CIPHER_AES_GCM_256: {
 283                 struct tls12_crypto_info_aes_gcm_256 *gcm_ctx_256 =
 284                         (struct tls12_crypto_info_aes_gcm_256 *)
 285                                         &csk->tlshws.crypto_info;
 286                 memcpy(key, gcm_ctx_256->key, keylen);
 287
 288                 key_p            = gcm_ctx_256->key;
 289                 salt             = gcm_ctx_256->salt;
 290                 ck_size          = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
 291                 salt_size        = TLS_CIPHER_AES_GCM_256_SALT_SIZE;
 292                 kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
 293                 break;
 294         }
 295         default:
 296                 pr_err("GCM: Invalid key length %d\n", keylen);
 297                 return -EINVAL;
 298         }
 299
 300         /* Calculate the H = CIPH(K, 0 repeated 16 times).
 301          * It will go in key context
 302          */
 303         ret = aes_expandkey(&aes, key, keylen);
 304         if (ret)
 305                 return ret;
 306
 307         memset(ghash_h, 0, AEAD_H_SIZE);
 308         aes_encrypt(&aes, ghash_h, ghash_h);
 309         memzero_explicit(&aes, sizeof(aes));
 310         csk->tlshws.keylen = key_ctx_size;
 311
 312         /* Copy the Key context */
 313         if (optname == TLS_RX) {
 314                 int key_ctx;
 315
 316                 key_ctx = ((key_ctx_size >> 4) << 3);
 317                 kctx->ctx_hdr = FILL_KEY_CRX_HDR(ck_size,
 318                                                  kctx_mackey_size,
 319                                                  0, 0, key_ctx);
 320                 chtls_rxkey_ivauth(kctx);
 321         } else {
 322                 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
 323                                                  kctx_mackey_size,
 324                                                  0, 0, key_ctx_size >> 4);
 325         }
 326
 327         memcpy(kctx->salt, salt, salt_size);
 328         memcpy(kctx->key, key_p, keylen);
 329         memcpy(kctx->key + keylen, ghash_h, AEAD_H_SIZE);
 330         /* erase key info from driver */
 331         memset(key_p, 0, keylen);
 332
 333         return 0;
 334 }
 335
 336 static void chtls_set_scmd(struct chtls_sock *csk)
 337 {
 338         struct chtls_hws *hws = &csk->tlshws;
 339
 340         hws->scmd.seqno_numivs =
 341                 SCMD_SEQ_NO_CTRL_V(3) |
 342                 SCMD_PROTO_VERSION_V(0) |
 343                 SCMD_ENC_DEC_CTRL_V(0) |
 344                 SCMD_CIPH_AUTH_SEQ_CTRL_V(1) |
 345                 SCMD_CIPH_MODE_V(2) |
 346                 SCMD_AUTH_MODE_V(4) |
 347                 SCMD_HMAC_CTRL_V(0) |
 348                 SCMD_IV_SIZE_V(4) |
 349                 SCMD_NUM_IVS_V(1);
 350
 351         hws->scmd.ivgen_hdrlen =
 352                 SCMD_IV_GEN_CTRL_V(1) |
 353                 SCMD_KEY_CTX_INLINE_V(0) |
 354                 SCMD_TLS_FRAG_ENABLE_V(1);
 355 }
 356
 357 int chtls_setkey(struct chtls_sock *csk, u32 keylen,
 358                  u32 optname, int cipher_type)
 359 {
 360         struct tls_key_req *kwr;
 361         struct chtls_dev *cdev;
 362         struct _key_ctx *kctx;
 363         int wrlen, klen, len;
 364         struct sk_buff *skb;
 365         struct sock *sk;
 366         int keyid;
 367         int kaddr;
 368         int ret;
 369
 370         cdev = csk->cdev;
 371         sk = csk->sk;
 372
 373         klen = roundup((keylen + AEAD_H_SIZE) + sizeof(*kctx), 32);
 374         wrlen = roundup(sizeof(*kwr), 16);
 375         len = klen + wrlen;
 376
 377         /* Flush out-standing data before new key takes effect */
 378         if (optname == TLS_TX) {
 379                 lock_sock(sk);
 380                 if (skb_queue_len(&csk->txq))
 381                         chtls_push_frames(csk, 0);
 382                 release_sock(sk);
 383         }
 384
 385         skb = alloc_skb(len, GFP_KERNEL);
 386         if (!skb)
 387                 return -ENOMEM;
 388
 389         keyid = get_new_keyid(csk, optname);
 390         if (keyid < 0) {
 391                 ret = -ENOSPC;
 392                 goto out_nokey;
 393         }
 394
 395         kaddr = keyid_to_addr(cdev->kmap.start, keyid);
 396         kwr = (struct tls_key_req *)__skb_put_zero(skb, len);
 397         kwr->wr.op_to_compl =
 398                 cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | FW_WR_COMPL_F |
 399                       FW_WR_ATOMIC_V(1U));
 400         kwr->wr.flowid_len16 =
 401                 cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16) |
 402                             FW_WR_FLOWID_V(csk->tid)));
 403         kwr->wr.protocol = 0;
 404         kwr->wr.mfs = htons(TLS_MFS);
 405         kwr->wr.reneg_to_write_rx = optname;
 406
 407         /* ulptx command */
 408         kwr->req.cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
 409                             T5_ULP_MEMIO_ORDER_V(1) |
 410                             T5_ULP_MEMIO_IMM_V(1));
 411         kwr->req.len16 = cpu_to_be32((csk->tid << 8) |
 412                               DIV_ROUND_UP(len - sizeof(kwr->wr), 16));
 413         kwr->req.dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(klen >> 5));
 414         kwr->req.lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(kaddr));
 415
 416         /* sub command */
 417         kwr->sc_imm.cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM));
 418         kwr->sc_imm.len = cpu_to_be32(klen);
 419
 420         lock_sock(sk);
 421         /* key info */
 422         kctx = (struct _key_ctx *)(kwr + 1);
 423         ret = chtls_key_info(csk, kctx, keylen, optname, cipher_type);
 424         if (ret)
 425                 goto out_notcb;
 426
 427         if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN)))
 428                 goto out_notcb;
 429
 430         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->tlshws.txqid);
 431         csk->wr_credits -= DIV_ROUND_UP(len, 16);
 432         csk->wr_unacked += DIV_ROUND_UP(len, 16);
 433         enqueue_wr(csk, skb);
 434         cxgb4_ofld_send(csk->egress_dev, skb);
 435         skb = NULL;
 436
 437         chtls_set_scmd(csk);
 438         /* Clear quiesce for Rx key */
 439         if (optname == TLS_RX) {
 440                 ret = chtls_set_tcb_keyid(sk, keyid);
 441                 if (ret)
 442                         goto out_notcb;
 443                 ret = chtls_set_tcb_field(sk, 0,
 444                                           TCB_ULP_RAW_V(TCB_ULP_RAW_M),
 445                                           TCB_ULP_RAW_V((TF_TLS_KEY_SIZE_V(1) |
 446                                           TF_TLS_CONTROL_V(1) |
 447                                           TF_TLS_ACTIVE_V(1) |
 448                                           TF_TLS_ENABLE_V(1))));
 449                 if (ret)
 450                         goto out_notcb;
 451                 ret = chtls_set_tcb_seqno(sk);
 452                 if (ret)
 453                         goto out_notcb;
 454                 ret = chtls_set_tcb_quiesce(sk, 0);
 455                 if (ret)
 456                         goto out_notcb;
 457                 csk->tlshws.rxkey = keyid;
 458         } else {
 459                 csk->tlshws.tx_seq_no = 0;
 460                 csk->tlshws.txkey = keyid;
 461         }
 462
 463         release_sock(sk);
 464         return ret;
 465 out_notcb:
 466         release_sock(sk);
 467         free_tls_keyid(sk);
 468 out_nokey:
 469         kfree_skb(skb);
 470         return ret;
 471 }