arch/powerpc/crypto/aes-gcm-p10-glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Glue code for accelerated AES-GCM stitched implementation for ppc64le.
   4  *
   5  * Copyright 2022- IBM Inc. All rights reserved
   6  */
   7
   8 #include <asm/unaligned.h>
   9 #include <asm/simd.h>
  10 #include <asm/switch_to.h>
  11 #include <crypto/aes.h>
  12 #include <crypto/algapi.h>
  13 #include <crypto/b128ops.h>
  14 #include <crypto/gf128mul.h>
  15 #include <crypto/internal/simd.h>
  16 #include <crypto/internal/aead.h>
  17 #include <crypto/internal/hash.h>
  18 #include <crypto/internal/skcipher.h>
  19 #include <crypto/scatterwalk.h>
  20 #include <linux/cpufeature.h>
  21 #include <linux/crypto.h>
  22 #include <linux/module.h>
  23 #include <linux/types.h>
  24
  25 #define PPC_ALIGN               16
  26 #define GCM_IV_SIZE             12
  27
  28 MODULE_DESCRIPTION("PPC64le AES-GCM with Stitched implementation");
  29 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com");
  30 MODULE_LICENSE("GPL v2");
  31 MODULE_ALIAS_CRYPTO("aes");
  32
  33 asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits,
  34                                       void *key);
  35 asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key);
  36 asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len,
  37                                     void *rkey, u8 *iv, void *Xi);
  38 asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len,
  39                                     void *rkey, u8 *iv, void *Xi);
  40 asmlinkage void gcm_init_htable(unsigned char htable[256], unsigned char Xi[16]);
  41 asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable,
  42                 unsigned char *aad, unsigned int alen);
  43
  44 struct aes_key {
  45         u8 key[AES_MAX_KEYLENGTH];
  46         u64 rounds;
  47 };
  48
  49 struct gcm_ctx {
  50         u8 iv[16];
  51         u8 ivtag[16];
  52         u8 aad_hash[16];
  53         u64 aadLen;
  54         u64 Plen;       /* offset 56 - used in aes_p10_gcm_{en/de}crypt */
  55 };
  56 struct Hash_ctx {
  57         u8 H[16];       /* subkey */
  58         u8 Htable[256]; /* Xi, Hash table(offset 32) */
  59 };
  60
  61 struct p10_aes_gcm_ctx {
  62         struct aes_key enc_key;
  63 };
  64
  65 static void vsx_begin(void)
  66 {
  67         preempt_disable();
  68         enable_kernel_vsx();
  69 }
  70
  71 static void vsx_end(void)
  72 {
  73         disable_kernel_vsx();
  74         preempt_enable();
  75 }
  76
  77 static void set_subkey(unsigned char *hash)
  78 {
  79         *(u64 *)&hash[0] = be64_to_cpup((__be64 *)&hash[0]);
  80         *(u64 *)&hash[8] = be64_to_cpup((__be64 *)&hash[8]);
  81 }
  82
  83 /*
  84  * Compute aad if any.
  85  *   - Hash aad and copy to Xi.
  86  */
  87 static void set_aad(struct gcm_ctx *gctx, struct Hash_ctx *hash,
  88                     unsigned char *aad, int alen)
  89 {
  90         int i;
  91         u8 nXi[16] = {0, };
  92
  93         gctx->aadLen = alen;
  94         i = alen & ~0xf;
  95         if (i) {
  96                 gcm_ghash_p10(nXi, hash->Htable+32, aad, i);
  97                 aad += i;
  98                 alen -= i;
  99         }
 100         if (alen) {
 101                 for (i = 0; i < alen; i++)
 102                         nXi[i] ^= aad[i];
 103
 104                 memset(gctx->aad_hash, 0, 16);
 105                 gcm_ghash_p10(gctx->aad_hash, hash->Htable+32, nXi, 16);
 106         } else {
 107                 memcpy(gctx->aad_hash, nXi, 16);
 108         }
 109
 110         memcpy(hash->Htable, gctx->aad_hash, 16);
 111 }
 112
 113 static void gcmp10_init(struct gcm_ctx *gctx, u8 *iv, unsigned char *rdkey,
 114                         struct Hash_ctx *hash, u8 *assoc, unsigned int assoclen)
 115 {
 116         __be32 counter = cpu_to_be32(1);
 117
 118         aes_p10_encrypt(hash->H, hash->H, rdkey);
 119         set_subkey(hash->H);
 120         gcm_init_htable(hash->Htable+32, hash->H);
 121
 122         *((__be32 *)(iv+12)) = counter;
 123
 124         gctx->Plen = 0;
 125
 126         /*
 127          * Encrypt counter vector as iv tag and increment counter.
 128          */
 129         aes_p10_encrypt(iv, gctx->ivtag, rdkey);
 130
 131         counter = cpu_to_be32(2);
 132         *((__be32 *)(iv+12)) = counter;
 133         memcpy(gctx->iv, iv, 16);
 134
 135         gctx->aadLen = assoclen;
 136         memset(gctx->aad_hash, 0, 16);
 137         if (assoclen)
 138                 set_aad(gctx, hash, assoc, assoclen);
 139 }
 140
 141 static void finish_tag(struct gcm_ctx *gctx, struct Hash_ctx *hash, int len)
 142 {
 143         int i;
 144         unsigned char len_ac[16 + PPC_ALIGN];
 145         unsigned char *aclen = PTR_ALIGN((void *)len_ac, PPC_ALIGN);
 146         __be64 clen = cpu_to_be64(len << 3);
 147         __be64 alen = cpu_to_be64(gctx->aadLen << 3);
 148
 149         if (len == 0 && gctx->aadLen == 0) {
 150                 memcpy(hash->Htable, gctx->ivtag, 16);
 151                 return;
 152         }
 153
 154         /*
 155          * Len is in bits.
 156          */
 157         *((__be64 *)(aclen)) = alen;
 158         *((__be64 *)(aclen+8)) = clen;
 159
 160         /*
 161          * hash (AAD len and len)
 162          */
 163         gcm_ghash_p10(hash->Htable, hash->Htable+32, aclen, 16);
 164
 165         for (i = 0; i < 16; i++)
 166                 hash->Htable[i] ^= gctx->ivtag[i];
 167 }
 168
 169 static int set_authsize(struct crypto_aead *tfm, unsigned int authsize)
 170 {
 171         switch (authsize) {
 172         case 4:
 173         case 8:
 174         case 12:
 175         case 13:
 176         case 14:
 177         case 15:
 178         case 16:
 179                 break;
 180         default:
 181                 return -EINVAL;
 182         }
 183
 184         return 0;
 185 }
 186
 187 static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
 188                              unsigned int keylen)
 189 {
 190         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
 191         struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
 192         int ret;
 193
 194         vsx_begin();
 195         ret = aes_p10_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
 196         vsx_end();
 197
 198         return ret ? -EINVAL : 0;
 199 }
 200
 201 static int p10_aes_gcm_crypt(struct aead_request *req, int enc)
 202 {
 203         struct crypto_tfm *tfm = req->base.tfm;
 204         struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
 205         u8 databuf[sizeof(struct gcm_ctx) + PPC_ALIGN];
 206         struct gcm_ctx *gctx = PTR_ALIGN((void *)databuf, PPC_ALIGN);
 207         u8 hashbuf[sizeof(struct Hash_ctx) + PPC_ALIGN];
 208         struct Hash_ctx *hash = PTR_ALIGN((void *)hashbuf, PPC_ALIGN);
 209         struct scatter_walk assoc_sg_walk;
 210         struct skcipher_walk walk;
 211         u8 *assocmem = NULL;
 212         u8 *assoc;
 213         unsigned int assoclen = req->assoclen;
 214         unsigned int cryptlen = req->cryptlen;
 215         unsigned char ivbuf[AES_BLOCK_SIZE+PPC_ALIGN];
 216         unsigned char *iv = PTR_ALIGN((void *)ivbuf, PPC_ALIGN);
 217         int ret;
 218         unsigned long auth_tag_len = crypto_aead_authsize(__crypto_aead_cast(tfm));
 219         u8 otag[16];
 220         int total_processed = 0;
 221
 222         memset(databuf, 0, sizeof(databuf));
 223         memset(hashbuf, 0, sizeof(hashbuf));
 224         memset(ivbuf, 0, sizeof(ivbuf));
 225         memcpy(iv, req->iv, GCM_IV_SIZE);
 226
 227         /* Linearize assoc, if not already linear */
 228         if (req->src->length >= assoclen && req->src->length) {
 229                 scatterwalk_start(&assoc_sg_walk, req->src);
 230                 assoc = scatterwalk_map(&assoc_sg_walk);
 231         } else {
 232                 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 233                               GFP_KERNEL : GFP_ATOMIC;
 234
 235                 /* assoc can be any length, so must be on heap */
 236                 assocmem = kmalloc(assoclen, flags);
 237                 if (unlikely(!assocmem))
 238                         return -ENOMEM;
 239                 assoc = assocmem;
 240
 241                 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
 242         }
 243
 244         vsx_begin();
 245         gcmp10_init(gctx, iv, (unsigned char *) &ctx->enc_key, hash, assoc, assoclen);
 246         vsx_end();
 247
 248         if (!assocmem)
 249                 scatterwalk_unmap(assoc);
 250         else
 251                 kfree(assocmem);
 252
 253         if (enc)
 254                 ret = skcipher_walk_aead_encrypt(&walk, req, false);
 255         else
 256                 ret = skcipher_walk_aead_decrypt(&walk, req, false);
 257         if (ret)
 258                 return ret;
 259
 260         while (walk.nbytes > 0 && ret == 0) {
 261
 262                 vsx_begin();
 263                 if (enc)
 264                         aes_p10_gcm_encrypt(walk.src.virt.addr,
 265                                             walk.dst.virt.addr,
 266                                             walk.nbytes,
 267                                             &ctx->enc_key, gctx->iv, hash->Htable);
 268                 else
 269                         aes_p10_gcm_decrypt(walk.src.virt.addr,
 270                                             walk.dst.virt.addr,
 271                                             walk.nbytes,
 272                                             &ctx->enc_key, gctx->iv, hash->Htable);
 273                 vsx_end();
 274
 275                 total_processed += walk.nbytes;
 276                 ret = skcipher_walk_done(&walk, 0);
 277         }
 278
 279         if (ret)
 280                 return ret;
 281
 282         /* Finalize hash */
 283         vsx_begin();
 284         finish_tag(gctx, hash, total_processed);
 285         vsx_end();
 286
 287         /* copy Xi to end of dst */
 288         if (enc)
 289                 scatterwalk_map_and_copy(hash->Htable, req->dst, req->assoclen + cryptlen,
 290                                          auth_tag_len, 1);
 291         else {
 292                 scatterwalk_map_and_copy(otag, req->src,
 293                                          req->assoclen + cryptlen - auth_tag_len,
 294                                          auth_tag_len, 0);
 295
 296                 if (crypto_memneq(otag, hash->Htable, auth_tag_len)) {
 297                         memzero_explicit(hash->Htable, 16);
 298                         return -EBADMSG;
 299                 }
 300         }
 301
 302         return 0;
 303 }
 304
 305 static int p10_aes_gcm_encrypt(struct aead_request *req)
 306 {
 307         return p10_aes_gcm_crypt(req, 1);
 308 }
 309
 310 static int p10_aes_gcm_decrypt(struct aead_request *req)
 311 {
 312         return p10_aes_gcm_crypt(req, 0);
 313 }
 314
 315 static struct aead_alg gcm_aes_alg = {
 316         .ivsize                 = GCM_IV_SIZE,
 317         .maxauthsize            = 16,
 318
 319         .setauthsize            = set_authsize,
 320         .setkey                 = p10_aes_gcm_setkey,
 321         .encrypt                = p10_aes_gcm_encrypt,
 322         .decrypt                = p10_aes_gcm_decrypt,
 323
 324         .base.cra_name          = "gcm(aes)",
 325         .base.cra_driver_name   = "aes_gcm_p10",
 326         .base.cra_priority      = 2100,
 327         .base.cra_blocksize     = 1,
 328         .base.cra_ctxsize       = sizeof(struct p10_aes_gcm_ctx),
 329         .base.cra_module        = THIS_MODULE,
 330 };
 331
 332 static int __init p10_init(void)
 333 {
 334         return crypto_register_aead(&gcm_aes_alg);
 335 }
 336
 337 static void __exit p10_exit(void)
 338 {
 339         crypto_unregister_aead(&gcm_aes_alg);
 340 }
 341
 342 module_cpu_feature_match(PPC_MODULE_FEATURE_P10, p10_init);
 343 module_exit(p10_exit);