arch/arm64/crypto/sm4-ce-gcm-glue.c

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  * SM4-GCM AEAD Algorithm using ARMv8 Crypto Extensions
   4  * as specified in rfc8998
   5  * https://datatracker.ietf.org/doc/html/rfc8998
   6  *
   7  * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
   8  */
   9
  10 #include <linux/module.h>
  11 #include <linux/crypto.h>
  12 #include <linux/kernel.h>
  13 #include <linux/cpufeature.h>
  14 #include <asm/neon.h>
  15 #include <crypto/b128ops.h>
  16 #include <crypto/scatterwalk.h>
  17 #include <crypto/internal/aead.h>
  18 #include <crypto/internal/skcipher.h>
  19 #include <crypto/sm4.h>
  20 #include "sm4-ce.h"
  21
  22 asmlinkage void sm4_ce_pmull_ghash_setup(const u32 *rkey_enc, u8 *ghash_table);
  23 asmlinkage void pmull_ghash_update(const u8 *ghash_table, u8 *ghash,
  24                                    const u8 *src, unsigned int nblocks);
  25 asmlinkage void sm4_ce_pmull_gcm_enc(const u32 *rkey_enc, u8 *dst,
  26                                      const u8 *src, u8 *iv,
  27                                      unsigned int nbytes, u8 *ghash,
  28                                      const u8 *ghash_table, const u8 *lengths);
  29 asmlinkage void sm4_ce_pmull_gcm_dec(const u32 *rkey_enc, u8 *dst,
  30                                      const u8 *src, u8 *iv,
  31                                      unsigned int nbytes, u8 *ghash,
  32                                      const u8 *ghash_table, const u8 *lengths);
  33
  34 #define GHASH_BLOCK_SIZE        16
  35 #define GCM_IV_SIZE             12
  36
  37 struct sm4_gcm_ctx {
  38         struct sm4_ctx key;
  39         u8 ghash_table[16 * 4];
  40 };
  41
  42
  43 static int gcm_setkey(struct crypto_aead *tfm, const u8 *key,
  44                       unsigned int key_len)
  45 {
  46         struct sm4_gcm_ctx *ctx = crypto_aead_ctx(tfm);
  47
  48         if (key_len != SM4_KEY_SIZE)
  49                 return -EINVAL;
  50
  51         kernel_neon_begin();
  52
  53         sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
  54                           crypto_sm4_fk, crypto_sm4_ck);
  55         sm4_ce_pmull_ghash_setup(ctx->key.rkey_enc, ctx->ghash_table);
  56
  57         kernel_neon_end();
  58         return 0;
  59 }
  60
  61 static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
  62 {
  63         switch (authsize) {
  64         case 4:
  65         case 8:
  66         case 12 ... 16:
  67                 return 0;
  68         default:
  69                 return -EINVAL;
  70         }
  71 }
  72
  73 static void gcm_calculate_auth_mac(struct aead_request *req, u8 ghash[])
  74 {
  75         struct crypto_aead *aead = crypto_aead_reqtfm(req);
  76         struct sm4_gcm_ctx *ctx = crypto_aead_ctx(aead);
  77         u8 __aligned(8) buffer[GHASH_BLOCK_SIZE];
  78         u32 assoclen = req->assoclen;
  79         struct scatter_walk walk;
  80         unsigned int buflen = 0;
  81
  82         scatterwalk_start(&walk, req->src);
  83
  84         do {
  85                 u32 n = scatterwalk_clamp(&walk, assoclen);
  86                 u8 *p, *ptr;
  87
  88                 if (!n) {
  89                         scatterwalk_start(&walk, sg_next(walk.sg));
  90                         n = scatterwalk_clamp(&walk, assoclen);
  91                 }
  92
  93                 p = ptr = scatterwalk_map(&walk);
  94                 assoclen -= n;
  95                 scatterwalk_advance(&walk, n);
  96
  97                 if (n + buflen < GHASH_BLOCK_SIZE) {
  98                         memcpy(&buffer[buflen], ptr, n);
  99                         buflen += n;
 100                 } else {
 101                         unsigned int nblocks;
 102
 103                         if (buflen) {
 104                                 unsigned int l = GHASH_BLOCK_SIZE - buflen;
 105
 106                                 memcpy(&buffer[buflen], ptr, l);
 107                                 ptr += l;
 108                                 n -= l;
 109
 110                                 pmull_ghash_update(ctx->ghash_table, ghash,
 111                                                    buffer, 1);
 112                         }
 113
 114                         nblocks = n / GHASH_BLOCK_SIZE;
 115                         if (nblocks) {
 116                                 pmull_ghash_update(ctx->ghash_table, ghash,
 117                                                    ptr, nblocks);
 118                                 ptr += nblocks * GHASH_BLOCK_SIZE;
 119                         }
 120
 121                         buflen = n % GHASH_BLOCK_SIZE;
 122                         if (buflen)
 123                                 memcpy(&buffer[0], ptr, buflen);
 124                 }
 125
 126                 scatterwalk_unmap(p);
 127                 scatterwalk_done(&walk, 0, assoclen);
 128         } while (assoclen);
 129
 130         /* padding with '0' */
 131         if (buflen) {
 132                 memset(&buffer[buflen], 0, GHASH_BLOCK_SIZE - buflen);
 133                 pmull_ghash_update(ctx->ghash_table, ghash, buffer, 1);
 134         }
 135 }
 136
 137 static int gcm_crypt(struct aead_request *req, struct skcipher_walk *walk,
 138                      u8 ghash[], int err,
 139                      void (*sm4_ce_pmull_gcm_crypt)(const u32 *rkey_enc,
 140                                 u8 *dst, const u8 *src, u8 *iv,
 141                                 unsigned int nbytes, u8 *ghash,
 142                                 const u8 *ghash_table, const u8 *lengths))
 143 {
 144         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 145         struct sm4_gcm_ctx *ctx = crypto_aead_ctx(aead);
 146         u8 __aligned(8) iv[SM4_BLOCK_SIZE];
 147         be128 __aligned(8) lengths;
 148
 149         memset(ghash, 0, SM4_BLOCK_SIZE);
 150
 151         lengths.a = cpu_to_be64(req->assoclen * 8);
 152         lengths.b = cpu_to_be64(walk->total * 8);
 153
 154         memcpy(iv, req->iv, GCM_IV_SIZE);
 155         put_unaligned_be32(2, iv + GCM_IV_SIZE);
 156
 157         kernel_neon_begin();
 158
 159         if (req->assoclen)
 160                 gcm_calculate_auth_mac(req, ghash);
 161
 162         while (walk->nbytes) {
 163                 unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE;
 164                 const u8 *src = walk->src.virt.addr;
 165                 u8 *dst = walk->dst.virt.addr;
 166
 167                 if (walk->nbytes == walk->total) {
 168                         sm4_ce_pmull_gcm_crypt(ctx->key.rkey_enc, dst, src, iv,
 169                                                walk->nbytes, ghash,
 170                                                ctx->ghash_table,
 171                                                (const u8 *)&lengths);
 172
 173                         kernel_neon_end();
 174
 175                         return skcipher_walk_done(walk, 0);
 176                 }
 177
 178                 sm4_ce_pmull_gcm_crypt(ctx->key.rkey_enc, dst, src, iv,
 179                                        walk->nbytes - tail, ghash,
 180                                        ctx->ghash_table, NULL);
 181
 182                 kernel_neon_end();
 183
 184                 err = skcipher_walk_done(walk, tail);
 185
 186                 kernel_neon_begin();
 187         }
 188
 189         sm4_ce_pmull_gcm_crypt(ctx->key.rkey_enc, NULL, NULL, iv,
 190                                walk->nbytes, ghash, ctx->ghash_table,
 191                                (const u8 *)&lengths);
 192
 193         kernel_neon_end();
 194
 195         return err;
 196 }
 197
 198 static int gcm_encrypt(struct aead_request *req)
 199 {
 200         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 201         u8 __aligned(8) ghash[SM4_BLOCK_SIZE];
 202         struct skcipher_walk walk;
 203         int err;
 204
 205         err = skcipher_walk_aead_encrypt(&walk, req, false);
 206         err = gcm_crypt(req, &walk, ghash, err, sm4_ce_pmull_gcm_enc);
 207         if (err)
 208                 return err;
 209
 210         /* copy authtag to end of dst */
 211         scatterwalk_map_and_copy(ghash, req->dst, req->assoclen + req->cryptlen,
 212                                  crypto_aead_authsize(aead), 1);
 213
 214         return 0;
 215 }
 216
 217 static int gcm_decrypt(struct aead_request *req)
 218 {
 219         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 220         unsigned int authsize = crypto_aead_authsize(aead);
 221         u8 __aligned(8) ghash[SM4_BLOCK_SIZE];
 222         u8 authtag[SM4_BLOCK_SIZE];
 223         struct skcipher_walk walk;
 224         int err;
 225
 226         err = skcipher_walk_aead_decrypt(&walk, req, false);
 227         err = gcm_crypt(req, &walk, ghash, err, sm4_ce_pmull_gcm_dec);
 228         if (err)
 229                 return err;
 230
 231         /* compare calculated auth tag with the stored one */
 232         scatterwalk_map_and_copy(authtag, req->src,
 233                                  req->assoclen + req->cryptlen - authsize,
 234                                  authsize, 0);
 235
 236         if (crypto_memneq(authtag, ghash, authsize))
 237                 return -EBADMSG;
 238
 239         return 0;
 240 }
 241
 242 static struct aead_alg sm4_gcm_alg = {
 243         .base = {
 244                 .cra_name               = "gcm(sm4)",
 245                 .cra_driver_name        = "gcm-sm4-ce",
 246                 .cra_priority           = 400,
 247                 .cra_blocksize          = 1,
 248                 .cra_ctxsize            = sizeof(struct sm4_gcm_ctx),
 249                 .cra_module             = THIS_MODULE,
 250         },
 251         .ivsize         = GCM_IV_SIZE,
 252         .chunksize      = SM4_BLOCK_SIZE,
 253         .maxauthsize    = SM4_BLOCK_SIZE,
 254         .setkey         = gcm_setkey,
 255         .setauthsize    = gcm_setauthsize,
 256         .encrypt        = gcm_encrypt,
 257         .decrypt        = gcm_decrypt,
 258 };
 259
 260 static int __init sm4_ce_gcm_init(void)
 261 {
 262         if (!cpu_have_named_feature(PMULL))
 263                 return -ENODEV;
 264
 265         return crypto_register_aead(&sm4_gcm_alg);
 266 }
 267
 268 static void __exit sm4_ce_gcm_exit(void)
 269 {
 270         crypto_unregister_aead(&sm4_gcm_alg);
 271 }
 272
 273 static const struct cpu_feature __maybe_unused sm4_ce_gcm_cpu_feature[] = {
 274         { cpu_feature(PMULL) },
 275         {}
 276 };
 277 MODULE_DEVICE_TABLE(cpu, sm4_ce_gcm_cpu_feature);
 278
 279 module_cpu_feature_match(SM4, sm4_ce_gcm_init);
 280 module_exit(sm4_ce_gcm_exit);
 281
 282 MODULE_DESCRIPTION("Synchronous SM4 in GCM mode using ARMv8 Crypto Extensions");
 283 MODULE_ALIAS_CRYPTO("gcm(sm4)");
 284 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
 285 MODULE_LICENSE("GPL v2");