1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Glue code for accelerated AES-GCM stitched implementation for ppc64le.
5 * Copyright 2022- IBM Inc. All rights reserved
8 #include <asm/unaligned.h>
10 #include <asm/switch_to.h>
11 #include <crypto/aes.h>
12 #include <crypto/algapi.h>
13 #include <crypto/b128ops.h>
14 #include <crypto/gf128mul.h>
15 #include <crypto/internal/simd.h>
16 #include <crypto/internal/aead.h>
17 #include <crypto/internal/hash.h>
18 #include <crypto/internal/skcipher.h>
19 #include <crypto/scatterwalk.h>
20 #include <linux/cpufeature.h>
21 #include <linux/crypto.h>
22 #include <linux/module.h>
23 #include <linux/types.h>
26 #define GCM_IV_SIZE 12
28 MODULE_DESCRIPTION("PPC64le AES-GCM with Stitched implementation");
29 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com");
30 MODULE_LICENSE("GPL v2");
31 MODULE_ALIAS_CRYPTO("aes");
33 asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits,
35 asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key);
36 asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len,
37 void *rkey, u8 *iv, void *Xi);
38 asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len,
39 void *rkey, u8 *iv, void *Xi);
40 asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]);
41 asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable,
42 unsigned char *aad, unsigned int alen);
45 u8 key[AES_MAX_KEYLENGTH];
54 u64 Plen; /* offset 56 - used in aes_p10_gcm_{en/de}crypt */
57 u8 H[16]; /* subkey */
58 u8 Htable[256]; /* Xi, Hash table(offset 32) */
61 struct p10_aes_gcm_ctx {
62 struct aes_key enc_key;
65 static void vsx_begin(void)
71 static void vsx_end(void)
77 static void set_subkey(unsigned char *hash)
79 *(u64 *)&hash[0] = be64_to_cpup((__be64 *)&hash[0]);
80 *(u64 *)&hash[8] = be64_to_cpup((__be64 *)&hash[8]);
85 * - Hash aad and copy to Xi.
87 static void set_aad(struct gcm_ctx *gctx, struct Hash_ctx *hash,
88 unsigned char *aad, int alen)
96 gcm_ghash_p10(nXi, hash->Htable+32, aad, i);
101 for (i = 0; i < alen; i++)
104 memset(gctx->aad_hash, 0, 16);
105 gcm_ghash_p10(gctx->aad_hash, hash->Htable+32, nXi, 16);
107 memcpy(gctx->aad_hash, nXi, 16);
110 memcpy(hash->Htable, gctx->aad_hash, 16);
113 static void gcmp10_init(struct gcm_ctx *gctx, u8 *iv, unsigned char *rdkey,
114 struct Hash_ctx *hash, u8 *assoc, unsigned int assoclen)
116 __be32 counter = cpu_to_be32(1);
118 aes_p10_encrypt(hash->H, hash->H, rdkey);
120 gcm_init_htable(hash->Htable+32, hash->H);
122 *((__be32 *)(iv+12)) = counter;
127 * Encrypt counter vector as iv tag and increment counter.
129 aes_p10_encrypt(iv, gctx->ivtag, rdkey);
131 counter = cpu_to_be32(2);
132 *((__be32 *)(iv+12)) = counter;
133 memcpy(gctx->iv, iv, 16);
135 gctx->aadLen = assoclen;
136 memset(gctx->aad_hash, 0, 16);
138 set_aad(gctx, hash, assoc, assoclen);
141 static void finish_tag(struct gcm_ctx *gctx, struct Hash_ctx *hash, int len)
144 unsigned char len_ac[16 + PPC_ALIGN];
145 unsigned char *aclen = PTR_ALIGN((void *)len_ac, PPC_ALIGN);
146 __be64 clen = cpu_to_be64(len << 3);
147 __be64 alen = cpu_to_be64(gctx->aadLen << 3);
149 if (len == 0 && gctx->aadLen == 0) {
150 memcpy(hash->Htable, gctx->ivtag, 16);
157 *((__be64 *)(aclen)) = alen;
158 *((__be64 *)(aclen+8)) = clen;
161 * hash (AAD len and len)
163 gcm_ghash_p10(hash->Htable, hash->Htable+32, aclen, 16);
165 for (i = 0; i < 16; i++)
166 hash->Htable[i] ^= gctx->ivtag[i];
169 static int set_authsize(struct crypto_aead *tfm, unsigned int authsize)
187 static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
190 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
191 struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
195 ret = aes_p10_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
198 return ret ? -EINVAL : 0;
201 static int p10_aes_gcm_crypt(struct aead_request *req, int enc)
203 struct crypto_tfm *tfm = req->base.tfm;
204 struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
205 u8 databuf[sizeof(struct gcm_ctx) + PPC_ALIGN];
206 struct gcm_ctx *gctx = PTR_ALIGN((void *)databuf, PPC_ALIGN);
207 u8 hashbuf[sizeof(struct Hash_ctx) + PPC_ALIGN];
208 struct Hash_ctx *hash = PTR_ALIGN((void *)hashbuf, PPC_ALIGN);
209 struct scatter_walk assoc_sg_walk;
210 struct skcipher_walk walk;
213 unsigned int assoclen = req->assoclen;
214 unsigned int cryptlen = req->cryptlen;
215 unsigned char ivbuf[AES_BLOCK_SIZE+PPC_ALIGN];
216 unsigned char *iv = PTR_ALIGN((void *)ivbuf, PPC_ALIGN);
218 unsigned long auth_tag_len = crypto_aead_authsize(__crypto_aead_cast(tfm));
220 int total_processed = 0;
222 memset(databuf, 0, sizeof(databuf));
223 memset(hashbuf, 0, sizeof(hashbuf));
224 memset(ivbuf, 0, sizeof(ivbuf));
225 memcpy(iv, req->iv, GCM_IV_SIZE);
227 /* Linearize assoc, if not already linear */
228 if (req->src->length >= assoclen && req->src->length) {
229 scatterwalk_start(&assoc_sg_walk, req->src);
230 assoc = scatterwalk_map(&assoc_sg_walk);
232 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
233 GFP_KERNEL : GFP_ATOMIC;
235 /* assoc can be any length, so must be on heap */
236 assocmem = kmalloc(assoclen, flags);
237 if (unlikely(!assocmem))
241 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
245 gcmp10_init(gctx, iv, (unsigned char *) &ctx->enc_key, hash, assoc, assoclen);
249 scatterwalk_unmap(assoc);
254 ret = skcipher_walk_aead_encrypt(&walk, req, false);
256 ret = skcipher_walk_aead_decrypt(&walk, req, false);
260 while (walk.nbytes > 0 && ret == 0) {
264 aes_p10_gcm_encrypt(walk.src.virt.addr,
267 &ctx->enc_key, gctx->iv, hash->Htable);
269 aes_p10_gcm_decrypt(walk.src.virt.addr,
272 &ctx->enc_key, gctx->iv, hash->Htable);
275 total_processed += walk.nbytes;
276 ret = skcipher_walk_done(&walk, 0);
284 finish_tag(gctx, hash, total_processed);
287 /* copy Xi to end of dst */
289 scatterwalk_map_and_copy(hash->Htable, req->dst, req->assoclen + cryptlen,
292 scatterwalk_map_and_copy(otag, req->src,
293 req->assoclen + cryptlen - auth_tag_len,
296 if (crypto_memneq(otag, hash->Htable, auth_tag_len)) {
297 memzero_explicit(hash->Htable, 16);
305 static int p10_aes_gcm_encrypt(struct aead_request *req)
307 return p10_aes_gcm_crypt(req, 1);
310 static int p10_aes_gcm_decrypt(struct aead_request *req)
312 return p10_aes_gcm_crypt(req, 0);
315 static struct aead_alg gcm_aes_alg = {
316 .ivsize = GCM_IV_SIZE,
319 .setauthsize = set_authsize,
320 .setkey = p10_aes_gcm_setkey,
321 .encrypt = p10_aes_gcm_encrypt,
322 .decrypt = p10_aes_gcm_decrypt,
324 .base.cra_name = "gcm(aes)",
325 .base.cra_driver_name = "aes_gcm_p10",
326 .base.cra_priority = 2100,
327 .base.cra_blocksize = 1,
328 .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx),
329 .base.cra_module = THIS_MODULE,
332 static int __init p10_init(void)
334 return crypto_register_aead(&gcm_aes_alg);
337 static void __exit p10_exit(void)
339 crypto_unregister_aead(&gcm_aes_alg);
342 module_cpu_feature_match(PPC_MODULE_FEATURE_P10, p10_init);
343 module_exit(p10_exit);