drivers/crypto/vmx/ghash.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /**
   3  * GHASH routines supporting VMX instructions on the Power 8
   4  *
   5  * Copyright (C) 2015, 2019 International Business Machines Inc.
   6  *
   7  * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
   8  *
   9  * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback
  10  * mechanism. The new approach is based on arm64 code, which is:
  11  *   Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
  12  */
  13
  14 #include <linux/types.h>
  15 #include <linux/err.h>
  16 #include <linux/crypto.h>
  17 #include <linux/delay.h>
  18 #include <linux/hardirq.h>
  19 #include <asm/switch_to.h>
  20 #include <crypto/aes.h>
  21 #include <crypto/ghash.h>
  22 #include <crypto/scatterwalk.h>
  23 #include <crypto/internal/hash.h>
  24 #include <crypto/b128ops.h>
  25
  26 #define IN_INTERRUPT in_interrupt()
  27
  28 void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
  29 void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
  30 void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
  31                   const u8 *in, size_t len);
  32
  33 struct p8_ghash_ctx {
  34         /* key used by vector asm */
  35         u128 htable[16];
  36         /* key used by software fallback */
  37         be128 key;
  38 };
  39
  40 struct p8_ghash_desc_ctx {
  41         u64 shash[2];
  42         u8 buffer[GHASH_DIGEST_SIZE];
  43         int bytes;
  44 };
  45
  46 static int p8_ghash_init(struct shash_desc *desc)
  47 {
  48         struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
  49
  50         dctx->bytes = 0;
  51         memset(dctx->shash, 0, GHASH_DIGEST_SIZE);
  52         return 0;
  53 }
  54
  55 static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
  56                            unsigned int keylen)
  57 {
  58         struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm));
  59
  60         if (keylen != GHASH_BLOCK_SIZE)
  61                 return -EINVAL;
  62
  63         preempt_disable();
  64         pagefault_disable();
  65         enable_kernel_altivec();
  66         enable_kernel_vsx();
  67         enable_kernel_fp();
  68         gcm_init_p8(ctx->htable, (const u64 *) key);
  69         pagefault_enable();
  70         preempt_enable();
  71
  72         memcpy(&ctx->key, key, GHASH_BLOCK_SIZE);
  73
  74         return 0;
  75 }
  76
  77 static inline void __ghash_block(struct p8_ghash_ctx *ctx,
  78                                  struct p8_ghash_desc_ctx *dctx)
  79 {
  80         if (!IN_INTERRUPT) {
  81                 preempt_disable();
  82                 pagefault_disable();
  83                 enable_kernel_altivec();
  84                 enable_kernel_vsx();
  85                 enable_kernel_fp();
  86                 gcm_ghash_p8(dctx->shash, ctx->htable,
  87                                 dctx->buffer, GHASH_DIGEST_SIZE);
  88                 pagefault_enable();
  89                 preempt_enable();
  90         } else {
  91                 crypto_xor((u8 *)dctx->shash, dctx->buffer, GHASH_BLOCK_SIZE);
  92                 gf128mul_lle((be128 *)dctx->shash, &ctx->key);
  93         }
  94 }
  95
  96 static inline void __ghash_blocks(struct p8_ghash_ctx *ctx,
  97                                   struct p8_ghash_desc_ctx *dctx,
  98                                   const u8 *src, unsigned int srclen)
  99 {
 100         if (!IN_INTERRUPT) {
 101                 preempt_disable();
 102                 pagefault_disable();
 103                 enable_kernel_altivec();
 104                 enable_kernel_vsx();
 105                 enable_kernel_fp();
 106                 gcm_ghash_p8(dctx->shash, ctx->htable,
 107                                 src, srclen);
 108                 pagefault_enable();
 109                 preempt_enable();
 110         } else {
 111                 while (srclen >= GHASH_BLOCK_SIZE) {
 112                         crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
 113                         gf128mul_lle((be128 *)dctx->shash, &ctx->key);
 114                         srclen -= GHASH_BLOCK_SIZE;
 115                         src += GHASH_BLOCK_SIZE;
 116                 }
 117         }
 118 }
 119
 120 static int p8_ghash_update(struct shash_desc *desc,
 121                            const u8 *src, unsigned int srclen)
 122 {
 123         unsigned int len;
 124         struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
 125         struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 126
 127         if (dctx->bytes) {
 128                 if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) {
 129                         memcpy(dctx->buffer + dctx->bytes, src,
 130                                 srclen);
 131                         dctx->bytes += srclen;
 132                         return 0;
 133                 }
 134                 memcpy(dctx->buffer + dctx->bytes, src,
 135                         GHASH_DIGEST_SIZE - dctx->bytes);
 136
 137                 __ghash_block(ctx, dctx);
 138
 139                 src += GHASH_DIGEST_SIZE - dctx->bytes;
 140                 srclen -= GHASH_DIGEST_SIZE - dctx->bytes;
 141                 dctx->bytes = 0;
 142         }
 143         len = srclen & ~(GHASH_DIGEST_SIZE - 1);
 144         if (len) {
 145                 __ghash_blocks(ctx, dctx, src, len);
 146                 src += len;
 147                 srclen -= len;
 148         }
 149         if (srclen) {
 150                 memcpy(dctx->buffer, src, srclen);
 151                 dctx->bytes = srclen;
 152         }
 153         return 0;
 154 }
 155
 156 static int p8_ghash_final(struct shash_desc *desc, u8 *out)
 157 {
 158         int i;
 159         struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
 160         struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 161
 162         if (dctx->bytes) {
 163                 for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++)
 164                         dctx->buffer[i] = 0;
 165                 __ghash_block(ctx, dctx);
 166                 dctx->bytes = 0;
 167         }
 168         memcpy(out, dctx->shash, GHASH_DIGEST_SIZE);
 169         return 0;
 170 }
 171
 172 struct shash_alg p8_ghash_alg = {
 173         .digestsize = GHASH_DIGEST_SIZE,
 174         .init = p8_ghash_init,
 175         .update = p8_ghash_update,
 176         .final = p8_ghash_final,
 177         .setkey = p8_ghash_setkey,
 178         .descsize = sizeof(struct p8_ghash_desc_ctx)
 179                 + sizeof(struct ghash_desc_ctx),
 180         .base = {
 181                  .cra_name = "ghash",
 182                  .cra_driver_name = "p8_ghash",
 183                  .cra_priority = 1000,
 184                  .cra_flags = CRYPTO_ALG_TYPE_SHASH,
 185                  .cra_blocksize = GHASH_BLOCK_SIZE,
 186                  .cra_ctxsize = sizeof(struct p8_ghash_ctx),
 187                  .cra_module = THIS_MODULE,
 188         },
 189 };