arch/arm/crypto/crc32-ce-glue.c

   1 /*
   2  * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
   3  *
   4  * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <linux/cpufeature.h>
  12 #include <linux/crc32.h>
  13 #include <linux/init.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/string.h>
  17
  18 #include <crypto/internal/hash.h>
  19
  20 #include <asm/hwcap.h>
  21 #include <asm/neon.h>
  22 #include <asm/simd.h>
  23 #include <asm/unaligned.h>
  24
  25 #define PMULL_MIN_LEN           64L     /* minimum size of buffer
  26                                          * for crc32_pmull_le_16 */
  27 #define SCALE_F                 16L     /* size of NEON register */
  28
  29 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
  30 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
  31
  32 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
  33 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
  34
  35 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len);
  36 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len);
  37
  38 static int crc32_cra_init(struct crypto_tfm *tfm)
  39 {
  40         u32 *key = crypto_tfm_ctx(tfm);
  41
  42         *key = 0;
  43         return 0;
  44 }
  45
  46 static int crc32c_cra_init(struct crypto_tfm *tfm)
  47 {
  48         u32 *key = crypto_tfm_ctx(tfm);
  49
  50         *key = ~0;
  51         return 0;
  52 }
  53
  54 static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
  55                         unsigned int keylen)
  56 {
  57         u32 *mctx = crypto_shash_ctx(hash);
  58
  59         if (keylen != sizeof(u32)) {
  60                 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
  61                 return -EINVAL;
  62         }
  63         *mctx = le32_to_cpup((__le32 *)key);
  64         return 0;
  65 }
  66
  67 static int crc32_init(struct shash_desc *desc)
  68 {
  69         u32 *mctx = crypto_shash_ctx(desc->tfm);
  70         u32 *crc = shash_desc_ctx(desc);
  71
  72         *crc = *mctx;
  73         return 0;
  74 }
  75
  76 static int crc32_update(struct shash_desc *desc, const u8 *data,
  77                         unsigned int length)
  78 {
  79         u32 *crc = shash_desc_ctx(desc);
  80
  81         *crc = crc32_armv8_le(*crc, data, length);
  82         return 0;
  83 }
  84
  85 static int crc32c_update(struct shash_desc *desc, const u8 *data,
  86                          unsigned int length)
  87 {
  88         u32 *crc = shash_desc_ctx(desc);
  89
  90         *crc = crc32c_armv8_le(*crc, data, length);
  91         return 0;
  92 }
  93
  94 static int crc32_final(struct shash_desc *desc, u8 *out)
  95 {
  96         u32 *crc = shash_desc_ctx(desc);
  97
  98         put_unaligned_le32(*crc, out);
  99         return 0;
 100 }
 101
 102 static int crc32c_final(struct shash_desc *desc, u8 *out)
 103 {
 104         u32 *crc = shash_desc_ctx(desc);
 105
 106         put_unaligned_le32(~*crc, out);
 107         return 0;
 108 }
 109
 110 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
 111                               unsigned int length)
 112 {
 113         u32 *crc = shash_desc_ctx(desc);
 114         unsigned int l;
 115
 116         if (may_use_simd()) {
 117                 if ((u32)data % SCALE_F) {
 118                         l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
 119
 120                         *crc = fallback_crc32(*crc, data, l);
 121
 122                         data += l;
 123                         length -= l;
 124                 }
 125
 126                 if (length >= PMULL_MIN_LEN) {
 127                         l = round_down(length, SCALE_F);
 128
 129                         kernel_neon_begin();
 130                         *crc = crc32_pmull_le(data, l, *crc);
 131                         kernel_neon_end();
 132
 133                         data += l;
 134                         length -= l;
 135                 }
 136         }
 137
 138         if (length > 0)
 139                 *crc = fallback_crc32(*crc, data, length);
 140
 141         return 0;
 142 }
 143
 144 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
 145                                unsigned int length)
 146 {
 147         u32 *crc = shash_desc_ctx(desc);
 148         unsigned int l;
 149
 150         if (may_use_simd()) {
 151                 if ((u32)data % SCALE_F) {
 152                         l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
 153
 154                         *crc = fallback_crc32c(*crc, data, l);
 155
 156                         data += l;
 157                         length -= l;
 158                 }
 159
 160                 if (length >= PMULL_MIN_LEN) {
 161                         l = round_down(length, SCALE_F);
 162
 163                         kernel_neon_begin();
 164                         *crc = crc32c_pmull_le(data, l, *crc);
 165                         kernel_neon_end();
 166
 167                         data += l;
 168                         length -= l;
 169                 }
 170         }
 171
 172         if (length > 0)
 173                 *crc = fallback_crc32c(*crc, data, length);
 174
 175         return 0;
 176 }
 177
 178 static struct shash_alg crc32_pmull_algs[] = { {
 179         .setkey                 = crc32_setkey,
 180         .init                   = crc32_init,
 181         .update                 = crc32_update,
 182         .final                  = crc32_final,
 183         .descsize               = sizeof(u32),
 184         .digestsize             = sizeof(u32),
 185
 186         .base.cra_ctxsize       = sizeof(u32),
 187         .base.cra_init          = crc32_cra_init,
 188         .base.cra_name          = "crc32",
 189         .base.cra_driver_name   = "crc32-arm-ce",
 190         .base.cra_priority      = 200,
 191         .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
 192         .base.cra_blocksize     = 1,
 193         .base.cra_module        = THIS_MODULE,
 194 }, {
 195         .setkey                 = crc32_setkey,
 196         .init                   = crc32_init,
 197         .update                 = crc32c_update,
 198         .final                  = crc32c_final,
 199         .descsize               = sizeof(u32),
 200         .digestsize             = sizeof(u32),
 201
 202         .base.cra_ctxsize       = sizeof(u32),
 203         .base.cra_init          = crc32c_cra_init,
 204         .base.cra_name          = "crc32c",
 205         .base.cra_driver_name   = "crc32c-arm-ce",
 206         .base.cra_priority      = 200,
 207         .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
 208         .base.cra_blocksize     = 1,
 209         .base.cra_module        = THIS_MODULE,
 210 } };
 211
 212 static int __init crc32_pmull_mod_init(void)
 213 {
 214         if (elf_hwcap2 & HWCAP2_PMULL) {
 215                 crc32_pmull_algs[0].update = crc32_pmull_update;
 216                 crc32_pmull_algs[1].update = crc32c_pmull_update;
 217
 218                 if (elf_hwcap2 & HWCAP2_CRC32) {
 219                         fallback_crc32 = crc32_armv8_le;
 220                         fallback_crc32c = crc32c_armv8_le;
 221                 } else {
 222                         fallback_crc32 = crc32_le;
 223                         fallback_crc32c = __crc32c_le;
 224                 }
 225         } else if (!(elf_hwcap2 & HWCAP2_CRC32)) {
 226                 return -ENODEV;
 227         }
 228
 229         return crypto_register_shashes(crc32_pmull_algs,
 230                                        ARRAY_SIZE(crc32_pmull_algs));
 231 }
 232
 233 static void __exit crc32_pmull_mod_exit(void)
 234 {
 235         crypto_unregister_shashes(crc32_pmull_algs,
 236                                   ARRAY_SIZE(crc32_pmull_algs));
 237 }
 238
 239 static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = {
 240         { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
 241 };
 242 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
 243
 244 module_init(crc32_pmull_mod_init);
 245 module_exit(crc32_pmull_mod_exit);
 246
 247 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 248 MODULE_LICENSE("GPL v2");
 249 MODULE_ALIAS_CRYPTO("crc32");
 250 MODULE_ALIAS_CRYPTO("crc32c");