arch/x86/crypto/sha1_ssse3_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Cryptographic API.
   4  *
   5  * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
   6  * Supplemental SSE3 instructions.
   7  *
   8  * This file is based on sha1_generic.c
   9  *
  10  * Copyright (c) Alan Smithee.
  11  * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
  12  * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
  13  * Copyright (c) Mathias Krause <minipli@googlemail.com>
  14  * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
  15  */
  16
  17 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  18
  19 #include <crypto/internal/hash.h>
  20 #include <crypto/internal/simd.h>
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/mm.h>
  24 #include <linux/cryptohash.h>
  25 #include <linux/types.h>
  26 #include <crypto/sha.h>
  27 #include <crypto/sha1_base.h>
  28 #include <asm/simd.h>
  29
  30 typedef void (sha1_transform_fn)(u32 *digest, const char *data,
  31                                 unsigned int rounds);
  32
  33 static int sha1_update(struct shash_desc *desc, const u8 *data,
  34                              unsigned int len, sha1_transform_fn *sha1_xform)
  35 {
  36         struct sha1_state *sctx = shash_desc_ctx(desc);
  37
  38         if (!crypto_simd_usable() ||
  39             (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
  40                 return crypto_sha1_update(desc, data, len);
  41
  42         /* make sure casting to sha1_block_fn() is safe */
  43         BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
  44
  45         kernel_fpu_begin();
  46         sha1_base_do_update(desc, data, len,
  47                             (sha1_block_fn *)sha1_xform);
  48         kernel_fpu_end();
  49
  50         return 0;
  51 }
  52
  53 static int sha1_finup(struct shash_desc *desc, const u8 *data,
  54                       unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
  55 {
  56         if (!crypto_simd_usable())
  57                 return crypto_sha1_finup(desc, data, len, out);
  58
  59         kernel_fpu_begin();
  60         if (len)
  61                 sha1_base_do_update(desc, data, len,
  62                                     (sha1_block_fn *)sha1_xform);
  63         sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform);
  64         kernel_fpu_end();
  65
  66         return sha1_base_finish(desc, out);
  67 }
  68
  69 asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
  70                                      unsigned int rounds);
  71
  72 static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
  73                              unsigned int len)
  74 {
  75         return sha1_update(desc, data, len,
  76                         (sha1_transform_fn *) sha1_transform_ssse3);
  77 }
  78
  79 static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
  80                               unsigned int len, u8 *out)
  81 {
  82         return sha1_finup(desc, data, len, out,
  83                         (sha1_transform_fn *) sha1_transform_ssse3);
  84 }
  85
  86 /* Add padding and return the message digest. */
  87 static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
  88 {
  89         return sha1_ssse3_finup(desc, NULL, 0, out);
  90 }
  91
  92 static struct shash_alg sha1_ssse3_alg = {
  93         .digestsize     =       SHA1_DIGEST_SIZE,
  94         .init           =       sha1_base_init,
  95         .update         =       sha1_ssse3_update,
  96         .final          =       sha1_ssse3_final,
  97         .finup          =       sha1_ssse3_finup,
  98         .descsize       =       sizeof(struct sha1_state),
  99         .base           =       {
 100                 .cra_name       =       "sha1",
 101                 .cra_driver_name =      "sha1-ssse3",
 102                 .cra_priority   =       150,
 103                 .cra_blocksize  =       SHA1_BLOCK_SIZE,
 104                 .cra_module     =       THIS_MODULE,
 105         }
 106 };
 107
 108 static int register_sha1_ssse3(void)
 109 {
 110         if (boot_cpu_has(X86_FEATURE_SSSE3))
 111                 return crypto_register_shash(&sha1_ssse3_alg);
 112         return 0;
 113 }
 114
 115 static void unregister_sha1_ssse3(void)
 116 {
 117         if (boot_cpu_has(X86_FEATURE_SSSE3))
 118                 crypto_unregister_shash(&sha1_ssse3_alg);
 119 }
 120
 121 #ifdef CONFIG_AS_AVX
 122 asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
 123                                    unsigned int rounds);
 124
 125 static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
 126                              unsigned int len)
 127 {
 128         return sha1_update(desc, data, len,
 129                         (sha1_transform_fn *) sha1_transform_avx);
 130 }
 131
 132 static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
 133                               unsigned int len, u8 *out)
 134 {
 135         return sha1_finup(desc, data, len, out,
 136                         (sha1_transform_fn *) sha1_transform_avx);
 137 }
 138
 139 static int sha1_avx_final(struct shash_desc *desc, u8 *out)
 140 {
 141         return sha1_avx_finup(desc, NULL, 0, out);
 142 }
 143
 144 static struct shash_alg sha1_avx_alg = {
 145         .digestsize     =       SHA1_DIGEST_SIZE,
 146         .init           =       sha1_base_init,
 147         .update         =       sha1_avx_update,
 148         .final          =       sha1_avx_final,
 149         .finup          =       sha1_avx_finup,
 150         .descsize       =       sizeof(struct sha1_state),
 151         .base           =       {
 152                 .cra_name       =       "sha1",
 153                 .cra_driver_name =      "sha1-avx",
 154                 .cra_priority   =       160,
 155                 .cra_blocksize  =       SHA1_BLOCK_SIZE,
 156                 .cra_module     =       THIS_MODULE,
 157         }
 158 };
 159
 160 static bool avx_usable(void)
 161 {
 162         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
 163                 if (boot_cpu_has(X86_FEATURE_AVX))
 164                         pr_info("AVX detected but unusable.\n");
 165                 return false;
 166         }
 167
 168         return true;
 169 }
 170
 171 static int register_sha1_avx(void)
 172 {
 173         if (avx_usable())
 174                 return crypto_register_shash(&sha1_avx_alg);
 175         return 0;
 176 }
 177
 178 static void unregister_sha1_avx(void)
 179 {
 180         if (avx_usable())
 181                 crypto_unregister_shash(&sha1_avx_alg);
 182 }
 183
 184 #else  /* CONFIG_AS_AVX */
 185 static inline int register_sha1_avx(void) { return 0; }
 186 static inline void unregister_sha1_avx(void) { }
 187 #endif /* CONFIG_AS_AVX */
 188
 189
 190 #if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
 191 #define SHA1_AVX2_BLOCK_OPTSIZE 4       /* optimal 4*64 bytes of SHA1 blocks */
 192
 193 asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
 194                                     unsigned int rounds);
 195
 196 static bool avx2_usable(void)
 197 {
 198         if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
 199                 && boot_cpu_has(X86_FEATURE_BMI1)
 200                 && boot_cpu_has(X86_FEATURE_BMI2))
 201                 return true;
 202
 203         return false;
 204 }
 205
 206 static void sha1_apply_transform_avx2(u32 *digest, const char *data,
 207                                 unsigned int rounds)
 208 {
 209         /* Select the optimal transform based on data block size */
 210         if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
 211                 sha1_transform_avx2(digest, data, rounds);
 212         else
 213                 sha1_transform_avx(digest, data, rounds);
 214 }
 215
 216 static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
 217                              unsigned int len)
 218 {
 219         return sha1_update(desc, data, len,
 220                 (sha1_transform_fn *) sha1_apply_transform_avx2);
 221 }
 222
 223 static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
 224                               unsigned int len, u8 *out)
 225 {
 226         return sha1_finup(desc, data, len, out,
 227                 (sha1_transform_fn *) sha1_apply_transform_avx2);
 228 }
 229
 230 static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
 231 {
 232         return sha1_avx2_finup(desc, NULL, 0, out);
 233 }
 234
 235 static struct shash_alg sha1_avx2_alg = {
 236         .digestsize     =       SHA1_DIGEST_SIZE,
 237         .init           =       sha1_base_init,
 238         .update         =       sha1_avx2_update,
 239         .final          =       sha1_avx2_final,
 240         .finup          =       sha1_avx2_finup,
 241         .descsize       =       sizeof(struct sha1_state),
 242         .base           =       {
 243                 .cra_name       =       "sha1",
 244                 .cra_driver_name =      "sha1-avx2",
 245                 .cra_priority   =       170,
 246                 .cra_blocksize  =       SHA1_BLOCK_SIZE,
 247                 .cra_module     =       THIS_MODULE,
 248         }
 249 };
 250
 251 static int register_sha1_avx2(void)
 252 {
 253         if (avx2_usable())
 254                 return crypto_register_shash(&sha1_avx2_alg);
 255         return 0;
 256 }
 257
 258 static void unregister_sha1_avx2(void)
 259 {
 260         if (avx2_usable())
 261                 crypto_unregister_shash(&sha1_avx2_alg);
 262 }
 263
 264 #else
 265 static inline int register_sha1_avx2(void) { return 0; }
 266 static inline void unregister_sha1_avx2(void) { }
 267 #endif
 268
 269 #ifdef CONFIG_AS_SHA1_NI
 270 asmlinkage void sha1_ni_transform(u32 *digest, const char *data,
 271                                    unsigned int rounds);
 272
 273 static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
 274                              unsigned int len)
 275 {
 276         return sha1_update(desc, data, len,
 277                 (sha1_transform_fn *) sha1_ni_transform);
 278 }
 279
 280 static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
 281                               unsigned int len, u8 *out)
 282 {
 283         return sha1_finup(desc, data, len, out,
 284                 (sha1_transform_fn *) sha1_ni_transform);
 285 }
 286
 287 static int sha1_ni_final(struct shash_desc *desc, u8 *out)
 288 {
 289         return sha1_ni_finup(desc, NULL, 0, out);
 290 }
 291
 292 static struct shash_alg sha1_ni_alg = {
 293         .digestsize     =       SHA1_DIGEST_SIZE,
 294         .init           =       sha1_base_init,
 295         .update         =       sha1_ni_update,
 296         .final          =       sha1_ni_final,
 297         .finup          =       sha1_ni_finup,
 298         .descsize       =       sizeof(struct sha1_state),
 299         .base           =       {
 300                 .cra_name       =       "sha1",
 301                 .cra_driver_name =      "sha1-ni",
 302                 .cra_priority   =       250,
 303                 .cra_blocksize  =       SHA1_BLOCK_SIZE,
 304                 .cra_module     =       THIS_MODULE,
 305         }
 306 };
 307
 308 static int register_sha1_ni(void)
 309 {
 310         if (boot_cpu_has(X86_FEATURE_SHA_NI))
 311                 return crypto_register_shash(&sha1_ni_alg);
 312         return 0;
 313 }
 314
 315 static void unregister_sha1_ni(void)
 316 {
 317         if (boot_cpu_has(X86_FEATURE_SHA_NI))
 318                 crypto_unregister_shash(&sha1_ni_alg);
 319 }
 320
 321 #else
 322 static inline int register_sha1_ni(void) { return 0; }
 323 static inline void unregister_sha1_ni(void) { }
 324 #endif
 325
 326 static int __init sha1_ssse3_mod_init(void)
 327 {
 328         if (register_sha1_ssse3())
 329                 goto fail;
 330
 331         if (register_sha1_avx()) {
 332                 unregister_sha1_ssse3();
 333                 goto fail;
 334         }
 335
 336         if (register_sha1_avx2()) {
 337                 unregister_sha1_avx();
 338                 unregister_sha1_ssse3();
 339                 goto fail;
 340         }
 341
 342         if (register_sha1_ni()) {
 343                 unregister_sha1_avx2();
 344                 unregister_sha1_avx();
 345                 unregister_sha1_ssse3();
 346                 goto fail;
 347         }
 348
 349         return 0;
 350 fail:
 351         return -ENODEV;
 352 }
 353
 354 static void __exit sha1_ssse3_mod_fini(void)
 355 {
 356         unregister_sha1_ni();
 357         unregister_sha1_avx2();
 358         unregister_sha1_avx();
 359         unregister_sha1_ssse3();
 360 }
 361
 362 module_init(sha1_ssse3_mod_init);
 363 module_exit(sha1_ssse3_mod_fini);
 364
 365 MODULE_LICENSE("GPL");
 366 MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 367
 368 MODULE_ALIAS_CRYPTO("sha1");
 369 MODULE_ALIAS_CRYPTO("sha1-ssse3");
 370 MODULE_ALIAS_CRYPTO("sha1-avx");
 371 MODULE_ALIAS_CRYPTO("sha1-avx2");
 372 #ifdef CONFIG_AS_SHA1_NI
 373 MODULE_ALIAS_CRYPTO("sha1-ni");
 374 #endif