arch/mips/crypto/crc32-mips.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions
   4  *
   5  * Module based on arm64/crypto/crc32-arm.c
   6  *
   7  * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
   8  * Copyright (C) 2018 MIPS Tech, LLC
   9  */
  10
  11 #include <linux/cpufeature.h>
  12 #include <linux/init.h>
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/string.h>
  16 #include <asm/mipsregs.h>
  17 #include <asm/unaligned.h>
  18
  19 #include <crypto/internal/hash.h>
  20
  21 enum crc_op_size {
  22         b, h, w, d,
  23 };
  24
  25 enum crc_type {
  26         crc32,
  27         crc32c,
  28 };
  29
  30 #ifndef TOOLCHAIN_SUPPORTS_CRC
  31 #define _ASM_SET_CRC(OP, SZ, TYPE)                                        \
  32 _ASM_MACRO_3R(OP, rt, rs, rt2,                                            \
  33         ".ifnc  \\rt, \\rt2\n\t"                                          \
  34         ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \
  35         ".endif\n\t"                                                      \
  36         _ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) |      \
  37                           ((SZ) <<  6) | ((TYPE) << 8))                   \
  38         _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) |      \
  39                           ((SZ) << 14) | ((TYPE) << 3)))
  40 #define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t"
  41 #else /* !TOOLCHAIN_SUPPORTS_CRC */
  42 #define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t"
  43 #define _ASM_UNSET_CRC(op, SZ, TYPE)
  44 #endif
  45
  46 #define __CRC32(crc, value, op, SZ, TYPE)               \
  47 do {                                                    \
  48         __asm__ __volatile__(                           \
  49                 ".set   push\n\t"                       \
  50                 _ASM_SET_CRC(op, SZ, TYPE)              \
  51                 #op "   %0, %1, %0\n\t"                 \
  52                 _ASM_UNSET_CRC(op, SZ, TYPE)            \
  53                 ".set   pop"                            \
  54                 : "+r" (crc)                            \
  55                 : "r" (value));                         \
  56 } while (0)
  57
  58 #define _CRC32_crc32b(crc, value)       __CRC32(crc, value, crc32b, 0, 0)
  59 #define _CRC32_crc32h(crc, value)       __CRC32(crc, value, crc32h, 1, 0)
  60 #define _CRC32_crc32w(crc, value)       __CRC32(crc, value, crc32w, 2, 0)
  61 #define _CRC32_crc32d(crc, value)       __CRC32(crc, value, crc32d, 3, 0)
  62 #define _CRC32_crc32cb(crc, value)      __CRC32(crc, value, crc32cb, 0, 1)
  63 #define _CRC32_crc32ch(crc, value)      __CRC32(crc, value, crc32ch, 1, 1)
  64 #define _CRC32_crc32cw(crc, value)      __CRC32(crc, value, crc32cw, 2, 1)
  65 #define _CRC32_crc32cd(crc, value)      __CRC32(crc, value, crc32cd, 3, 1)
  66
  67 #define _CRC32(crc, value, size, op) \
  68         _CRC32_##op##size(crc, value)
  69
  70 #define CRC32(crc, value, size) \
  71         _CRC32(crc, value, size, crc32)
  72
  73 #define CRC32C(crc, value, size) \
  74         _CRC32(crc, value, size, crc32c)
  75
  76 static u32 crc32_mips_le_hw(u32 crc_, const u8 *p, unsigned int len)
  77 {
  78         u32 crc = crc_;
  79
  80 #ifdef CONFIG_64BIT
  81         while (len >= sizeof(u64)) {
  82                 u64 value = get_unaligned_le64(p);
  83
  84                 CRC32(crc, value, d);
  85                 p += sizeof(u64);
  86                 len -= sizeof(u64);
  87         }
  88
  89         if (len & sizeof(u32)) {
  90 #else /* !CONFIG_64BIT */
  91         while (len >= sizeof(u32)) {
  92 #endif
  93                 u32 value = get_unaligned_le32(p);
  94
  95                 CRC32(crc, value, w);
  96                 p += sizeof(u32);
  97                 len -= sizeof(u32);
  98         }
  99
 100         if (len & sizeof(u16)) {
 101                 u16 value = get_unaligned_le16(p);
 102
 103                 CRC32(crc, value, h);
 104                 p += sizeof(u16);
 105         }
 106
 107         if (len & sizeof(u8)) {
 108                 u8 value = *p++;
 109
 110                 CRC32(crc, value, b);
 111         }
 112
 113         return crc;
 114 }
 115
 116 static u32 crc32c_mips_le_hw(u32 crc_, const u8 *p, unsigned int len)
 117 {
 118         u32 crc = crc_;
 119
 120 #ifdef CONFIG_64BIT
 121         while (len >= sizeof(u64)) {
 122                 u64 value = get_unaligned_le64(p);
 123
 124                 CRC32C(crc, value, d);
 125                 p += sizeof(u64);
 126                 len -= sizeof(u64);
 127         }
 128
 129         if (len & sizeof(u32)) {
 130 #else /* !CONFIG_64BIT */
 131         while (len >= sizeof(u32)) {
 132 #endif
 133                 u32 value = get_unaligned_le32(p);
 134
 135                 CRC32C(crc, value, w);
 136                 p += sizeof(u32);
 137                 len -= sizeof(u32);
 138         }
 139
 140         if (len & sizeof(u16)) {
 141                 u16 value = get_unaligned_le16(p);
 142
 143                 CRC32C(crc, value, h);
 144                 p += sizeof(u16);
 145         }
 146
 147         if (len & sizeof(u8)) {
 148                 u8 value = *p++;
 149
 150                 CRC32C(crc, value, b);
 151         }
 152         return crc;
 153 }
 154
 155 #define CHKSUM_BLOCK_SIZE       1
 156 #define CHKSUM_DIGEST_SIZE      4
 157
 158 struct chksum_ctx {
 159         u32 key;
 160 };
 161
 162 struct chksum_desc_ctx {
 163         u32 crc;
 164 };
 165
 166 static int chksum_init(struct shash_desc *desc)
 167 {
 168         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 169         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 170
 171         ctx->crc = mctx->key;
 172
 173         return 0;
 174 }
 175
 176 /*
 177  * Setting the seed allows arbitrary accumulators and flexible XOR policy
 178  * If your algorithm starts with ~0, then XOR with ~0 before you set
 179  * the seed.
 180  */
 181 static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
 182                          unsigned int keylen)
 183 {
 184         struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
 185
 186         if (keylen != sizeof(mctx->key))
 187                 return -EINVAL;
 188         mctx->key = get_unaligned_le32(key);
 189         return 0;
 190 }
 191
 192 static int chksum_update(struct shash_desc *desc, const u8 *data,
 193                          unsigned int length)
 194 {
 195         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 196
 197         ctx->crc = crc32_mips_le_hw(ctx->crc, data, length);
 198         return 0;
 199 }
 200
 201 static int chksumc_update(struct shash_desc *desc, const u8 *data,
 202                          unsigned int length)
 203 {
 204         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 205
 206         ctx->crc = crc32c_mips_le_hw(ctx->crc, data, length);
 207         return 0;
 208 }
 209
 210 static int chksum_final(struct shash_desc *desc, u8 *out)
 211 {
 212         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 213
 214         put_unaligned_le32(ctx->crc, out);
 215         return 0;
 216 }
 217
 218 static int chksumc_final(struct shash_desc *desc, u8 *out)
 219 {
 220         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 221
 222         put_unaligned_le32(~ctx->crc, out);
 223         return 0;
 224 }
 225
 226 static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 227 {
 228         put_unaligned_le32(crc32_mips_le_hw(crc, data, len), out);
 229         return 0;
 230 }
 231
 232 static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 233 {
 234         put_unaligned_le32(~crc32c_mips_le_hw(crc, data, len), out);
 235         return 0;
 236 }
 237
 238 static int chksum_finup(struct shash_desc *desc, const u8 *data,
 239                         unsigned int len, u8 *out)
 240 {
 241         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 242
 243         return __chksum_finup(ctx->crc, data, len, out);
 244 }
 245
 246 static int chksumc_finup(struct shash_desc *desc, const u8 *data,
 247                         unsigned int len, u8 *out)
 248 {
 249         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 250
 251         return __chksumc_finup(ctx->crc, data, len, out);
 252 }
 253
 254 static int chksum_digest(struct shash_desc *desc, const u8 *data,
 255                          unsigned int length, u8 *out)
 256 {
 257         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 258
 259         return __chksum_finup(mctx->key, data, length, out);
 260 }
 261
 262 static int chksumc_digest(struct shash_desc *desc, const u8 *data,
 263                          unsigned int length, u8 *out)
 264 {
 265         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 266
 267         return __chksumc_finup(mctx->key, data, length, out);
 268 }
 269
 270 static int chksum_cra_init(struct crypto_tfm *tfm)
 271 {
 272         struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
 273
 274         mctx->key = ~0;
 275         return 0;
 276 }
 277
 278 static struct shash_alg crc32_alg = {
 279         .digestsize             =       CHKSUM_DIGEST_SIZE,
 280         .setkey                 =       chksum_setkey,
 281         .init                   =       chksum_init,
 282         .update                 =       chksum_update,
 283         .final                  =       chksum_final,
 284         .finup                  =       chksum_finup,
 285         .digest                 =       chksum_digest,
 286         .descsize               =       sizeof(struct chksum_desc_ctx),
 287         .base                   =       {
 288                 .cra_name               =       "crc32",
 289                 .cra_driver_name        =       "crc32-mips-hw",
 290                 .cra_priority           =       300,
 291                 .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
 292                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 293                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 294                 .cra_module             =       THIS_MODULE,
 295                 .cra_init               =       chksum_cra_init,
 296         }
 297 };
 298
 299 static struct shash_alg crc32c_alg = {
 300         .digestsize             =       CHKSUM_DIGEST_SIZE,
 301         .setkey                 =       chksum_setkey,
 302         .init                   =       chksum_init,
 303         .update                 =       chksumc_update,
 304         .final                  =       chksumc_final,
 305         .finup                  =       chksumc_finup,
 306         .digest                 =       chksumc_digest,
 307         .descsize               =       sizeof(struct chksum_desc_ctx),
 308         .base                   =       {
 309                 .cra_name               =       "crc32c",
 310                 .cra_driver_name        =       "crc32c-mips-hw",
 311                 .cra_priority           =       300,
 312                 .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
 313                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 314                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 315                 .cra_module             =       THIS_MODULE,
 316                 .cra_init               =       chksum_cra_init,
 317         }
 318 };
 319
 320 static int __init crc32_mod_init(void)
 321 {
 322         int err;
 323
 324         err = crypto_register_shash(&crc32_alg);
 325
 326         if (err)
 327                 return err;
 328
 329         err = crypto_register_shash(&crc32c_alg);
 330
 331         if (err) {
 332                 crypto_unregister_shash(&crc32_alg);
 333                 return err;
 334         }
 335
 336         return 0;
 337 }
 338
 339 static void __exit crc32_mod_exit(void)
 340 {
 341         crypto_unregister_shash(&crc32_alg);
 342         crypto_unregister_shash(&crc32c_alg);
 343 }
 344
 345 MODULE_AUTHOR("Marcin Nowakowski <marcin.nowakowski@mips.com");
 346 MODULE_DESCRIPTION("CRC32 and CRC32C using optional MIPS instructions");
 347 MODULE_LICENSE("GPL v2");
 348
 349 module_cpu_feature_match(MIPS_CRC32, crc32_mod_init);
 350 module_exit(crc32_mod_exit);