arch/loongarch/crypto/crc32-loongarch.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * crc32.c - CRC32 and CRC32C using LoongArch crc* instructions
   4  *
   5  * Module based on mips/crypto/crc32-mips.c
   6  *
   7  * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
   8  * Copyright (C) 2018 MIPS Tech, LLC
   9  * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
  10  */
  11
  12 #include <linux/module.h>
  13 #include <crypto/internal/hash.h>
  14
  15 #include <asm/cpu-features.h>
  16 #include <asm/unaligned.h>
  17
  18 #define _CRC32(crc, value, size, type)                  \
  19 do {                                                    \
  20         __asm__ __volatile__(                           \
  21                 #type ".w." #size ".w" " %0, %1, %0\n\t"\
  22                 : "+r" (crc)                            \
  23                 : "r" (value)                           \
  24                 : "memory");                            \
  25 } while (0)
  26
  27 #define CRC32(crc, value, size)         _CRC32(crc, value, size, crc)
  28 #define CRC32C(crc, value, size)        _CRC32(crc, value, size, crcc)
  29
  30 static u32 crc32_loongarch_hw(u32 crc_, const u8 *p, unsigned int len)
  31 {
  32         u32 crc = crc_;
  33
  34         while (len >= sizeof(u64)) {
  35                 u64 value = get_unaligned_le64(p);
  36
  37                 CRC32(crc, value, d);
  38                 p += sizeof(u64);
  39                 len -= sizeof(u64);
  40         }
  41
  42         if (len & sizeof(u32)) {
  43                 u32 value = get_unaligned_le32(p);
  44
  45                 CRC32(crc, value, w);
  46                 p += sizeof(u32);
  47                 len -= sizeof(u32);
  48         }
  49
  50         if (len & sizeof(u16)) {
  51                 u16 value = get_unaligned_le16(p);
  52
  53                 CRC32(crc, value, h);
  54                 p += sizeof(u16);
  55         }
  56
  57         if (len & sizeof(u8)) {
  58                 u8 value = *p++;
  59
  60                 CRC32(crc, value, b);
  61         }
  62
  63         return crc;
  64 }
  65
  66 static u32 crc32c_loongarch_hw(u32 crc_, const u8 *p, unsigned int len)
  67 {
  68         u32 crc = crc_;
  69
  70         while (len >= sizeof(u64)) {
  71                 u64 value = get_unaligned_le64(p);
  72
  73                 CRC32C(crc, value, d);
  74                 p += sizeof(u64);
  75                 len -= sizeof(u64);
  76         }
  77
  78         if (len & sizeof(u32)) {
  79                 u32 value = get_unaligned_le32(p);
  80
  81                 CRC32C(crc, value, w);
  82                 p += sizeof(u32);
  83                 len -= sizeof(u32);
  84         }
  85
  86         if (len & sizeof(u16)) {
  87                 u16 value = get_unaligned_le16(p);
  88
  89                 CRC32C(crc, value, h);
  90                 p += sizeof(u16);
  91         }
  92
  93         if (len & sizeof(u8)) {
  94                 u8 value = *p++;
  95
  96                 CRC32C(crc, value, b);
  97         }
  98
  99         return crc;
 100 }
 101
 102 #define CHKSUM_BLOCK_SIZE       1
 103 #define CHKSUM_DIGEST_SIZE      4
 104
 105 struct chksum_ctx {
 106         u32 key;
 107 };
 108
 109 struct chksum_desc_ctx {
 110         u32 crc;
 111 };
 112
 113 static int chksum_init(struct shash_desc *desc)
 114 {
 115         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 116         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 117
 118         ctx->crc = mctx->key;
 119
 120         return 0;
 121 }
 122
 123 /*
 124  * Setting the seed allows arbitrary accumulators and flexible XOR policy
 125  * If your algorithm starts with ~0, then XOR with ~0 before you set the seed.
 126  */
 127 static int chksum_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen)
 128 {
 129         struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
 130
 131         if (keylen != sizeof(mctx->key))
 132                 return -EINVAL;
 133
 134         mctx->key = get_unaligned_le32(key);
 135
 136         return 0;
 137 }
 138
 139 static int chksum_update(struct shash_desc *desc, const u8 *data, unsigned int length)
 140 {
 141         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 142
 143         ctx->crc = crc32_loongarch_hw(ctx->crc, data, length);
 144         return 0;
 145 }
 146
 147 static int chksumc_update(struct shash_desc *desc, const u8 *data, unsigned int length)
 148 {
 149         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 150
 151         ctx->crc = crc32c_loongarch_hw(ctx->crc, data, length);
 152         return 0;
 153 }
 154
 155 static int chksum_final(struct shash_desc *desc, u8 *out)
 156 {
 157         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 158
 159         put_unaligned_le32(ctx->crc, out);
 160         return 0;
 161 }
 162
 163 static int chksumc_final(struct shash_desc *desc, u8 *out)
 164 {
 165         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 166
 167         put_unaligned_le32(~ctx->crc, out);
 168         return 0;
 169 }
 170
 171 static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 172 {
 173         put_unaligned_le32(crc32_loongarch_hw(crc, data, len), out);
 174         return 0;
 175 }
 176
 177 static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
 178 {
 179         put_unaligned_le32(~crc32c_loongarch_hw(crc, data, len), out);
 180         return 0;
 181 }
 182
 183 static int chksum_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out)
 184 {
 185         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 186
 187         return __chksum_finup(ctx->crc, data, len, out);
 188 }
 189
 190 static int chksumc_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out)
 191 {
 192         struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 193
 194         return __chksumc_finup(ctx->crc, data, len, out);
 195 }
 196
 197 static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out)
 198 {
 199         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 200
 201         return __chksum_finup(mctx->key, data, length, out);
 202 }
 203
 204 static int chksumc_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out)
 205 {
 206         struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
 207
 208         return __chksumc_finup(mctx->key, data, length, out);
 209 }
 210
 211 static int chksum_cra_init(struct crypto_tfm *tfm)
 212 {
 213         struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
 214
 215         mctx->key = 0;
 216         return 0;
 217 }
 218
 219 static int chksumc_cra_init(struct crypto_tfm *tfm)
 220 {
 221         struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
 222
 223         mctx->key = ~0;
 224         return 0;
 225 }
 226
 227 static struct shash_alg crc32_alg = {
 228         .digestsize             =       CHKSUM_DIGEST_SIZE,
 229         .setkey                 =       chksum_setkey,
 230         .init                   =       chksum_init,
 231         .update                 =       chksum_update,
 232         .final                  =       chksum_final,
 233         .finup                  =       chksum_finup,
 234         .digest                 =       chksum_digest,
 235         .descsize               =       sizeof(struct chksum_desc_ctx),
 236         .base                   =       {
 237                 .cra_name               =       "crc32",
 238                 .cra_driver_name        =       "crc32-loongarch",
 239                 .cra_priority           =       300,
 240                 .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
 241                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 242                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 243                 .cra_module             =       THIS_MODULE,
 244                 .cra_init               =       chksum_cra_init,
 245         }
 246 };
 247
 248 static struct shash_alg crc32c_alg = {
 249         .digestsize             =       CHKSUM_DIGEST_SIZE,
 250         .setkey                 =       chksum_setkey,
 251         .init                   =       chksum_init,
 252         .update                 =       chksumc_update,
 253         .final                  =       chksumc_final,
 254         .finup                  =       chksumc_finup,
 255         .digest                 =       chksumc_digest,
 256         .descsize               =       sizeof(struct chksum_desc_ctx),
 257         .base                   =       {
 258                 .cra_name               =       "crc32c",
 259                 .cra_driver_name        =       "crc32c-loongarch",
 260                 .cra_priority           =       300,
 261                 .cra_flags              =       CRYPTO_ALG_OPTIONAL_KEY,
 262                 .cra_blocksize          =       CHKSUM_BLOCK_SIZE,
 263                 .cra_ctxsize            =       sizeof(struct chksum_ctx),
 264                 .cra_module             =       THIS_MODULE,
 265                 .cra_init               =       chksumc_cra_init,
 266         }
 267 };
 268
 269 static int __init crc32_mod_init(void)
 270 {
 271         int err;
 272
 273         if (!cpu_has(CPU_FEATURE_CRC32))
 274                 return 0;
 275
 276         err = crypto_register_shash(&crc32_alg);
 277         if (err)
 278                 return err;
 279
 280         err = crypto_register_shash(&crc32c_alg);
 281         if (err)
 282                 return err;
 283
 284         return 0;
 285 }
 286
 287 static void __exit crc32_mod_exit(void)
 288 {
 289         if (!cpu_has(CPU_FEATURE_CRC32))
 290                 return;
 291
 292         crypto_unregister_shash(&crc32_alg);
 293         crypto_unregister_shash(&crc32c_alg);
 294 }
 295
 296 module_init(crc32_mod_init);
 297 module_exit(crc32_mod_exit);
 298
 299 MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>");
 300 MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>");
 301 MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions");
 302 MODULE_LICENSE("GPL v2");