GNU Linux-libre 5.13.14-gnu1
[releases.git] / arch / s390 / crypto / crc32-vx.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Crypto-API module for CRC-32 algorithms implemented with the
4  * z/Architecture Vector Extension Facility.
5  *
6  * Copyright IBM Corp. 2015
7  * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
8  */
9 #define KMSG_COMPONENT  "crc32-vx"
10 #define pr_fmt(fmt)     KMSG_COMPONENT ": " fmt
11
12 #include <linux/module.h>
13 #include <linux/cpufeature.h>
14 #include <linux/crc32.h>
15 #include <crypto/internal/hash.h>
16 #include <asm/fpu/api.h>
17
18
19 #define CRC32_BLOCK_SIZE        1
20 #define CRC32_DIGEST_SIZE       4
21
22 #define VX_MIN_LEN              64
23 #define VX_ALIGNMENT            16L
24 #define VX_ALIGN_MASK           (VX_ALIGNMENT - 1)
25
26 struct crc_ctx {
27         u32 key;
28 };
29
30 struct crc_desc_ctx {
31         u32 crc;
32 };
33
34 /* Prototypes for functions in assembly files */
35 u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
36 u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
37 u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
38
39 /*
40  * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
41  *
42  * Creates a function to perform a particular CRC-32 computation. Depending
43  * on the message buffer, the hardware-accelerated or software implementation
44  * is used.   Note that the message buffer is aligned to improve fetch
45  * operations of VECTOR LOAD MULTIPLE instructions.
46  *
47  */
48 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw)                 \
49         static u32 __pure ___fname(u32 crc,                                 \
50                                 unsigned char const *data, size_t datalen)  \
51         {                                                                   \
52                 struct kernel_fpu vxstate;                                  \
53                 unsigned long prealign, aligned, remaining;                 \
54                                                                             \
55                 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK)                   \
56                         return ___crc32_sw(crc, data, datalen);             \
57                                                                             \
58                 if ((unsigned long)data & VX_ALIGN_MASK) {                  \
59                         prealign = VX_ALIGNMENT -                           \
60                                   ((unsigned long)data & VX_ALIGN_MASK);    \
61                         datalen -= prealign;                                \
62                         crc = ___crc32_sw(crc, data, prealign);             \
63                         data = (void *)((unsigned long)data + prealign);    \
64                 }                                                           \
65                                                                             \
66                 aligned = datalen & ~VX_ALIGN_MASK;                         \
67                 remaining = datalen & VX_ALIGN_MASK;                        \
68                                                                             \
69                 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW);                 \
70                 crc = ___crc32_vx(crc, data, aligned);                      \
71                 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW);                   \
72                                                                             \
73                 if (remaining)                                              \
74                         crc = ___crc32_sw(crc, data + aligned, remaining);  \
75                                                                             \
76                 return crc;                                                 \
77         }
78
79 DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
80 DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
81 DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
82
83
84 static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
85 {
86         struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
87
88         mctx->key = 0;
89         return 0;
90 }
91
92 static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
93 {
94         struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
95
96         mctx->key = ~0;
97         return 0;
98 }
99
100 static int crc32_vx_init(struct shash_desc *desc)
101 {
102         struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
103         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
104
105         ctx->crc = mctx->key;
106         return 0;
107 }
108
109 static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
110                            unsigned int newkeylen)
111 {
112         struct crc_ctx *mctx = crypto_shash_ctx(tfm);
113
114         if (newkeylen != sizeof(mctx->key))
115                 return -EINVAL;
116         mctx->key = le32_to_cpu(*(__le32 *)newkey);
117         return 0;
118 }
119
120 static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
121                              unsigned int newkeylen)
122 {
123         struct crc_ctx *mctx = crypto_shash_ctx(tfm);
124
125         if (newkeylen != sizeof(mctx->key))
126                 return -EINVAL;
127         mctx->key = be32_to_cpu(*(__be32 *)newkey);
128         return 0;
129 }
130
131 static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
132 {
133         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
134
135         *(__le32 *)out = cpu_to_le32p(&ctx->crc);
136         return 0;
137 }
138
139 static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
140 {
141         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
142
143         *(__be32 *)out = cpu_to_be32p(&ctx->crc);
144         return 0;
145 }
146
147 static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
148 {
149         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
150
151         /*
152          * Perform a final XOR with 0xFFFFFFFF to be in sync
153          * with the generic crc32c shash implementation.
154          */
155         *(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
156         return 0;
157 }
158
159 static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
160                               u8 *out)
161 {
162         *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
163         return 0;
164 }
165
166 static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
167                               u8 *out)
168 {
169         *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
170         return 0;
171 }
172
173 static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
174                              u8 *out)
175 {
176         /*
177          * Perform a final XOR with 0xFFFFFFFF to be in sync
178          * with the generic crc32c shash implementation.
179          */
180         *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
181         return 0;
182 }
183
184
185 #define CRC32_VX_FINUP(alg, func)                                             \
186         static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data,  \
187                                    unsigned int datalen, u8 *out)             \
188         {                                                                     \
189                 return __ ## alg ## _vx_finup(shash_desc_ctx(desc),           \
190                                               data, datalen, out);            \
191         }
192
193 CRC32_VX_FINUP(crc32le, crc32_le_vx)
194 CRC32_VX_FINUP(crc32be, crc32_be_vx)
195 CRC32_VX_FINUP(crc32c, crc32c_le_vx)
196
197 #define CRC32_VX_DIGEST(alg, func)                                            \
198         static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
199                                      unsigned int len, u8 *out)               \
200         {                                                                     \
201                 return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm),    \
202                                               data, len, out);                \
203         }
204
205 CRC32_VX_DIGEST(crc32le, crc32_le_vx)
206 CRC32_VX_DIGEST(crc32be, crc32_be_vx)
207 CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
208
209 #define CRC32_VX_UPDATE(alg, func)                                            \
210         static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
211                                      unsigned int datalen)                    \
212         {                                                                     \
213                 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);              \
214                 ctx->crc = func(ctx->crc, data, datalen);                     \
215                 return 0;                                                     \
216         }
217
218 CRC32_VX_UPDATE(crc32le, crc32_le_vx)
219 CRC32_VX_UPDATE(crc32be, crc32_be_vx)
220 CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
221
222
223 static struct shash_alg crc32_vx_algs[] = {
224         /* CRC-32 LE */
225         {
226                 .init           =       crc32_vx_init,
227                 .setkey         =       crc32_vx_setkey,
228                 .update         =       crc32le_vx_update,
229                 .final          =       crc32le_vx_final,
230                 .finup          =       crc32le_vx_finup,
231                 .digest         =       crc32le_vx_digest,
232                 .descsize       =       sizeof(struct crc_desc_ctx),
233                 .digestsize     =       CRC32_DIGEST_SIZE,
234                 .base           =       {
235                         .cra_name        = "crc32",
236                         .cra_driver_name = "crc32-vx",
237                         .cra_priority    = 200,
238                         .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
239                         .cra_blocksize   = CRC32_BLOCK_SIZE,
240                         .cra_ctxsize     = sizeof(struct crc_ctx),
241                         .cra_module      = THIS_MODULE,
242                         .cra_init        = crc32_vx_cra_init_zero,
243                 },
244         },
245         /* CRC-32 BE */
246         {
247                 .init           =       crc32_vx_init,
248                 .setkey         =       crc32be_vx_setkey,
249                 .update         =       crc32be_vx_update,
250                 .final          =       crc32be_vx_final,
251                 .finup          =       crc32be_vx_finup,
252                 .digest         =       crc32be_vx_digest,
253                 .descsize       =       sizeof(struct crc_desc_ctx),
254                 .digestsize     =       CRC32_DIGEST_SIZE,
255                 .base           =       {
256                         .cra_name        = "crc32be",
257                         .cra_driver_name = "crc32be-vx",
258                         .cra_priority    = 200,
259                         .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
260                         .cra_blocksize   = CRC32_BLOCK_SIZE,
261                         .cra_ctxsize     = sizeof(struct crc_ctx),
262                         .cra_module      = THIS_MODULE,
263                         .cra_init        = crc32_vx_cra_init_zero,
264                 },
265         },
266         /* CRC-32C LE */
267         {
268                 .init           =       crc32_vx_init,
269                 .setkey         =       crc32_vx_setkey,
270                 .update         =       crc32c_vx_update,
271                 .final          =       crc32c_vx_final,
272                 .finup          =       crc32c_vx_finup,
273                 .digest         =       crc32c_vx_digest,
274                 .descsize       =       sizeof(struct crc_desc_ctx),
275                 .digestsize     =       CRC32_DIGEST_SIZE,
276                 .base           =       {
277                         .cra_name        = "crc32c",
278                         .cra_driver_name = "crc32c-vx",
279                         .cra_priority    = 200,
280                         .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
281                         .cra_blocksize   = CRC32_BLOCK_SIZE,
282                         .cra_ctxsize     = sizeof(struct crc_ctx),
283                         .cra_module      = THIS_MODULE,
284                         .cra_init        = crc32_vx_cra_init_invert,
285                 },
286         },
287 };
288
289
290 static int __init crc_vx_mod_init(void)
291 {
292         return crypto_register_shashes(crc32_vx_algs,
293                                        ARRAY_SIZE(crc32_vx_algs));
294 }
295
296 static void __exit crc_vx_mod_exit(void)
297 {
298         crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
299 }
300
301 module_cpu_feature_match(VXRS, crc_vx_mod_init);
302 module_exit(crc_vx_mod_exit);
303
304 MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
305 MODULE_LICENSE("GPL");
306
307 MODULE_ALIAS_CRYPTO("crc32");
308 MODULE_ALIAS_CRYPTO("crc32-vx");
309 MODULE_ALIAS_CRYPTO("crc32c");
310 MODULE_ALIAS_CRYPTO("crc32c-vx");