GNU Linux-libre 5.19-rc6-gnu
[releases.git] / lib / crypto / chacha20poly1305.c
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4  *
5  * This is an implementation of the ChaCha20Poly1305 AEAD construction.
6  *
7  * Information: https://tools.ietf.org/html/rfc8439
8  */
9
10 #include <crypto/algapi.h>
11 #include <crypto/chacha20poly1305.h>
12 #include <crypto/chacha.h>
13 #include <crypto/poly1305.h>
14 #include <crypto/scatterwalk.h>
15
16 #include <asm/unaligned.h>
17 #include <linux/kernel.h>
18 #include <linux/init.h>
19 #include <linux/mm.h>
20 #include <linux/module.h>
21
22 #define CHACHA_KEY_WORDS        (CHACHA_KEY_SIZE / sizeof(u32))
23
24 static void chacha_load_key(u32 *k, const u8 *in)
25 {
26         k[0] = get_unaligned_le32(in);
27         k[1] = get_unaligned_le32(in + 4);
28         k[2] = get_unaligned_le32(in + 8);
29         k[3] = get_unaligned_le32(in + 12);
30         k[4] = get_unaligned_le32(in + 16);
31         k[5] = get_unaligned_le32(in + 20);
32         k[6] = get_unaligned_le32(in + 24);
33         k[7] = get_unaligned_le32(in + 28);
34 }
35
36 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
37 {
38         u32 k[CHACHA_KEY_WORDS];
39         u8 iv[CHACHA_IV_SIZE];
40
41         memset(iv, 0, 8);
42         memcpy(iv + 8, nonce + 16, 8);
43
44         chacha_load_key(k, key);
45
46         /* Compute the subkey given the original key and first 128 nonce bits */
47         chacha_init(chacha_state, k, nonce);
48         hchacha_block(chacha_state, k, 20);
49
50         chacha_init(chacha_state, k, iv);
51
52         memzero_explicit(k, sizeof(k));
53         memzero_explicit(iv, sizeof(iv));
54 }
55
56 static void
57 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
58                            const u8 *ad, const size_t ad_len, u32 *chacha_state)
59 {
60         const u8 *pad0 = page_address(ZERO_PAGE(0));
61         struct poly1305_desc_ctx poly1305_state;
62         union {
63                 u8 block0[POLY1305_KEY_SIZE];
64                 __le64 lens[2];
65         } b;
66
67         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
68         poly1305_init(&poly1305_state, b.block0);
69
70         poly1305_update(&poly1305_state, ad, ad_len);
71         if (ad_len & 0xf)
72                 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
73
74         chacha20_crypt(chacha_state, dst, src, src_len);
75
76         poly1305_update(&poly1305_state, dst, src_len);
77         if (src_len & 0xf)
78                 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
79
80         b.lens[0] = cpu_to_le64(ad_len);
81         b.lens[1] = cpu_to_le64(src_len);
82         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
83
84         poly1305_final(&poly1305_state, dst + src_len);
85
86         memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
87         memzero_explicit(&b, sizeof(b));
88 }
89
90 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
91                               const u8 *ad, const size_t ad_len,
92                               const u64 nonce,
93                               const u8 key[CHACHA20POLY1305_KEY_SIZE])
94 {
95         u32 chacha_state[CHACHA_STATE_WORDS];
96         u32 k[CHACHA_KEY_WORDS];
97         __le64 iv[2];
98
99         chacha_load_key(k, key);
100
101         iv[0] = 0;
102         iv[1] = cpu_to_le64(nonce);
103
104         chacha_init(chacha_state, k, (u8 *)iv);
105         __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
106
107         memzero_explicit(iv, sizeof(iv));
108         memzero_explicit(k, sizeof(k));
109 }
110 EXPORT_SYMBOL(chacha20poly1305_encrypt);
111
112 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
113                                const u8 *ad, const size_t ad_len,
114                                const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
115                                const u8 key[CHACHA20POLY1305_KEY_SIZE])
116 {
117         u32 chacha_state[CHACHA_STATE_WORDS];
118
119         xchacha_init(chacha_state, key, nonce);
120         __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
121 }
122 EXPORT_SYMBOL(xchacha20poly1305_encrypt);
123
124 static bool
125 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
126                            const u8 *ad, const size_t ad_len, u32 *chacha_state)
127 {
128         const u8 *pad0 = page_address(ZERO_PAGE(0));
129         struct poly1305_desc_ctx poly1305_state;
130         size_t dst_len;
131         int ret;
132         union {
133                 u8 block0[POLY1305_KEY_SIZE];
134                 u8 mac[POLY1305_DIGEST_SIZE];
135                 __le64 lens[2];
136         } b;
137
138         if (unlikely(src_len < POLY1305_DIGEST_SIZE))
139                 return false;
140
141         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
142         poly1305_init(&poly1305_state, b.block0);
143
144         poly1305_update(&poly1305_state, ad, ad_len);
145         if (ad_len & 0xf)
146                 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
147
148         dst_len = src_len - POLY1305_DIGEST_SIZE;
149         poly1305_update(&poly1305_state, src, dst_len);
150         if (dst_len & 0xf)
151                 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
152
153         b.lens[0] = cpu_to_le64(ad_len);
154         b.lens[1] = cpu_to_le64(dst_len);
155         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
156
157         poly1305_final(&poly1305_state, b.mac);
158
159         ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
160         if (likely(!ret))
161                 chacha20_crypt(chacha_state, dst, src, dst_len);
162
163         memzero_explicit(&b, sizeof(b));
164
165         return !ret;
166 }
167
168 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
169                               const u8 *ad, const size_t ad_len,
170                               const u64 nonce,
171                               const u8 key[CHACHA20POLY1305_KEY_SIZE])
172 {
173         u32 chacha_state[CHACHA_STATE_WORDS];
174         u32 k[CHACHA_KEY_WORDS];
175         __le64 iv[2];
176         bool ret;
177
178         chacha_load_key(k, key);
179
180         iv[0] = 0;
181         iv[1] = cpu_to_le64(nonce);
182
183         chacha_init(chacha_state, k, (u8 *)iv);
184         ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
185                                          chacha_state);
186
187         memzero_explicit(chacha_state, sizeof(chacha_state));
188         memzero_explicit(iv, sizeof(iv));
189         memzero_explicit(k, sizeof(k));
190         return ret;
191 }
192 EXPORT_SYMBOL(chacha20poly1305_decrypt);
193
194 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
195                                const u8 *ad, const size_t ad_len,
196                                const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
197                                const u8 key[CHACHA20POLY1305_KEY_SIZE])
198 {
199         u32 chacha_state[CHACHA_STATE_WORDS];
200
201         xchacha_init(chacha_state, key, nonce);
202         return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
203                                           chacha_state);
204 }
205 EXPORT_SYMBOL(xchacha20poly1305_decrypt);
206
207 static
208 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
209                                        const size_t src_len,
210                                        const u8 *ad, const size_t ad_len,
211                                        const u64 nonce,
212                                        const u8 key[CHACHA20POLY1305_KEY_SIZE],
213                                        int encrypt)
214 {
215         const u8 *pad0 = page_address(ZERO_PAGE(0));
216         struct poly1305_desc_ctx poly1305_state;
217         u32 chacha_state[CHACHA_STATE_WORDS];
218         struct sg_mapping_iter miter;
219         size_t partial = 0;
220         unsigned int flags;
221         bool ret = true;
222         int sl;
223         union {
224                 struct {
225                         u32 k[CHACHA_KEY_WORDS];
226                         __le64 iv[2];
227                 };
228                 u8 block0[POLY1305_KEY_SIZE];
229                 u8 chacha_stream[CHACHA_BLOCK_SIZE];
230                 struct {
231                         u8 mac[2][POLY1305_DIGEST_SIZE];
232                 };
233                 __le64 lens[2];
234         } b __aligned(16);
235
236         if (WARN_ON(src_len > INT_MAX))
237                 return false;
238
239         chacha_load_key(b.k, key);
240
241         b.iv[0] = 0;
242         b.iv[1] = cpu_to_le64(nonce);
243
244         chacha_init(chacha_state, b.k, (u8 *)b.iv);
245         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
246         poly1305_init(&poly1305_state, b.block0);
247
248         if (unlikely(ad_len)) {
249                 poly1305_update(&poly1305_state, ad, ad_len);
250                 if (ad_len & 0xf)
251                         poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
252         }
253
254         flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
255
256         sg_miter_start(&miter, src, sg_nents(src), flags);
257
258         for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
259                 u8 *addr = miter.addr;
260                 size_t length = min_t(size_t, sl, miter.length);
261
262                 if (!encrypt)
263                         poly1305_update(&poly1305_state, addr, length);
264
265                 if (unlikely(partial)) {
266                         size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
267
268                         crypto_xor(addr, b.chacha_stream + partial, l);
269                         partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
270
271                         addr += l;
272                         length -= l;
273                 }
274
275                 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
276                         size_t l = length;
277
278                         if (unlikely(length < sl))
279                                 l &= ~(CHACHA_BLOCK_SIZE - 1);
280                         chacha20_crypt(chacha_state, addr, addr, l);
281                         addr += l;
282                         length -= l;
283                 }
284
285                 if (unlikely(length > 0)) {
286                         chacha20_crypt(chacha_state, b.chacha_stream, pad0,
287                                        CHACHA_BLOCK_SIZE);
288                         crypto_xor(addr, b.chacha_stream, length);
289                         partial = length;
290                 }
291
292                 if (encrypt)
293                         poly1305_update(&poly1305_state, miter.addr,
294                                         min_t(size_t, sl, miter.length));
295         }
296
297         if (src_len & 0xf)
298                 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
299
300         b.lens[0] = cpu_to_le64(ad_len);
301         b.lens[1] = cpu_to_le64(src_len);
302         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
303
304         if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
305                 if (encrypt) {
306                         poly1305_final(&poly1305_state,
307                                        miter.addr + miter.length + sl);
308                         ret = true;
309                 } else {
310                         poly1305_final(&poly1305_state, b.mac[0]);
311                         ret = !crypto_memneq(b.mac[0],
312                                              miter.addr + miter.length + sl,
313                                              POLY1305_DIGEST_SIZE);
314                 }
315         }
316
317         sg_miter_stop(&miter);
318
319         if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
320                 poly1305_final(&poly1305_state, b.mac[1]);
321                 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
322                                          sizeof(b.mac[1]), encrypt);
323                 ret = encrypt ||
324                       !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
325         }
326
327         memzero_explicit(chacha_state, sizeof(chacha_state));
328         memzero_explicit(&b, sizeof(b));
329
330         return ret;
331 }
332
333 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
334                                          const u8 *ad, const size_t ad_len,
335                                          const u64 nonce,
336                                          const u8 key[CHACHA20POLY1305_KEY_SIZE])
337 {
338         return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
339                                                  nonce, key, 1);
340 }
341 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
342
343 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
344                                          const u8 *ad, const size_t ad_len,
345                                          const u64 nonce,
346                                          const u8 key[CHACHA20POLY1305_KEY_SIZE])
347 {
348         if (unlikely(src_len < POLY1305_DIGEST_SIZE))
349                 return false;
350
351         return chacha20poly1305_crypt_sg_inplace(src,
352                                                  src_len - POLY1305_DIGEST_SIZE,
353                                                  ad, ad_len, nonce, key, 0);
354 }
355 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
356
357 static int __init chacha20poly1305_init(void)
358 {
359         if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
360             WARN_ON(!chacha20poly1305_selftest()))
361                 return -ENODEV;
362         return 0;
363 }
364
365 static void __exit chacha20poly1305_exit(void)
366 {
367 }
368
369 module_init(chacha20poly1305_init);
370 module_exit(chacha20poly1305_exit);
371 MODULE_LICENSE("GPL v2");
372 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
373 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");