1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Copyright (C) 2018 Google, Inc.
6 #include <linux/linkage.h>
7 #include <asm/assembler.h>
12 * 16 registers would be needed to hold the state matrix, but only 14 are
13 * available because 'sp' and 'pc' cannot be used. So we spill the elements
14 * (x8, x9) to the stack and swap them out with (x10, x11). This adds one
15 * 'ldrd' and one 'strd' instruction per round.
17 * All rotates are performed using the implicit rotate operand accepted by the
18 * 'add' and 'eor' instructions. This is faster than using explicit rotate
19 * instructions. To make this work, we allow the values in the second and last
20 * rows of the ChaCha state matrix (rows 'b' and 'd') to temporarily have the
21 * wrong rotation amount. The rotation amount is then fixed up just in time
22 * when the values are used. 'brot' is the number of bits the values in row 'b'
23 * need to be rotated right to arrive at the correct values, and 'drot'
24 * similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such
25 * that they end up as (25, 24) after every round.
28 // ChaCha state registers
37 X8_X10 .req r8 // shared by x8 and x10
38 X9_X11 .req r9 // shared by x9 and x11
44 .macro _le32_bswap_4x a, b, c, d, tmp
53 .macro __ldrd a, b, src, offset
54 #if __LINUX_ARM_ARCH__ >= 6
55 ldrd \a, \b, [\src, #\offset]
57 ldr \a, [\src, #\offset]
58 ldr \b, [\src, #\offset + 4]
62 .macro __strd a, b, dst, offset
63 #if __LINUX_ARM_ARCH__ >= 6
64 strd \a, \b, [\dst, #\offset]
66 str \a, [\dst, #\offset]
67 str \b, [\dst, #\offset + 4]
71 .macro _halfround a1, b1, c1, d1, a2, b2, c2, d2
73 // a += b; d ^= a; d = rol(d, 16);
74 add \a1, \a1, \b1, ror #brot
75 add \a2, \a2, \b2, ror #brot
76 eor \d1, \a1, \d1, ror #drot
77 eor \d2, \a2, \d2, ror #drot
78 // drot == 32 - 16 == 16
80 // c += d; b ^= c; b = rol(b, 12);
81 add \c1, \c1, \d1, ror #16
82 add \c2, \c2, \d2, ror #16
83 eor \b1, \c1, \b1, ror #brot
84 eor \b2, \c2, \b2, ror #brot
85 // brot == 32 - 12 == 20
87 // a += b; d ^= a; d = rol(d, 8);
88 add \a1, \a1, \b1, ror #20
89 add \a2, \a2, \b2, ror #20
90 eor \d1, \a1, \d1, ror #16
91 eor \d2, \a2, \d2, ror #16
92 // drot == 32 - 8 == 24
94 // c += d; b ^= c; b = rol(b, 7);
95 add \c1, \c1, \d1, ror #24
96 add \c2, \c2, \d2, ror #24
97 eor \b1, \c1, \b1, ror #20
98 eor \b2, \c2, \b2, ror #20
99 // brot == 32 - 7 == 25
106 // quarterrounds: (x0, x4, x8, x12) and (x1, x5, x9, x13)
107 _halfround X0, X4, X8_X10, X12, X1, X5, X9_X11, X13
109 // save (x8, x9); restore (x10, x11)
110 __strd X8_X10, X9_X11, sp, 0
111 __ldrd X8_X10, X9_X11, sp, 8
113 // quarterrounds: (x2, x6, x10, x14) and (x3, x7, x11, x15)
114 _halfround X2, X6, X8_X10, X14, X3, X7, X9_X11, X15
121 // quarterrounds: (x0, x5, x10, x15) and (x1, x6, x11, x12)
122 _halfround X0, X5, X8_X10, X15, X1, X6, X9_X11, X12
124 // save (x10, x11); restore (x8, x9)
125 __strd X8_X10, X9_X11, sp, 8
126 __ldrd X8_X10, X9_X11, sp, 0
128 // quarterrounds: (x2, x7, x8, x13) and (x3, x4, x9, x14)
129 _halfround X2, X7, X8_X10, X13, X3, X4, X9_X11, X14
132 .macro _chacha_permute nrounds
140 .macro _chacha nrounds
143 // Stack: unused0-unused1 x10-x11 x0-x15 OUT IN LEN
144 // Registers contain x0-x9,x12-x15.
146 // Do the core ChaCha permutation to update x0-x15.
147 _chacha_permute \nrounds
150 // Stack: x10-x11 orig_x0-orig_x15 OUT IN LEN
151 // Registers contain x0-x9,x12-x15.
152 // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.
154 // Free up some registers (r8-r12,r14) by pushing (x8-x9,x12-x15).
155 push {X8_X10, X9_X11, X12, X13, X14, X15}
157 // Load (OUT, IN, LEN).
164 // Use slow path if fewer than 64 bytes remain.
168 // Use slow path if IN and/or OUT isn't 4-byte aligned. Needed even on
169 // ARMv6+, since ldmia and stmia (used below) still require alignment.
173 // Fast path: XOR 64 bytes of aligned data.
175 // Stack: x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN
176 // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is OUT.
177 // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.
180 __ldrd r8, r9, sp, 32
181 __ldrd r10, r11, sp, 40
186 _le32_bswap_4x X0, X1, X2, X3, r8
195 __ldrd r8, r9, sp, 48
196 __ldrd r10, r11, sp, 56
197 add X4, r8, X4, ror #brot
198 add X5, r9, X5, ror #brot
200 add X6, r10, X6, ror #brot
201 add X7, r11, X7, ror #brot
202 _le32_bswap_4x X4, X5, X6, X7, r8
210 pop {r0-r7} // (x8-x9,x12-x15,x10-x11)
211 __ldrd r8, r9, sp, 32
212 __ldrd r10, r11, sp, 40
215 add r6, r6, r10 // x10
216 add r7, r7, r11 // x11
217 _le32_bswap_4x r0, r1, r6, r7, r8
221 eor r6, r6, r10 // x10
222 eor r7, r7, r11 // x11
223 stmia r14!, {r0,r1,r6,r7}
224 ldmia r12!, {r0,r1,r6,r7}
225 __ldrd r8, r9, sp, 48
226 __ldrd r10, r11, sp, 56
227 add r2, r8, r2, ror #drot // x12
228 add r3, r9, r3, ror #drot // x13
229 add r4, r10, r4, ror #drot // x14
230 add r5, r11, r5, ror #drot // x15
231 _le32_bswap_4x r2, r3, r4, r5, r9
232 ldr r9, [sp, #72] // load LEN
233 eor r2, r2, r0 // x12
234 eor r3, r3, r1 // x13
235 eor r4, r4, r6 // x14
236 eor r5, r5, r7 // x15
237 subs r9, #64 // decrement and check LEN
242 .Lprepare_for_next_block\@:
244 // Stack: x0-x15 OUT IN LEN
246 // Increment block counter (x12)
249 // Store updated (OUT, IN, LEN)
256 // Store updated block counter (x12)
261 // Reload state and do next block
262 ldmia r14!, {r0-r11} // load x0-x11
263 __strd r10, r11, sp, 8 // store x10-x11 before state
264 ldmia r14, {r10-r12,r14} // load x12-x15
268 // Slow path: < 64 bytes remaining, or unaligned input or output buffer.
269 // We handle it by storing the 64 bytes of keystream to the stack, then
270 // XOR-ing the needed portion with the data.
272 // Allocate keystream buffer
276 // Stack: ks0-ks15 x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN
277 // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is &ks0.
278 // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.
280 // Save keystream for x0-x3
281 __ldrd r8, r9, sp, 96
282 __ldrd r10, r11, sp, 104
287 _le32_bswap_4x X0, X1, X2, X3, r8
290 // Save keystream for x4-x7
291 __ldrd r8, r9, sp, 112
292 __ldrd r10, r11, sp, 120
293 add X4, r8, X4, ror #brot
294 add X5, r9, X5, ror #brot
295 add X6, r10, X6, ror #brot
296 add X7, r11, X7, ror #brot
297 _le32_bswap_4x X4, X5, X6, X7, r8
301 // Save keystream for x8-x15
302 ldm r8, {r0-r7} // (x8-x9,x12-x15,x10-x11)
303 __ldrd r8, r9, sp, 128
304 __ldrd r10, r11, sp, 136
307 add r6, r6, r10 // x10
308 add r7, r7, r11 // x11
309 _le32_bswap_4x r0, r1, r6, r7, r8
310 stmia r14!, {r0,r1,r6,r7}
311 __ldrd r8, r9, sp, 144
312 __ldrd r10, r11, sp, 152
313 add r2, r8, r2, ror #drot // x12
314 add r3, r9, r3, ror #drot // x13
315 add r4, r10, r4, ror #drot // x14
316 add r5, r11, r5, ror #drot // x15
317 _le32_bswap_4x r2, r3, r4, r5, r9
320 // Stack: ks0-ks15 unused0-unused7 x0-x15 OUT IN LEN
321 // Registers: r8 is block counter, r12 is IN.
323 ldr r9, [sp, #168] // LEN
324 ldr r14, [sp, #160] // OUT
329 // r1 is number of bytes to XOR, in range [1, 64]
331 .if __LINUX_ARM_ARCH__ < 6
333 tst r2, #3 // IN or OUT misaligned?
334 bne .Lxor_next_byte\@
337 // XOR a word at a time
340 blt .Lxor_words_done\@
346 b .Lxor_slowpath_done\@
349 beq .Lxor_slowpath_done\@
351 // XOR a byte at a time
358 bne .Lxor_next_byte\@
360 .Lxor_slowpath_done\@:
363 bgt .Lprepare_for_next_block\@
369 * void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
370 * const u32 *state, int nrounds);
373 cmp r2, #0 // len == 0?
379 push {r0-r2,r4-r11,lr}
381 // Push state x0-x15 onto stack.
382 // Also store an extra copy of x10-x11 just before the state.
385 ldm X12, {X12,X13,X14,X15}
386 push {X12,X13,X14,X15}
389 __ldrd X8_X10, X9_X11, r3, 40
390 __strd X8_X10, X9_X11, sp, 8
391 __strd X8_X10, X9_X11, sp, 56
393 __strd X0, X1, sp, 16
394 __strd X2, X3, sp, 24
395 __strd X4, X5, sp, 32
396 __strd X6, X7, sp, 40
397 __strd X8_X10, X9_X11, sp, 48
407 ENDPROC(chacha_doarm)
410 * void hchacha_block_arm(const u32 state[16], u32 out[8], int nrounds);
412 ENTRY(hchacha_block_arm)
415 cmp r2, #12 // ChaCha12 ?
418 ldmia r14!, {r0-r11} // load x0-x11
419 push {r10-r11} // store x10-x11 to stack
420 ldm r14, {r10-r12,r14} // load x12-x15
426 // Skip over (unused0-unused1, x10-x11)
429 // Fix up rotations of x12-x15
432 pop {r4} // load 'out'
436 // Store (x0-x3,x12-x15) to 'out'
437 stm r4, {X0,X1,X2,X3,X12,X13,X14,X15}
441 1: _chacha_permute 12
443 ENDPROC(hchacha_block_arm)