GNU Linux-libre 4.19.314-gnu1
[releases.git] / arch / arm / crypto / sha256-armv4.pl
1 #!/usr/bin/env perl
2 # SPDX-License-Identifier: GPL-2.0
3
4 # This code is taken from the OpenSSL project but the author (Andy Polyakov)
5 # has relicensed it under the GPLv2. Therefore this program is free software;
6 # you can redistribute it and/or modify it under the terms of the GNU General
7 # Public License version 2 as published by the Free Software Foundation.
8 #
9 # The original headers, including the original license headers, are
10 # included below for completeness.
11
12 # ====================================================================
13 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
14 # project. The module is, however, dual licensed under OpenSSL and
15 # CRYPTOGAMS licenses depending on where you obtain it. For further
16 # details see http://www.openssl.org/~appro/cryptogams/.
17 # ====================================================================
18
19 # SHA256 block procedure for ARMv4. May 2007.
20
21 # Performance is ~2x better than gcc 3.4 generated code and in "abso-
22 # lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per
23 # byte [on single-issue Xscale PXA250 core].
24
25 # July 2010.
26 #
27 # Rescheduling for dual-issue pipeline resulted in 22% improvement on
28 # Cortex A8 core and ~20 cycles per processed byte.
29
30 # February 2011.
31 #
32 # Profiler-assisted and platform-specific optimization resulted in 16%
33 # improvement on Cortex A8 core and ~15.4 cycles per processed byte.
34
35 # September 2013.
36 #
37 # Add NEON implementation. On Cortex A8 it was measured to process one
38 # byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon
39 # S4 does it in 12.5 cycles too, but it's 50% faster than integer-only
40 # code (meaning that latter performs sub-optimally, nothing was done
41 # about it).
42
43 # May 2014.
44 #
45 # Add ARMv8 code path performing at 2.0 cpb on Apple A7.
46
47 while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
48 open STDOUT,">$output";
49
50 $ctx="r0";      $t0="r0";
51 $inp="r1";      $t4="r1";
52 $len="r2";      $t1="r2";
53 $T1="r3";       $t3="r3";
54 $A="r4";
55 $B="r5";
56 $C="r6";
57 $D="r7";
58 $E="r8";
59 $F="r9";
60 $G="r10";
61 $H="r11";
62 @V=($A,$B,$C,$D,$E,$F,$G,$H);
63 $t2="r12";
64 $Ktbl="r14";
65
66 @Sigma0=( 2,13,22);
67 @Sigma1=( 6,11,25);
68 @sigma0=( 7,18, 3);
69 @sigma1=(17,19,10);
70
71 sub BODY_00_15 {
72 my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
73
74 $code.=<<___ if ($i<16);
75 #if __ARM_ARCH__>=7
76         @ ldr   $t1,[$inp],#4                   @ $i
77 # if $i==15
78         str     $inp,[sp,#17*4]                 @ make room for $t4
79 # endif
80         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
81         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
82         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
83 # ifndef __ARMEB__
84         rev     $t1,$t1
85 # endif
86 #else
87         @ ldrb  $t1,[$inp,#3]                   @ $i
88         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
89         ldrb    $t2,[$inp,#2]
90         ldrb    $t0,[$inp,#1]
91         orr     $t1,$t1,$t2,lsl#8
92         ldrb    $t2,[$inp],#4
93         orr     $t1,$t1,$t0,lsl#16
94 # if $i==15
95         str     $inp,[sp,#17*4]                 @ make room for $t4
96 # endif
97         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`
98         orr     $t1,$t1,$t2,lsl#24
99         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
100 #endif
101 ___
102 $code.=<<___;
103         ldr     $t2,[$Ktbl],#4                  @ *K256++
104         add     $h,$h,$t1                       @ h+=X[i]
105         str     $t1,[sp,#`$i%16`*4]
106         eor     $t1,$f,$g
107         add     $h,$h,$t0,ror#$Sigma1[0]        @ h+=Sigma1(e)
108         and     $t1,$t1,$e
109         add     $h,$h,$t2                       @ h+=K256[i]
110         eor     $t1,$t1,$g                      @ Ch(e,f,g)
111         eor     $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]`
112         add     $h,$h,$t1                       @ h+=Ch(e,f,g)
113 #if $i==31
114         and     $t2,$t2,#0xff
115         cmp     $t2,#0xf2                       @ done?
116 #endif
117 #if $i<15
118 # if __ARM_ARCH__>=7
119         ldr     $t1,[$inp],#4                   @ prefetch
120 # else
121         ldrb    $t1,[$inp,#3]
122 # endif
123         eor     $t2,$a,$b                       @ a^b, b^c in next round
124 #else
125         ldr     $t1,[sp,#`($i+2)%16`*4]         @ from future BODY_16_xx
126         eor     $t2,$a,$b                       @ a^b, b^c in next round
127         ldr     $t4,[sp,#`($i+15)%16`*4]        @ from future BODY_16_xx
128 #endif
129         eor     $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]`  @ Sigma0(a)
130         and     $t3,$t3,$t2                     @ (b^c)&=(a^b)
131         add     $d,$d,$h                        @ d+=h
132         eor     $t3,$t3,$b                      @ Maj(a,b,c)
133         add     $h,$h,$t0,ror#$Sigma0[0]        @ h+=Sigma0(a)
134         @ add   $h,$h,$t3                       @ h+=Maj(a,b,c)
135 ___
136         ($t2,$t3)=($t3,$t2);
137 }
138
139 sub BODY_16_XX {
140 my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
141
142 $code.=<<___;
143         @ ldr   $t1,[sp,#`($i+1)%16`*4]         @ $i
144         @ ldr   $t4,[sp,#`($i+14)%16`*4]
145         mov     $t0,$t1,ror#$sigma0[0]
146         add     $a,$a,$t2                       @ h+=Maj(a,b,c) from the past
147         mov     $t2,$t4,ror#$sigma1[0]
148         eor     $t0,$t0,$t1,ror#$sigma0[1]
149         eor     $t2,$t2,$t4,ror#$sigma1[1]
150         eor     $t0,$t0,$t1,lsr#$sigma0[2]      @ sigma0(X[i+1])
151         ldr     $t1,[sp,#`($i+0)%16`*4]
152         eor     $t2,$t2,$t4,lsr#$sigma1[2]      @ sigma1(X[i+14])
153         ldr     $t4,[sp,#`($i+9)%16`*4]
154
155         add     $t2,$t2,$t0
156         eor     $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]`   @ from BODY_00_15
157         add     $t1,$t1,$t2
158         eor     $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]`  @ Sigma1(e)
159         add     $t1,$t1,$t4                     @ X[i]
160 ___
161         &BODY_00_15(@_);
162 }
163
164 $code=<<___;
165 #ifndef __KERNEL__
166 # include "arm_arch.h"
167 #else
168 # define __ARM_ARCH__ __LINUX_ARM_ARCH__
169 # define __ARM_MAX_ARCH__ 7
170 #endif
171
172 .text
173 #if __ARM_ARCH__<7
174 .code   32
175 #else
176 .syntax unified
177 # ifdef __thumb2__
178 #  define adrl adr
179 .thumb
180 # else
181 .code   32
182 # endif
183 #endif
184
185 .type   K256,%object
186 .align  5
187 K256:
188 .word   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
189 .word   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
190 .word   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
191 .word   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
192 .word   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
193 .word   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
194 .word   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
195 .word   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
196 .word   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
197 .word   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
198 .word   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
199 .word   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
200 .word   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
201 .word   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
202 .word   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
203 .word   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
204 .size   K256,.-K256
205 .word   0                               @ terminator
206 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
207 .LOPENSSL_armcap:
208 .word   OPENSSL_armcap_P-sha256_block_data_order
209 #endif
210 .align  5
211
212 .global sha256_block_data_order
213 .type   sha256_block_data_order,%function
214 sha256_block_data_order:
215 .Lsha256_block_data_order:
216 #if __ARM_ARCH__<7
217         sub     r3,pc,#8                @ sha256_block_data_order
218 #else
219         adr     r3,.Lsha256_block_data_order
220 #endif
221 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
222         ldr     r12,.LOPENSSL_armcap
223         ldr     r12,[r3,r12]            @ OPENSSL_armcap_P
224         tst     r12,#ARMV8_SHA256
225         bne     .LARMv8
226         tst     r12,#ARMV7_NEON
227         bne     .LNEON
228 #endif
229         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
230         stmdb   sp!,{$ctx,$inp,$len,r4-r11,lr}
231         ldmia   $ctx,{$A,$B,$C,$D,$E,$F,$G,$H}
232         sub     $Ktbl,r3,#256+32        @ K256
233         sub     sp,sp,#16*4             @ alloca(X[16])
234 .Loop:
235 # if __ARM_ARCH__>=7
236         ldr     $t1,[$inp],#4
237 # else
238         ldrb    $t1,[$inp,#3]
239 # endif
240         eor     $t3,$B,$C               @ magic
241         eor     $t2,$t2,$t2
242 ___
243 for($i=0;$i<16;$i++)    { &BODY_00_15($i,@V); unshift(@V,pop(@V)); }
244 $code.=".Lrounds_16_xx:\n";
245 for (;$i<32;$i++)       { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); }
246 $code.=<<___;
247 #if __ARM_ARCH__>=7
248         ite     eq                      @ Thumb2 thing, sanity check in ARM
249 #endif
250         ldreq   $t3,[sp,#16*4]          @ pull ctx
251         bne     .Lrounds_16_xx
252
253         add     $A,$A,$t2               @ h+=Maj(a,b,c) from the past
254         ldr     $t0,[$t3,#0]
255         ldr     $t1,[$t3,#4]
256         ldr     $t2,[$t3,#8]
257         add     $A,$A,$t0
258         ldr     $t0,[$t3,#12]
259         add     $B,$B,$t1
260         ldr     $t1,[$t3,#16]
261         add     $C,$C,$t2
262         ldr     $t2,[$t3,#20]
263         add     $D,$D,$t0
264         ldr     $t0,[$t3,#24]
265         add     $E,$E,$t1
266         ldr     $t1,[$t3,#28]
267         add     $F,$F,$t2
268         ldr     $inp,[sp,#17*4]         @ pull inp
269         ldr     $t2,[sp,#18*4]          @ pull inp+len
270         add     $G,$G,$t0
271         add     $H,$H,$t1
272         stmia   $t3,{$A,$B,$C,$D,$E,$F,$G,$H}
273         cmp     $inp,$t2
274         sub     $Ktbl,$Ktbl,#256        @ rewind Ktbl
275         bne     .Loop
276
277         add     sp,sp,#`16+3`*4 @ destroy frame
278 #if __ARM_ARCH__>=5
279         ldmia   sp!,{r4-r11,pc}
280 #else
281         ldmia   sp!,{r4-r11,lr}
282         tst     lr,#1
283         moveq   pc,lr                   @ be binary compatible with V4, yet
284         bx      lr                      @ interoperable with Thumb ISA:-)
285 #endif
286 .size   sha256_block_data_order,.-sha256_block_data_order
287 ___
288 ######################################################################
289 # NEON stuff
290 #
291 {{{
292 my @X=map("q$_",(0..3));
293 my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25");
294 my $Xfer=$t4;
295 my $j=0;
296
297 sub Dlo()   { shift=~m|q([1]?[0-9])|?"d".($1*2):"";     }
298 sub Dhi()   { shift=~m|q([1]?[0-9])|?"d".($1*2+1):"";   }
299
300 sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
301 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
302   my $arg = pop;
303     $arg = "#$arg" if ($arg*1 eq $arg);
304     $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
305 }
306
307 sub Xupdate()
308 { use integer;
309   my $body = shift;
310   my @insns = (&$body,&$body,&$body,&$body);
311   my ($a,$b,$c,$d,$e,$f,$g,$h);
312
313         &vext_8         ($T0,@X[0],@X[1],4);    # X[1..4]
314          eval(shift(@insns));
315          eval(shift(@insns));
316          eval(shift(@insns));
317         &vext_8         ($T1,@X[2],@X[3],4);    # X[9..12]
318          eval(shift(@insns));
319          eval(shift(@insns));
320          eval(shift(@insns));
321         &vshr_u32       ($T2,$T0,$sigma0[0]);
322          eval(shift(@insns));
323          eval(shift(@insns));
324         &vadd_i32       (@X[0],@X[0],$T1);      # X[0..3] += X[9..12]
325          eval(shift(@insns));
326          eval(shift(@insns));
327         &vshr_u32       ($T1,$T0,$sigma0[2]);
328          eval(shift(@insns));
329          eval(shift(@insns));
330         &vsli_32        ($T2,$T0,32-$sigma0[0]);
331          eval(shift(@insns));
332          eval(shift(@insns));
333         &vshr_u32       ($T3,$T0,$sigma0[1]);
334          eval(shift(@insns));
335          eval(shift(@insns));
336         &veor           ($T1,$T1,$T2);
337          eval(shift(@insns));
338          eval(shift(@insns));
339         &vsli_32        ($T3,$T0,32-$sigma0[1]);
340          eval(shift(@insns));
341          eval(shift(@insns));
342           &vshr_u32     ($T4,&Dhi(@X[3]),$sigma1[0]);
343          eval(shift(@insns));
344          eval(shift(@insns));
345         &veor           ($T1,$T1,$T3);          # sigma0(X[1..4])
346          eval(shift(@insns));
347          eval(shift(@insns));
348           &vsli_32      ($T4,&Dhi(@X[3]),32-$sigma1[0]);
349          eval(shift(@insns));
350          eval(shift(@insns));
351           &vshr_u32     ($T5,&Dhi(@X[3]),$sigma1[2]);
352          eval(shift(@insns));
353          eval(shift(@insns));
354         &vadd_i32       (@X[0],@X[0],$T1);      # X[0..3] += sigma0(X[1..4])
355          eval(shift(@insns));
356          eval(shift(@insns));
357           &veor         ($T5,$T5,$T4);
358          eval(shift(@insns));
359          eval(shift(@insns));
360           &vshr_u32     ($T4,&Dhi(@X[3]),$sigma1[1]);
361          eval(shift(@insns));
362          eval(shift(@insns));
363           &vsli_32      ($T4,&Dhi(@X[3]),32-$sigma1[1]);
364          eval(shift(@insns));
365          eval(shift(@insns));
366           &veor         ($T5,$T5,$T4);          # sigma1(X[14..15])
367          eval(shift(@insns));
368          eval(shift(@insns));
369         &vadd_i32       (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15])
370          eval(shift(@insns));
371          eval(shift(@insns));
372           &vshr_u32     ($T4,&Dlo(@X[0]),$sigma1[0]);
373          eval(shift(@insns));
374          eval(shift(@insns));
375           &vsli_32      ($T4,&Dlo(@X[0]),32-$sigma1[0]);
376          eval(shift(@insns));
377          eval(shift(@insns));
378           &vshr_u32     ($T5,&Dlo(@X[0]),$sigma1[2]);
379          eval(shift(@insns));
380          eval(shift(@insns));
381           &veor         ($T5,$T5,$T4);
382          eval(shift(@insns));
383          eval(shift(@insns));
384           &vshr_u32     ($T4,&Dlo(@X[0]),$sigma1[1]);
385          eval(shift(@insns));
386          eval(shift(@insns));
387         &vld1_32        ("{$T0}","[$Ktbl,:128]!");
388          eval(shift(@insns));
389          eval(shift(@insns));
390           &vsli_32      ($T4,&Dlo(@X[0]),32-$sigma1[1]);
391          eval(shift(@insns));
392          eval(shift(@insns));
393           &veor         ($T5,$T5,$T4);          # sigma1(X[16..17])
394          eval(shift(@insns));
395          eval(shift(@insns));
396         &vadd_i32       (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17])
397          eval(shift(@insns));
398          eval(shift(@insns));
399         &vadd_i32       ($T0,$T0,@X[0]);
400          while($#insns>=2) { eval(shift(@insns)); }
401         &vst1_32        ("{$T0}","[$Xfer,:128]!");
402          eval(shift(@insns));
403          eval(shift(@insns));
404
405         push(@X,shift(@X));             # "rotate" X[]
406 }
407
408 sub Xpreload()
409 { use integer;
410   my $body = shift;
411   my @insns = (&$body,&$body,&$body,&$body);
412   my ($a,$b,$c,$d,$e,$f,$g,$h);
413
414          eval(shift(@insns));
415          eval(shift(@insns));
416          eval(shift(@insns));
417          eval(shift(@insns));
418         &vld1_32        ("{$T0}","[$Ktbl,:128]!");
419          eval(shift(@insns));
420          eval(shift(@insns));
421          eval(shift(@insns));
422          eval(shift(@insns));
423         &vrev32_8       (@X[0],@X[0]);
424          eval(shift(@insns));
425          eval(shift(@insns));
426          eval(shift(@insns));
427          eval(shift(@insns));
428         &vadd_i32       ($T0,$T0,@X[0]);
429          foreach (@insns) { eval; }     # remaining instructions
430         &vst1_32        ("{$T0}","[$Xfer,:128]!");
431
432         push(@X,shift(@X));             # "rotate" X[]
433 }
434
435 sub body_00_15 () {
436         (
437         '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
438         '&add   ($h,$h,$t1)',                   # h+=X[i]+K[i]
439         '&eor   ($t1,$f,$g)',
440         '&eor   ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
441         '&add   ($a,$a,$t2)',                   # h+=Maj(a,b,c) from the past
442         '&and   ($t1,$t1,$e)',
443         '&eor   ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))',   # Sigma1(e)
444         '&eor   ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
445         '&eor   ($t1,$t1,$g)',                  # Ch(e,f,g)
446         '&add   ($h,$h,$t2,"ror#$Sigma1[0]")',  # h+=Sigma1(e)
447         '&eor   ($t2,$a,$b)',                   # a^b, b^c in next round
448         '&eor   ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))',   # Sigma0(a)
449         '&add   ($h,$h,$t1)',                   # h+=Ch(e,f,g)
450         '&ldr   ($t1,sprintf "[sp,#%d]",4*(($j+1)&15))  if (($j&15)!=15);'.
451         '&ldr   ($t1,"[$Ktbl]")                         if ($j==15);'.
452         '&ldr   ($t1,"[sp,#64]")                        if ($j==31)',
453         '&and   ($t3,$t3,$t2)',                 # (b^c)&=(a^b)
454         '&add   ($d,$d,$h)',                    # d+=h
455         '&add   ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a)
456         '&eor   ($t3,$t3,$b)',                  # Maj(a,b,c)
457         '$j++;  unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
458         )
459 }
460
461 $code.=<<___;
462 #if __ARM_MAX_ARCH__>=7
463 .arch   armv7-a
464 .fpu    neon
465
466 .global sha256_block_data_order_neon
467 .type   sha256_block_data_order_neon,%function
468 .align  4
469 sha256_block_data_order_neon:
470 .LNEON:
471         stmdb   sp!,{r4-r12,lr}
472
473         sub     $H,sp,#16*4+16
474         adrl    $Ktbl,K256
475         bic     $H,$H,#15               @ align for 128-bit stores
476         mov     $t2,sp
477         mov     sp,$H                   @ alloca
478         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
479
480         vld1.8          {@X[0]},[$inp]!
481         vld1.8          {@X[1]},[$inp]!
482         vld1.8          {@X[2]},[$inp]!
483         vld1.8          {@X[3]},[$inp]!
484         vld1.32         {$T0},[$Ktbl,:128]!
485         vld1.32         {$T1},[$Ktbl,:128]!
486         vld1.32         {$T2},[$Ktbl,:128]!
487         vld1.32         {$T3},[$Ktbl,:128]!
488         vrev32.8        @X[0],@X[0]             @ yes, even on
489         str             $ctx,[sp,#64]
490         vrev32.8        @X[1],@X[1]             @ big-endian
491         str             $inp,[sp,#68]
492         mov             $Xfer,sp
493         vrev32.8        @X[2],@X[2]
494         str             $len,[sp,#72]
495         vrev32.8        @X[3],@X[3]
496         str             $t2,[sp,#76]            @ save original sp
497         vadd.i32        $T0,$T0,@X[0]
498         vadd.i32        $T1,$T1,@X[1]
499         vst1.32         {$T0},[$Xfer,:128]!
500         vadd.i32        $T2,$T2,@X[2]
501         vst1.32         {$T1},[$Xfer,:128]!
502         vadd.i32        $T3,$T3,@X[3]
503         vst1.32         {$T2},[$Xfer,:128]!
504         vst1.32         {$T3},[$Xfer,:128]!
505
506         ldmia           $ctx,{$A-$H}
507         sub             $Xfer,$Xfer,#64
508         ldr             $t1,[sp,#0]
509         eor             $t2,$t2,$t2
510         eor             $t3,$B,$C
511         b               .L_00_48
512
513 .align  4
514 .L_00_48:
515 ___
516         &Xupdate(\&body_00_15);
517         &Xupdate(\&body_00_15);
518         &Xupdate(\&body_00_15);
519         &Xupdate(\&body_00_15);
520 $code.=<<___;
521         teq     $t1,#0                          @ check for K256 terminator
522         ldr     $t1,[sp,#0]
523         sub     $Xfer,$Xfer,#64
524         bne     .L_00_48
525
526         ldr             $inp,[sp,#68]
527         ldr             $t0,[sp,#72]
528         sub             $Ktbl,$Ktbl,#256        @ rewind $Ktbl
529         teq             $inp,$t0
530         it              eq
531         subeq           $inp,$inp,#64           @ avoid SEGV
532         vld1.8          {@X[0]},[$inp]!         @ load next input block
533         vld1.8          {@X[1]},[$inp]!
534         vld1.8          {@X[2]},[$inp]!
535         vld1.8          {@X[3]},[$inp]!
536         it              ne
537         strne           $inp,[sp,#68]
538         mov             $Xfer,sp
539 ___
540         &Xpreload(\&body_00_15);
541         &Xpreload(\&body_00_15);
542         &Xpreload(\&body_00_15);
543         &Xpreload(\&body_00_15);
544 $code.=<<___;
545         ldr     $t0,[$t1,#0]
546         add     $A,$A,$t2                       @ h+=Maj(a,b,c) from the past
547         ldr     $t2,[$t1,#4]
548         ldr     $t3,[$t1,#8]
549         ldr     $t4,[$t1,#12]
550         add     $A,$A,$t0                       @ accumulate
551         ldr     $t0,[$t1,#16]
552         add     $B,$B,$t2
553         ldr     $t2,[$t1,#20]
554         add     $C,$C,$t3
555         ldr     $t3,[$t1,#24]
556         add     $D,$D,$t4
557         ldr     $t4,[$t1,#28]
558         add     $E,$E,$t0
559         str     $A,[$t1],#4
560         add     $F,$F,$t2
561         str     $B,[$t1],#4
562         add     $G,$G,$t3
563         str     $C,[$t1],#4
564         add     $H,$H,$t4
565         str     $D,[$t1],#4
566         stmia   $t1,{$E-$H}
567
568         ittte   ne
569         movne   $Xfer,sp
570         ldrne   $t1,[sp,#0]
571         eorne   $t2,$t2,$t2
572         ldreq   sp,[sp,#76]                     @ restore original sp
573         itt     ne
574         eorne   $t3,$B,$C
575         bne     .L_00_48
576
577         ldmia   sp!,{r4-r12,pc}
578 .size   sha256_block_data_order_neon,.-sha256_block_data_order_neon
579 #endif
580 ___
581 }}}
582 ######################################################################
583 # ARMv8 stuff
584 #
585 {{{
586 my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2));
587 my @MSG=map("q$_",(8..11));
588 my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15));
589 my $Ktbl="r3";
590
591 $code.=<<___;
592 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
593
594 # ifdef __thumb2__
595 #  define INST(a,b,c,d) .byte   c,d|0xc,a,b
596 # else
597 #  define INST(a,b,c,d) .byte   a,b,c,d
598 # endif
599
600 .type   sha256_block_data_order_armv8,%function
601 .align  5
602 sha256_block_data_order_armv8:
603 .LARMv8:
604         vld1.32 {$ABCD,$EFGH},[$ctx]
605 # ifdef __thumb2__
606         adr     $Ktbl,.LARMv8
607         sub     $Ktbl,$Ktbl,#.LARMv8-K256
608 # else
609         adrl    $Ktbl,K256
610 # endif
611         add     $len,$inp,$len,lsl#6    @ len to point at the end of inp
612
613 .Loop_v8:
614         vld1.8          {@MSG[0]-@MSG[1]},[$inp]!
615         vld1.8          {@MSG[2]-@MSG[3]},[$inp]!
616         vld1.32         {$W0},[$Ktbl]!
617         vrev32.8        @MSG[0],@MSG[0]
618         vrev32.8        @MSG[1],@MSG[1]
619         vrev32.8        @MSG[2],@MSG[2]
620         vrev32.8        @MSG[3],@MSG[3]
621         vmov            $ABCD_SAVE,$ABCD        @ offload
622         vmov            $EFGH_SAVE,$EFGH
623         teq             $inp,$len
624 ___
625 for($i=0;$i<12;$i++) {
626 $code.=<<___;
627         vld1.32         {$W1},[$Ktbl]!
628         vadd.i32        $W0,$W0,@MSG[0]
629         sha256su0       @MSG[0],@MSG[1]
630         vmov            $abcd,$ABCD
631         sha256h         $ABCD,$EFGH,$W0
632         sha256h2        $EFGH,$abcd,$W0
633         sha256su1       @MSG[0],@MSG[2],@MSG[3]
634 ___
635         ($W0,$W1)=($W1,$W0);    push(@MSG,shift(@MSG));
636 }
637 $code.=<<___;
638         vld1.32         {$W1},[$Ktbl]!
639         vadd.i32        $W0,$W0,@MSG[0]
640         vmov            $abcd,$ABCD
641         sha256h         $ABCD,$EFGH,$W0
642         sha256h2        $EFGH,$abcd,$W0
643
644         vld1.32         {$W0},[$Ktbl]!
645         vadd.i32        $W1,$W1,@MSG[1]
646         vmov            $abcd,$ABCD
647         sha256h         $ABCD,$EFGH,$W1
648         sha256h2        $EFGH,$abcd,$W1
649
650         vld1.32         {$W1},[$Ktbl]
651         vadd.i32        $W0,$W0,@MSG[2]
652         sub             $Ktbl,$Ktbl,#256-16     @ rewind
653         vmov            $abcd,$ABCD
654         sha256h         $ABCD,$EFGH,$W0
655         sha256h2        $EFGH,$abcd,$W0
656
657         vadd.i32        $W1,$W1,@MSG[3]
658         vmov            $abcd,$ABCD
659         sha256h         $ABCD,$EFGH,$W1
660         sha256h2        $EFGH,$abcd,$W1
661
662         vadd.i32        $ABCD,$ABCD,$ABCD_SAVE
663         vadd.i32        $EFGH,$EFGH,$EFGH_SAVE
664         it              ne
665         bne             .Loop_v8
666
667         vst1.32         {$ABCD,$EFGH},[$ctx]
668
669         ret             @ bx lr
670 .size   sha256_block_data_order_armv8,.-sha256_block_data_order_armv8
671 #endif
672 ___
673 }}}
674 $code.=<<___;
675 .asciz  "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
676 .align  2
677 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
678 .comm   OPENSSL_armcap_P,4,4
679 #endif
680 ___
681
682 open SELF,$0;
683 while(<SELF>) {
684         next if (/^#!/);
685         last if (!s/^#/@/ and !/^$/);
686         print;
687 }
688 close SELF;
689
690 {   my  %opcode = (
691         "sha256h"       => 0xf3000c40,  "sha256h2"      => 0xf3100c40,
692         "sha256su0"     => 0xf3ba03c0,  "sha256su1"     => 0xf3200c40   );
693
694     sub unsha256 {
695         my ($mnemonic,$arg)=@_;
696
697         if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) {
698             my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19)
699                                          |(($2&7)<<17)|(($2&8)<<4)
700                                          |(($3&7)<<1) |(($3&8)<<2);
701             # since ARMv7 instructions are always encoded little-endian.
702             # correct solution is to use .inst directive, but older
703             # assemblers don't implement it:-(
704             sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
705                         $word&0xff,($word>>8)&0xff,
706                         ($word>>16)&0xff,($word>>24)&0xff,
707                         $mnemonic,$arg;
708         }
709     }
710 }
711
712 foreach (split($/,$code)) {
713
714         s/\`([^\`]*)\`/eval $1/geo;
715
716         s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo;
717
718         s/\bret\b/bx    lr/go           or
719         s/\bbx\s+lr\b/.word\t0xe12fff1e/go;     # make it possible to compile with -march=armv4
720
721         print $_,"\n";
722 }
723
724 close STDOUT; # enforce flush