arch/arm64/crypto/chacha-neon-core.S

   1 /*
   2  * ChaCha/XChaCha NEON helper functions
   3  *
   4  * Copyright (C) 2016-2018 Linaro, Ltd. <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * Originally based on:
  11  * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSSE3 functions
  12  *
  13  * Copyright (C) 2015 Martin Willi
  14  *
  15  * This program is free software; you can redistribute it and/or modify
  16  * it under the terms of the GNU General Public License as published by
  17  * the Free Software Foundation; either version 2 of the License, or
  18  * (at your option) any later version.
  19  */
  20
  21 #include <linux/linkage.h>
  22 #include <asm/assembler.h>
  23 #include <asm/cache.h>
  24
  25         .text
  26         .align          6
  27
  28 /*
  29  * chacha_permute - permute one block
  30  *
  31  * Permute one 64-byte block where the state matrix is stored in the four NEON
  32  * registers v0-v3.  It performs matrix operations on four words in parallel,
  33  * but requires shuffling to rearrange the words after each round.
  34  *
  35  * The round count is given in w3.
  36  *
  37  * Clobbers: w3, x10, v4, v12
  38  */
  39 chacha_permute:
  40
  41         adr_l           x10, ROT8
  42         ld1             {v12.4s}, [x10]
  43
  44 .Ldoubleround:
  45         // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
  46         add             v0.4s, v0.4s, v1.4s
  47         eor             v3.16b, v3.16b, v0.16b
  48         rev32           v3.8h, v3.8h
  49
  50         // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
  51         add             v2.4s, v2.4s, v3.4s
  52         eor             v4.16b, v1.16b, v2.16b
  53         shl             v1.4s, v4.4s, #12
  54         sri             v1.4s, v4.4s, #20
  55
  56         // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
  57         add             v0.4s, v0.4s, v1.4s
  58         eor             v3.16b, v3.16b, v0.16b
  59         tbl             v3.16b, {v3.16b}, v12.16b
  60
  61         // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
  62         add             v2.4s, v2.4s, v3.4s
  63         eor             v4.16b, v1.16b, v2.16b
  64         shl             v1.4s, v4.4s, #7
  65         sri             v1.4s, v4.4s, #25
  66
  67         // x1 = shuffle32(x1, MASK(0, 3, 2, 1))
  68         ext             v1.16b, v1.16b, v1.16b, #4
  69         // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
  70         ext             v2.16b, v2.16b, v2.16b, #8
  71         // x3 = shuffle32(x3, MASK(2, 1, 0, 3))
  72         ext             v3.16b, v3.16b, v3.16b, #12
  73
  74         // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
  75         add             v0.4s, v0.4s, v1.4s
  76         eor             v3.16b, v3.16b, v0.16b
  77         rev32           v3.8h, v3.8h
  78
  79         // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
  80         add             v2.4s, v2.4s, v3.4s
  81         eor             v4.16b, v1.16b, v2.16b
  82         shl             v1.4s, v4.4s, #12
  83         sri             v1.4s, v4.4s, #20
  84
  85         // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
  86         add             v0.4s, v0.4s, v1.4s
  87         eor             v3.16b, v3.16b, v0.16b
  88         tbl             v3.16b, {v3.16b}, v12.16b
  89
  90         // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
  91         add             v2.4s, v2.4s, v3.4s
  92         eor             v4.16b, v1.16b, v2.16b
  93         shl             v1.4s, v4.4s, #7
  94         sri             v1.4s, v4.4s, #25
  95
  96         // x1 = shuffle32(x1, MASK(2, 1, 0, 3))
  97         ext             v1.16b, v1.16b, v1.16b, #12
  98         // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
  99         ext             v2.16b, v2.16b, v2.16b, #8
 100         // x3 = shuffle32(x3, MASK(0, 3, 2, 1))
 101         ext             v3.16b, v3.16b, v3.16b, #4
 102
 103         subs            w3, w3, #2
 104         b.ne            .Ldoubleround
 105
 106         ret
 107 ENDPROC(chacha_permute)
 108
 109 ENTRY(chacha_block_xor_neon)
 110         // x0: Input state matrix, s
 111         // x1: 1 data block output, o
 112         // x2: 1 data block input, i
 113         // w3: nrounds
 114
 115         stp             x29, x30, [sp, #-16]!
 116         mov             x29, sp
 117
 118         // x0..3 = s0..3
 119         ld1             {v0.4s-v3.4s}, [x0]
 120         ld1             {v8.4s-v11.4s}, [x0]
 121
 122         bl              chacha_permute
 123
 124         ld1             {v4.16b-v7.16b}, [x2]
 125
 126         // o0 = i0 ^ (x0 + s0)
 127         add             v0.4s, v0.4s, v8.4s
 128         eor             v0.16b, v0.16b, v4.16b
 129
 130         // o1 = i1 ^ (x1 + s1)
 131         add             v1.4s, v1.4s, v9.4s
 132         eor             v1.16b, v1.16b, v5.16b
 133
 134         // o2 = i2 ^ (x2 + s2)
 135         add             v2.4s, v2.4s, v10.4s
 136         eor             v2.16b, v2.16b, v6.16b
 137
 138         // o3 = i3 ^ (x3 + s3)
 139         add             v3.4s, v3.4s, v11.4s
 140         eor             v3.16b, v3.16b, v7.16b
 141
 142         st1             {v0.16b-v3.16b}, [x1]
 143
 144         ldp             x29, x30, [sp], #16
 145         ret
 146 ENDPROC(chacha_block_xor_neon)
 147
 148 ENTRY(hchacha_block_neon)
 149         // x0: Input state matrix, s
 150         // x1: output (8 32-bit words)
 151         // w2: nrounds
 152
 153         stp             x29, x30, [sp, #-16]!
 154         mov             x29, sp
 155
 156         ld1             {v0.4s-v3.4s}, [x0]
 157
 158         mov             w3, w2
 159         bl              chacha_permute
 160
 161         st1             {v0.4s}, [x1], #16
 162         st1             {v3.4s}, [x1]
 163
 164         ldp             x29, x30, [sp], #16
 165         ret
 166 ENDPROC(hchacha_block_neon)
 167
 168         a0              .req    w12
 169         a1              .req    w13
 170         a2              .req    w14
 171         a3              .req    w15
 172         a4              .req    w16
 173         a5              .req    w17
 174         a6              .req    w19
 175         a7              .req    w20
 176         a8              .req    w21
 177         a9              .req    w22
 178         a10             .req    w23
 179         a11             .req    w24
 180         a12             .req    w25
 181         a13             .req    w26
 182         a14             .req    w27
 183         a15             .req    w28
 184
 185         .align          6
 186 ENTRY(chacha_4block_xor_neon)
 187         frame_push      10
 188
 189         // x0: Input state matrix, s
 190         // x1: 4 data blocks output, o
 191         // x2: 4 data blocks input, i
 192         // w3: nrounds
 193         // x4: byte count
 194
 195         adr_l           x10, .Lpermute
 196         and             x5, x4, #63
 197         add             x10, x10, x5
 198         add             x11, x10, #64
 199
 200         //
 201         // This function encrypts four consecutive ChaCha blocks by loading
 202         // the state matrix in NEON registers four times. The algorithm performs
 203         // each operation on the corresponding word of each state matrix, hence
 204         // requires no word shuffling. For final XORing step we transpose the
 205         // matrix by interleaving 32- and then 64-bit words, which allows us to
 206         // do XOR in NEON registers.
 207         //
 208         // At the same time, a fifth block is encrypted in parallel using
 209         // scalar registers
 210         //
 211         adr_l           x9, CTRINC              // ... and ROT8
 212         ld1             {v30.4s-v31.4s}, [x9]
 213
 214         // x0..15[0-3] = s0..3[0..3]
 215         add             x8, x0, #16
 216         ld4r            { v0.4s- v3.4s}, [x0]
 217         ld4r            { v4.4s- v7.4s}, [x8], #16
 218         ld4r            { v8.4s-v11.4s}, [x8], #16
 219         ld4r            {v12.4s-v15.4s}, [x8]
 220
 221         mov             a0, v0.s[0]
 222         mov             a1, v1.s[0]
 223         mov             a2, v2.s[0]
 224         mov             a3, v3.s[0]
 225         mov             a4, v4.s[0]
 226         mov             a5, v5.s[0]
 227         mov             a6, v6.s[0]
 228         mov             a7, v7.s[0]
 229         mov             a8, v8.s[0]
 230         mov             a9, v9.s[0]
 231         mov             a10, v10.s[0]
 232         mov             a11, v11.s[0]
 233         mov             a12, v12.s[0]
 234         mov             a13, v13.s[0]
 235         mov             a14, v14.s[0]
 236         mov             a15, v15.s[0]
 237
 238         // x12 += counter values 1-4
 239         add             v12.4s, v12.4s, v30.4s
 240
 241 .Ldoubleround4:
 242         // x0 += x4, x12 = rotl32(x12 ^ x0, 16)
 243         // x1 += x5, x13 = rotl32(x13 ^ x1, 16)
 244         // x2 += x6, x14 = rotl32(x14 ^ x2, 16)
 245         // x3 += x7, x15 = rotl32(x15 ^ x3, 16)
 246         add             v0.4s, v0.4s, v4.4s
 247           add           a0, a0, a4
 248         add             v1.4s, v1.4s, v5.4s
 249           add           a1, a1, a5
 250         add             v2.4s, v2.4s, v6.4s
 251           add           a2, a2, a6
 252         add             v3.4s, v3.4s, v7.4s
 253           add           a3, a3, a7
 254
 255         eor             v12.16b, v12.16b, v0.16b
 256           eor           a12, a12, a0
 257         eor             v13.16b, v13.16b, v1.16b
 258           eor           a13, a13, a1
 259         eor             v14.16b, v14.16b, v2.16b
 260           eor           a14, a14, a2
 261         eor             v15.16b, v15.16b, v3.16b
 262           eor           a15, a15, a3
 263
 264         rev32           v12.8h, v12.8h
 265           ror           a12, a12, #16
 266         rev32           v13.8h, v13.8h
 267           ror           a13, a13, #16
 268         rev32           v14.8h, v14.8h
 269           ror           a14, a14, #16
 270         rev32           v15.8h, v15.8h
 271           ror           a15, a15, #16
 272
 273         // x8 += x12, x4 = rotl32(x4 ^ x8, 12)
 274         // x9 += x13, x5 = rotl32(x5 ^ x9, 12)
 275         // x10 += x14, x6 = rotl32(x6 ^ x10, 12)
 276         // x11 += x15, x7 = rotl32(x7 ^ x11, 12)
 277         add             v8.4s, v8.4s, v12.4s
 278           add           a8, a8, a12
 279         add             v9.4s, v9.4s, v13.4s
 280           add           a9, a9, a13
 281         add             v10.4s, v10.4s, v14.4s
 282           add           a10, a10, a14
 283         add             v11.4s, v11.4s, v15.4s
 284           add           a11, a11, a15
 285
 286         eor             v16.16b, v4.16b, v8.16b
 287           eor           a4, a4, a8
 288         eor             v17.16b, v5.16b, v9.16b
 289           eor           a5, a5, a9
 290         eor             v18.16b, v6.16b, v10.16b
 291           eor           a6, a6, a10
 292         eor             v19.16b, v7.16b, v11.16b
 293           eor           a7, a7, a11
 294
 295         shl             v4.4s, v16.4s, #12
 296         shl             v5.4s, v17.4s, #12
 297         shl             v6.4s, v18.4s, #12
 298         shl             v7.4s, v19.4s, #12
 299
 300         sri             v4.4s, v16.4s, #20
 301           ror           a4, a4, #20
 302         sri             v5.4s, v17.4s, #20
 303           ror           a5, a5, #20
 304         sri             v6.4s, v18.4s, #20
 305           ror           a6, a6, #20
 306         sri             v7.4s, v19.4s, #20
 307           ror           a7, a7, #20
 308
 309         // x0 += x4, x12 = rotl32(x12 ^ x0, 8)
 310         // x1 += x5, x13 = rotl32(x13 ^ x1, 8)
 311         // x2 += x6, x14 = rotl32(x14 ^ x2, 8)
 312         // x3 += x7, x15 = rotl32(x15 ^ x3, 8)
 313         add             v0.4s, v0.4s, v4.4s
 314           add           a0, a0, a4
 315         add             v1.4s, v1.4s, v5.4s
 316           add           a1, a1, a5
 317         add             v2.4s, v2.4s, v6.4s
 318           add           a2, a2, a6
 319         add             v3.4s, v3.4s, v7.4s
 320           add           a3, a3, a7
 321
 322         eor             v12.16b, v12.16b, v0.16b
 323           eor           a12, a12, a0
 324         eor             v13.16b, v13.16b, v1.16b
 325           eor           a13, a13, a1
 326         eor             v14.16b, v14.16b, v2.16b
 327           eor           a14, a14, a2
 328         eor             v15.16b, v15.16b, v3.16b
 329           eor           a15, a15, a3
 330
 331         tbl             v12.16b, {v12.16b}, v31.16b
 332           ror           a12, a12, #24
 333         tbl             v13.16b, {v13.16b}, v31.16b
 334           ror           a13, a13, #24
 335         tbl             v14.16b, {v14.16b}, v31.16b
 336           ror           a14, a14, #24
 337         tbl             v15.16b, {v15.16b}, v31.16b
 338           ror           a15, a15, #24
 339
 340         // x8 += x12, x4 = rotl32(x4 ^ x8, 7)
 341         // x9 += x13, x5 = rotl32(x5 ^ x9, 7)
 342         // x10 += x14, x6 = rotl32(x6 ^ x10, 7)
 343         // x11 += x15, x7 = rotl32(x7 ^ x11, 7)
 344         add             v8.4s, v8.4s, v12.4s
 345           add           a8, a8, a12
 346         add             v9.4s, v9.4s, v13.4s
 347           add           a9, a9, a13
 348         add             v10.4s, v10.4s, v14.4s
 349           add           a10, a10, a14
 350         add             v11.4s, v11.4s, v15.4s
 351           add           a11, a11, a15
 352
 353         eor             v16.16b, v4.16b, v8.16b
 354           eor           a4, a4, a8
 355         eor             v17.16b, v5.16b, v9.16b
 356           eor           a5, a5, a9
 357         eor             v18.16b, v6.16b, v10.16b
 358           eor           a6, a6, a10
 359         eor             v19.16b, v7.16b, v11.16b
 360           eor           a7, a7, a11
 361
 362         shl             v4.4s, v16.4s, #7
 363         shl             v5.4s, v17.4s, #7
 364         shl             v6.4s, v18.4s, #7
 365         shl             v7.4s, v19.4s, #7
 366
 367         sri             v4.4s, v16.4s, #25
 368           ror           a4, a4, #25
 369         sri             v5.4s, v17.4s, #25
 370           ror           a5, a5, #25
 371         sri             v6.4s, v18.4s, #25
 372          ror            a6, a6, #25
 373         sri             v7.4s, v19.4s, #25
 374           ror           a7, a7, #25
 375
 376         // x0 += x5, x15 = rotl32(x15 ^ x0, 16)
 377         // x1 += x6, x12 = rotl32(x12 ^ x1, 16)
 378         // x2 += x7, x13 = rotl32(x13 ^ x2, 16)
 379         // x3 += x4, x14 = rotl32(x14 ^ x3, 16)
 380         add             v0.4s, v0.4s, v5.4s
 381           add           a0, a0, a5
 382         add             v1.4s, v1.4s, v6.4s
 383           add           a1, a1, a6
 384         add             v2.4s, v2.4s, v7.4s
 385           add           a2, a2, a7
 386         add             v3.4s, v3.4s, v4.4s
 387           add           a3, a3, a4
 388
 389         eor             v15.16b, v15.16b, v0.16b
 390           eor           a15, a15, a0
 391         eor             v12.16b, v12.16b, v1.16b
 392           eor           a12, a12, a1
 393         eor             v13.16b, v13.16b, v2.16b
 394           eor           a13, a13, a2
 395         eor             v14.16b, v14.16b, v3.16b
 396           eor           a14, a14, a3
 397
 398         rev32           v15.8h, v15.8h
 399           ror           a15, a15, #16
 400         rev32           v12.8h, v12.8h
 401           ror           a12, a12, #16
 402         rev32           v13.8h, v13.8h
 403           ror           a13, a13, #16
 404         rev32           v14.8h, v14.8h
 405           ror           a14, a14, #16
 406
 407         // x10 += x15, x5 = rotl32(x5 ^ x10, 12)
 408         // x11 += x12, x6 = rotl32(x6 ^ x11, 12)
 409         // x8 += x13, x7 = rotl32(x7 ^ x8, 12)
 410         // x9 += x14, x4 = rotl32(x4 ^ x9, 12)
 411         add             v10.4s, v10.4s, v15.4s
 412           add           a10, a10, a15
 413         add             v11.4s, v11.4s, v12.4s
 414           add           a11, a11, a12
 415         add             v8.4s, v8.4s, v13.4s
 416           add           a8, a8, a13
 417         add             v9.4s, v9.4s, v14.4s
 418           add           a9, a9, a14
 419
 420         eor             v16.16b, v5.16b, v10.16b
 421           eor           a5, a5, a10
 422         eor             v17.16b, v6.16b, v11.16b
 423           eor           a6, a6, a11
 424         eor             v18.16b, v7.16b, v8.16b
 425           eor           a7, a7, a8
 426         eor             v19.16b, v4.16b, v9.16b
 427           eor           a4, a4, a9
 428
 429         shl             v5.4s, v16.4s, #12
 430         shl             v6.4s, v17.4s, #12
 431         shl             v7.4s, v18.4s, #12
 432         shl             v4.4s, v19.4s, #12
 433
 434         sri             v5.4s, v16.4s, #20
 435           ror           a5, a5, #20
 436         sri             v6.4s, v17.4s, #20
 437           ror           a6, a6, #20
 438         sri             v7.4s, v18.4s, #20
 439           ror           a7, a7, #20
 440         sri             v4.4s, v19.4s, #20
 441           ror           a4, a4, #20
 442
 443         // x0 += x5, x15 = rotl32(x15 ^ x0, 8)
 444         // x1 += x6, x12 = rotl32(x12 ^ x1, 8)
 445         // x2 += x7, x13 = rotl32(x13 ^ x2, 8)
 446         // x3 += x4, x14 = rotl32(x14 ^ x3, 8)
 447         add             v0.4s, v0.4s, v5.4s
 448           add           a0, a0, a5
 449         add             v1.4s, v1.4s, v6.4s
 450           add           a1, a1, a6
 451         add             v2.4s, v2.4s, v7.4s
 452           add           a2, a2, a7
 453         add             v3.4s, v3.4s, v4.4s
 454           add           a3, a3, a4
 455
 456         eor             v15.16b, v15.16b, v0.16b
 457           eor           a15, a15, a0
 458         eor             v12.16b, v12.16b, v1.16b
 459           eor           a12, a12, a1
 460         eor             v13.16b, v13.16b, v2.16b
 461           eor           a13, a13, a2
 462         eor             v14.16b, v14.16b, v3.16b
 463           eor           a14, a14, a3
 464
 465         tbl             v15.16b, {v15.16b}, v31.16b
 466           ror           a15, a15, #24
 467         tbl             v12.16b, {v12.16b}, v31.16b
 468           ror           a12, a12, #24
 469         tbl             v13.16b, {v13.16b}, v31.16b
 470           ror           a13, a13, #24
 471         tbl             v14.16b, {v14.16b}, v31.16b
 472           ror           a14, a14, #24
 473
 474         // x10 += x15, x5 = rotl32(x5 ^ x10, 7)
 475         // x11 += x12, x6 = rotl32(x6 ^ x11, 7)
 476         // x8 += x13, x7 = rotl32(x7 ^ x8, 7)
 477         // x9 += x14, x4 = rotl32(x4 ^ x9, 7)
 478         add             v10.4s, v10.4s, v15.4s
 479           add           a10, a10, a15
 480         add             v11.4s, v11.4s, v12.4s
 481           add           a11, a11, a12
 482         add             v8.4s, v8.4s, v13.4s
 483           add           a8, a8, a13
 484         add             v9.4s, v9.4s, v14.4s
 485           add           a9, a9, a14
 486
 487         eor             v16.16b, v5.16b, v10.16b
 488           eor           a5, a5, a10
 489         eor             v17.16b, v6.16b, v11.16b
 490           eor           a6, a6, a11
 491         eor             v18.16b, v7.16b, v8.16b
 492           eor           a7, a7, a8
 493         eor             v19.16b, v4.16b, v9.16b
 494           eor           a4, a4, a9
 495
 496         shl             v5.4s, v16.4s, #7
 497         shl             v6.4s, v17.4s, #7
 498         shl             v7.4s, v18.4s, #7
 499         shl             v4.4s, v19.4s, #7
 500
 501         sri             v5.4s, v16.4s, #25
 502           ror           a5, a5, #25
 503         sri             v6.4s, v17.4s, #25
 504           ror           a6, a6, #25
 505         sri             v7.4s, v18.4s, #25
 506           ror           a7, a7, #25
 507         sri             v4.4s, v19.4s, #25
 508           ror           a4, a4, #25
 509
 510         subs            w3, w3, #2
 511         b.ne            .Ldoubleround4
 512
 513         ld4r            {v16.4s-v19.4s}, [x0], #16
 514         ld4r            {v20.4s-v23.4s}, [x0], #16
 515
 516         // x12 += counter values 0-3
 517         add             v12.4s, v12.4s, v30.4s
 518
 519         // x0[0-3] += s0[0]
 520         // x1[0-3] += s0[1]
 521         // x2[0-3] += s0[2]
 522         // x3[0-3] += s0[3]
 523         add             v0.4s, v0.4s, v16.4s
 524           mov           w6, v16.s[0]
 525           mov           w7, v17.s[0]
 526         add             v1.4s, v1.4s, v17.4s
 527           mov           w8, v18.s[0]
 528           mov           w9, v19.s[0]
 529         add             v2.4s, v2.4s, v18.4s
 530           add           a0, a0, w6
 531           add           a1, a1, w7
 532         add             v3.4s, v3.4s, v19.4s
 533           add           a2, a2, w8
 534           add           a3, a3, w9
 535 CPU_BE(   rev           a0, a0          )
 536 CPU_BE(   rev           a1, a1          )
 537 CPU_BE(   rev           a2, a2          )
 538 CPU_BE(   rev           a3, a3          )
 539
 540         ld4r            {v24.4s-v27.4s}, [x0], #16
 541         ld4r            {v28.4s-v31.4s}, [x0]
 542
 543         // x4[0-3] += s1[0]
 544         // x5[0-3] += s1[1]
 545         // x6[0-3] += s1[2]
 546         // x7[0-3] += s1[3]
 547         add             v4.4s, v4.4s, v20.4s
 548           mov           w6, v20.s[0]
 549           mov           w7, v21.s[0]
 550         add             v5.4s, v5.4s, v21.4s
 551           mov           w8, v22.s[0]
 552           mov           w9, v23.s[0]
 553         add             v6.4s, v6.4s, v22.4s
 554           add           a4, a4, w6
 555           add           a5, a5, w7
 556         add             v7.4s, v7.4s, v23.4s
 557           add           a6, a6, w8
 558           add           a7, a7, w9
 559 CPU_BE(   rev           a4, a4          )
 560 CPU_BE(   rev           a5, a5          )
 561 CPU_BE(   rev           a6, a6          )
 562 CPU_BE(   rev           a7, a7          )
 563
 564         // x8[0-3] += s2[0]
 565         // x9[0-3] += s2[1]
 566         // x10[0-3] += s2[2]
 567         // x11[0-3] += s2[3]
 568         add             v8.4s, v8.4s, v24.4s
 569           mov           w6, v24.s[0]
 570           mov           w7, v25.s[0]
 571         add             v9.4s, v9.4s, v25.4s
 572           mov           w8, v26.s[0]
 573           mov           w9, v27.s[0]
 574         add             v10.4s, v10.4s, v26.4s
 575           add           a8, a8, w6
 576           add           a9, a9, w7
 577         add             v11.4s, v11.4s, v27.4s
 578           add           a10, a10, w8
 579           add           a11, a11, w9
 580 CPU_BE(   rev           a8, a8          )
 581 CPU_BE(   rev           a9, a9          )
 582 CPU_BE(   rev           a10, a10        )
 583 CPU_BE(   rev           a11, a11        )
 584
 585         // x12[0-3] += s3[0]
 586         // x13[0-3] += s3[1]
 587         // x14[0-3] += s3[2]
 588         // x15[0-3] += s3[3]
 589         add             v12.4s, v12.4s, v28.4s
 590           mov           w6, v28.s[0]
 591           mov           w7, v29.s[0]
 592         add             v13.4s, v13.4s, v29.4s
 593           mov           w8, v30.s[0]
 594           mov           w9, v31.s[0]
 595         add             v14.4s, v14.4s, v30.4s
 596           add           a12, a12, w6
 597           add           a13, a13, w7
 598         add             v15.4s, v15.4s, v31.4s
 599           add           a14, a14, w8
 600           add           a15, a15, w9
 601 CPU_BE(   rev           a12, a12        )
 602 CPU_BE(   rev           a13, a13        )
 603 CPU_BE(   rev           a14, a14        )
 604 CPU_BE(   rev           a15, a15        )
 605
 606         // interleave 32-bit words in state n, n+1
 607           ldp           w6, w7, [x2], #64
 608         zip1            v16.4s, v0.4s, v1.4s
 609           ldp           w8, w9, [x2, #-56]
 610           eor           a0, a0, w6
 611         zip2            v17.4s, v0.4s, v1.4s
 612           eor           a1, a1, w7
 613         zip1            v18.4s, v2.4s, v3.4s
 614           eor           a2, a2, w8
 615         zip2            v19.4s, v2.4s, v3.4s
 616           eor           a3, a3, w9
 617           ldp           w6, w7, [x2, #-48]
 618         zip1            v20.4s, v4.4s, v5.4s
 619           ldp           w8, w9, [x2, #-40]
 620           eor           a4, a4, w6
 621         zip2            v21.4s, v4.4s, v5.4s
 622           eor           a5, a5, w7
 623         zip1            v22.4s, v6.4s, v7.4s
 624           eor           a6, a6, w8
 625         zip2            v23.4s, v6.4s, v7.4s
 626           eor           a7, a7, w9
 627           ldp           w6, w7, [x2, #-32]
 628         zip1            v24.4s, v8.4s, v9.4s
 629           ldp           w8, w9, [x2, #-24]
 630           eor           a8, a8, w6
 631         zip2            v25.4s, v8.4s, v9.4s
 632           eor           a9, a9, w7
 633         zip1            v26.4s, v10.4s, v11.4s
 634           eor           a10, a10, w8
 635         zip2            v27.4s, v10.4s, v11.4s
 636           eor           a11, a11, w9
 637           ldp           w6, w7, [x2, #-16]
 638         zip1            v28.4s, v12.4s, v13.4s
 639           ldp           w8, w9, [x2, #-8]
 640           eor           a12, a12, w6
 641         zip2            v29.4s, v12.4s, v13.4s
 642           eor           a13, a13, w7
 643         zip1            v30.4s, v14.4s, v15.4s
 644           eor           a14, a14, w8
 645         zip2            v31.4s, v14.4s, v15.4s
 646           eor           a15, a15, w9
 647
 648         mov             x3, #64
 649         subs            x5, x4, #128
 650         add             x6, x5, x2
 651         csel            x3, x3, xzr, ge
 652         csel            x2, x2, x6, ge
 653
 654         // interleave 64-bit words in state n, n+2
 655         zip1            v0.2d, v16.2d, v18.2d
 656         zip2            v4.2d, v16.2d, v18.2d
 657           stp           a0, a1, [x1], #64
 658         zip1            v8.2d, v17.2d, v19.2d
 659         zip2            v12.2d, v17.2d, v19.2d
 660           stp           a2, a3, [x1, #-56]
 661         ld1             {v16.16b-v19.16b}, [x2], x3
 662
 663         subs            x6, x4, #192
 664         ccmp            x3, xzr, #4, lt
 665         add             x7, x6, x2
 666         csel            x3, x3, xzr, eq
 667         csel            x2, x2, x7, eq
 668
 669         zip1            v1.2d, v20.2d, v22.2d
 670         zip2            v5.2d, v20.2d, v22.2d
 671           stp           a4, a5, [x1, #-48]
 672         zip1            v9.2d, v21.2d, v23.2d
 673         zip2            v13.2d, v21.2d, v23.2d
 674           stp           a6, a7, [x1, #-40]
 675         ld1             {v20.16b-v23.16b}, [x2], x3
 676
 677         subs            x7, x4, #256
 678         ccmp            x3, xzr, #4, lt
 679         add             x8, x7, x2
 680         csel            x3, x3, xzr, eq
 681         csel            x2, x2, x8, eq
 682
 683         zip1            v2.2d, v24.2d, v26.2d
 684         zip2            v6.2d, v24.2d, v26.2d
 685           stp           a8, a9, [x1, #-32]
 686         zip1            v10.2d, v25.2d, v27.2d
 687         zip2            v14.2d, v25.2d, v27.2d
 688           stp           a10, a11, [x1, #-24]
 689         ld1             {v24.16b-v27.16b}, [x2], x3
 690
 691         subs            x8, x4, #320
 692         ccmp            x3, xzr, #4, lt
 693         add             x9, x8, x2
 694         csel            x2, x2, x9, eq
 695
 696         zip1            v3.2d, v28.2d, v30.2d
 697         zip2            v7.2d, v28.2d, v30.2d
 698           stp           a12, a13, [x1, #-16]
 699         zip1            v11.2d, v29.2d, v31.2d
 700         zip2            v15.2d, v29.2d, v31.2d
 701           stp           a14, a15, [x1, #-8]
 702         ld1             {v28.16b-v31.16b}, [x2]
 703
 704         // xor with corresponding input, write to output
 705         tbnz            x5, #63, 0f
 706         eor             v16.16b, v16.16b, v0.16b
 707         eor             v17.16b, v17.16b, v1.16b
 708         eor             v18.16b, v18.16b, v2.16b
 709         eor             v19.16b, v19.16b, v3.16b
 710         st1             {v16.16b-v19.16b}, [x1], #64
 711         cbz             x5, .Lout
 712
 713         tbnz            x6, #63, 1f
 714         eor             v20.16b, v20.16b, v4.16b
 715         eor             v21.16b, v21.16b, v5.16b
 716         eor             v22.16b, v22.16b, v6.16b
 717         eor             v23.16b, v23.16b, v7.16b
 718         st1             {v20.16b-v23.16b}, [x1], #64
 719         cbz             x6, .Lout
 720
 721         tbnz            x7, #63, 2f
 722         eor             v24.16b, v24.16b, v8.16b
 723         eor             v25.16b, v25.16b, v9.16b
 724         eor             v26.16b, v26.16b, v10.16b
 725         eor             v27.16b, v27.16b, v11.16b
 726         st1             {v24.16b-v27.16b}, [x1], #64
 727         cbz             x7, .Lout
 728
 729         tbnz            x8, #63, 3f
 730         eor             v28.16b, v28.16b, v12.16b
 731         eor             v29.16b, v29.16b, v13.16b
 732         eor             v30.16b, v30.16b, v14.16b
 733         eor             v31.16b, v31.16b, v15.16b
 734         st1             {v28.16b-v31.16b}, [x1]
 735
 736 .Lout:  frame_pop
 737         ret
 738
 739         // fewer than 128 bytes of in/output
 740 0:      ld1             {v8.16b}, [x10]
 741         ld1             {v9.16b}, [x11]
 742         movi            v10.16b, #16
 743         sub             x2, x1, #64
 744         add             x1, x1, x5
 745         ld1             {v16.16b-v19.16b}, [x2]
 746         tbl             v4.16b, {v0.16b-v3.16b}, v8.16b
 747         tbx             v20.16b, {v16.16b-v19.16b}, v9.16b
 748         add             v8.16b, v8.16b, v10.16b
 749         add             v9.16b, v9.16b, v10.16b
 750         tbl             v5.16b, {v0.16b-v3.16b}, v8.16b
 751         tbx             v21.16b, {v16.16b-v19.16b}, v9.16b
 752         add             v8.16b, v8.16b, v10.16b
 753         add             v9.16b, v9.16b, v10.16b
 754         tbl             v6.16b, {v0.16b-v3.16b}, v8.16b
 755         tbx             v22.16b, {v16.16b-v19.16b}, v9.16b
 756         add             v8.16b, v8.16b, v10.16b
 757         add             v9.16b, v9.16b, v10.16b
 758         tbl             v7.16b, {v0.16b-v3.16b}, v8.16b
 759         tbx             v23.16b, {v16.16b-v19.16b}, v9.16b
 760
 761         eor             v20.16b, v20.16b, v4.16b
 762         eor             v21.16b, v21.16b, v5.16b
 763         eor             v22.16b, v22.16b, v6.16b
 764         eor             v23.16b, v23.16b, v7.16b
 765         st1             {v20.16b-v23.16b}, [x1]
 766         b               .Lout
 767
 768         // fewer than 192 bytes of in/output
 769 1:      ld1             {v8.16b}, [x10]
 770         ld1             {v9.16b}, [x11]
 771         movi            v10.16b, #16
 772         add             x1, x1, x6
 773         tbl             v0.16b, {v4.16b-v7.16b}, v8.16b
 774         tbx             v20.16b, {v16.16b-v19.16b}, v9.16b
 775         add             v8.16b, v8.16b, v10.16b
 776         add             v9.16b, v9.16b, v10.16b
 777         tbl             v1.16b, {v4.16b-v7.16b}, v8.16b
 778         tbx             v21.16b, {v16.16b-v19.16b}, v9.16b
 779         add             v8.16b, v8.16b, v10.16b
 780         add             v9.16b, v9.16b, v10.16b
 781         tbl             v2.16b, {v4.16b-v7.16b}, v8.16b
 782         tbx             v22.16b, {v16.16b-v19.16b}, v9.16b
 783         add             v8.16b, v8.16b, v10.16b
 784         add             v9.16b, v9.16b, v10.16b
 785         tbl             v3.16b, {v4.16b-v7.16b}, v8.16b
 786         tbx             v23.16b, {v16.16b-v19.16b}, v9.16b
 787
 788         eor             v20.16b, v20.16b, v0.16b
 789         eor             v21.16b, v21.16b, v1.16b
 790         eor             v22.16b, v22.16b, v2.16b
 791         eor             v23.16b, v23.16b, v3.16b
 792         st1             {v20.16b-v23.16b}, [x1]
 793         b               .Lout
 794
 795         // fewer than 256 bytes of in/output
 796 2:      ld1             {v4.16b}, [x10]
 797         ld1             {v5.16b}, [x11]
 798         movi            v6.16b, #16
 799         add             x1, x1, x7
 800         tbl             v0.16b, {v8.16b-v11.16b}, v4.16b
 801         tbx             v24.16b, {v20.16b-v23.16b}, v5.16b
 802         add             v4.16b, v4.16b, v6.16b
 803         add             v5.16b, v5.16b, v6.16b
 804         tbl             v1.16b, {v8.16b-v11.16b}, v4.16b
 805         tbx             v25.16b, {v20.16b-v23.16b}, v5.16b
 806         add             v4.16b, v4.16b, v6.16b
 807         add             v5.16b, v5.16b, v6.16b
 808         tbl             v2.16b, {v8.16b-v11.16b}, v4.16b
 809         tbx             v26.16b, {v20.16b-v23.16b}, v5.16b
 810         add             v4.16b, v4.16b, v6.16b
 811         add             v5.16b, v5.16b, v6.16b
 812         tbl             v3.16b, {v8.16b-v11.16b}, v4.16b
 813         tbx             v27.16b, {v20.16b-v23.16b}, v5.16b
 814
 815         eor             v24.16b, v24.16b, v0.16b
 816         eor             v25.16b, v25.16b, v1.16b
 817         eor             v26.16b, v26.16b, v2.16b
 818         eor             v27.16b, v27.16b, v3.16b
 819         st1             {v24.16b-v27.16b}, [x1]
 820         b               .Lout
 821
 822         // fewer than 320 bytes of in/output
 823 3:      ld1             {v4.16b}, [x10]
 824         ld1             {v5.16b}, [x11]
 825         movi            v6.16b, #16
 826         add             x1, x1, x8
 827         tbl             v0.16b, {v12.16b-v15.16b}, v4.16b
 828         tbx             v28.16b, {v24.16b-v27.16b}, v5.16b
 829         add             v4.16b, v4.16b, v6.16b
 830         add             v5.16b, v5.16b, v6.16b
 831         tbl             v1.16b, {v12.16b-v15.16b}, v4.16b
 832         tbx             v29.16b, {v24.16b-v27.16b}, v5.16b
 833         add             v4.16b, v4.16b, v6.16b
 834         add             v5.16b, v5.16b, v6.16b
 835         tbl             v2.16b, {v12.16b-v15.16b}, v4.16b
 836         tbx             v30.16b, {v24.16b-v27.16b}, v5.16b
 837         add             v4.16b, v4.16b, v6.16b
 838         add             v5.16b, v5.16b, v6.16b
 839         tbl             v3.16b, {v12.16b-v15.16b}, v4.16b
 840         tbx             v31.16b, {v24.16b-v27.16b}, v5.16b
 841
 842         eor             v28.16b, v28.16b, v0.16b
 843         eor             v29.16b, v29.16b, v1.16b
 844         eor             v30.16b, v30.16b, v2.16b
 845         eor             v31.16b, v31.16b, v3.16b
 846         st1             {v28.16b-v31.16b}, [x1]
 847         b               .Lout
 848 ENDPROC(chacha_4block_xor_neon)
 849
 850         .section        ".rodata", "a", %progbits
 851         .align          L1_CACHE_SHIFT
 852 .Lpermute:
 853         .set            .Li, 0
 854         .rept           192
 855         .byte           (.Li - 64)
 856         .set            .Li, .Li + 1
 857         .endr
 858
 859 CTRINC: .word           1, 2, 3, 4
 860 ROT8:   .word           0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f