arch/arm64/crypto/ghash-ce-core.S

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Accelerated GHASH implementation with ARMv8 PMULL instructions.
   4  *
   5  * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
   6  */
   7
   8 #include <linux/linkage.h>
   9 #include <linux/cfi_types.h>
  10 #include <asm/assembler.h>
  11
  12         SHASH           .req    v0
  13         SHASH2          .req    v1
  14         T1              .req    v2
  15         T2              .req    v3
  16         MASK            .req    v4
  17         XM              .req    v5
  18         XL              .req    v6
  19         XH              .req    v7
  20         IN1             .req    v7
  21
  22         k00_16          .req    v8
  23         k32_48          .req    v9
  24
  25         t3              .req    v10
  26         t4              .req    v11
  27         t5              .req    v12
  28         t6              .req    v13
  29         t7              .req    v14
  30         t8              .req    v15
  31         t9              .req    v16
  32
  33         perm1           .req    v17
  34         perm2           .req    v18
  35         perm3           .req    v19
  36
  37         sh1             .req    v20
  38         sh2             .req    v21
  39         sh3             .req    v22
  40         sh4             .req    v23
  41
  42         ss1             .req    v24
  43         ss2             .req    v25
  44         ss3             .req    v26
  45         ss4             .req    v27
  46
  47         XL2             .req    v8
  48         XM2             .req    v9
  49         XH2             .req    v10
  50         XL3             .req    v11
  51         XM3             .req    v12
  52         XH3             .req    v13
  53         TT3             .req    v14
  54         TT4             .req    v15
  55         HH              .req    v16
  56         HH3             .req    v17
  57         HH4             .req    v18
  58         HH34            .req    v19
  59
  60         .text
  61         .arch           armv8-a+crypto
  62
  63         .macro          __pmull_p64, rd, rn, rm
  64         pmull           \rd\().1q, \rn\().1d, \rm\().1d
  65         .endm
  66
  67         .macro          __pmull2_p64, rd, rn, rm
  68         pmull2          \rd\().1q, \rn\().2d, \rm\().2d
  69         .endm
  70
  71         .macro          __pmull_p8, rq, ad, bd
  72         ext             t3.8b, \ad\().8b, \ad\().8b, #1         // A1
  73         ext             t5.8b, \ad\().8b, \ad\().8b, #2         // A2
  74         ext             t7.8b, \ad\().8b, \ad\().8b, #3         // A3
  75
  76         __pmull_p8_\bd  \rq, \ad
  77         .endm
  78
  79         .macro          __pmull2_p8, rq, ad, bd
  80         tbl             t3.16b, {\ad\().16b}, perm1.16b         // A1
  81         tbl             t5.16b, {\ad\().16b}, perm2.16b         // A2
  82         tbl             t7.16b, {\ad\().16b}, perm3.16b         // A3
  83
  84         __pmull2_p8_\bd \rq, \ad
  85         .endm
  86
  87         .macro          __pmull_p8_SHASH, rq, ad
  88         __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4
  89         .endm
  90
  91         .macro          __pmull_p8_SHASH2, rq, ad
  92         __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4
  93         .endm
  94
  95         .macro          __pmull2_p8_SHASH, rq, ad
  96         __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4
  97         .endm
  98
  99         .macro          __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4
 100         pmull\t         t3.8h, t3.\nb, \bd                      // F = A1*B
 101         pmull\t         t4.8h, \ad, \b1\().\nb                  // E = A*B1
 102         pmull\t         t5.8h, t5.\nb, \bd                      // H = A2*B
 103         pmull\t         t6.8h, \ad, \b2\().\nb                  // G = A*B2
 104         pmull\t         t7.8h, t7.\nb, \bd                      // J = A3*B
 105         pmull\t         t8.8h, \ad, \b3\().\nb                  // I = A*B3
 106         pmull\t         t9.8h, \ad, \b4\().\nb                  // K = A*B4
 107         pmull\t         \rq\().8h, \ad, \bd                     // D = A*B
 108
 109         eor             t3.16b, t3.16b, t4.16b                  // L = E + F
 110         eor             t5.16b, t5.16b, t6.16b                  // M = G + H
 111         eor             t7.16b, t7.16b, t8.16b                  // N = I + J
 112
 113         uzp1            t4.2d, t3.2d, t5.2d
 114         uzp2            t3.2d, t3.2d, t5.2d
 115         uzp1            t6.2d, t7.2d, t9.2d
 116         uzp2            t7.2d, t7.2d, t9.2d
 117
 118         // t3 = (L) (P0 + P1) << 8
 119         // t5 = (M) (P2 + P3) << 16
 120         eor             t4.16b, t4.16b, t3.16b
 121         and             t3.16b, t3.16b, k32_48.16b
 122
 123         // t7 = (N) (P4 + P5) << 24
 124         // t9 = (K) (P6 + P7) << 32
 125         eor             t6.16b, t6.16b, t7.16b
 126         and             t7.16b, t7.16b, k00_16.16b
 127
 128         eor             t4.16b, t4.16b, t3.16b
 129         eor             t6.16b, t6.16b, t7.16b
 130
 131         zip2            t5.2d, t4.2d, t3.2d
 132         zip1            t3.2d, t4.2d, t3.2d
 133         zip2            t9.2d, t6.2d, t7.2d
 134         zip1            t7.2d, t6.2d, t7.2d
 135
 136         ext             t3.16b, t3.16b, t3.16b, #15
 137         ext             t5.16b, t5.16b, t5.16b, #14
 138         ext             t7.16b, t7.16b, t7.16b, #13
 139         ext             t9.16b, t9.16b, t9.16b, #12
 140
 141         eor             t3.16b, t3.16b, t5.16b
 142         eor             t7.16b, t7.16b, t9.16b
 143         eor             \rq\().16b, \rq\().16b, t3.16b
 144         eor             \rq\().16b, \rq\().16b, t7.16b
 145         .endm
 146
 147         .macro          __pmull_pre_p64
 148         add             x8, x3, #16
 149         ld1             {HH.2d-HH4.2d}, [x8]
 150
 151         trn1            SHASH2.2d, SHASH.2d, HH.2d
 152         trn2            T1.2d, SHASH.2d, HH.2d
 153         eor             SHASH2.16b, SHASH2.16b, T1.16b
 154
 155         trn1            HH34.2d, HH3.2d, HH4.2d
 156         trn2            T1.2d, HH3.2d, HH4.2d
 157         eor             HH34.16b, HH34.16b, T1.16b
 158
 159         movi            MASK.16b, #0xe1
 160         shl             MASK.2d, MASK.2d, #57
 161         .endm
 162
 163         .macro          __pmull_pre_p8
 164         ext             SHASH2.16b, SHASH.16b, SHASH.16b, #8
 165         eor             SHASH2.16b, SHASH2.16b, SHASH.16b
 166
 167         // k00_16 := 0x0000000000000000_000000000000ffff
 168         // k32_48 := 0x00000000ffffffff_0000ffffffffffff
 169         movi            k32_48.2d, #0xffffffff
 170         mov             k32_48.h[2], k32_48.h[0]
 171         ushr            k00_16.2d, k32_48.2d, #32
 172
 173         // prepare the permutation vectors
 174         mov_q           x5, 0x080f0e0d0c0b0a09
 175         movi            T1.8b, #8
 176         dup             perm1.2d, x5
 177         eor             perm1.16b, perm1.16b, T1.16b
 178         ushr            perm2.2d, perm1.2d, #8
 179         ushr            perm3.2d, perm1.2d, #16
 180         ushr            T1.2d, perm1.2d, #24
 181         sli             perm2.2d, perm1.2d, #56
 182         sli             perm3.2d, perm1.2d, #48
 183         sli             T1.2d, perm1.2d, #40
 184
 185         // precompute loop invariants
 186         tbl             sh1.16b, {SHASH.16b}, perm1.16b
 187         tbl             sh2.16b, {SHASH.16b}, perm2.16b
 188         tbl             sh3.16b, {SHASH.16b}, perm3.16b
 189         tbl             sh4.16b, {SHASH.16b}, T1.16b
 190         ext             ss1.8b, SHASH2.8b, SHASH2.8b, #1
 191         ext             ss2.8b, SHASH2.8b, SHASH2.8b, #2
 192         ext             ss3.8b, SHASH2.8b, SHASH2.8b, #3
 193         ext             ss4.8b, SHASH2.8b, SHASH2.8b, #4
 194         .endm
 195
 196         //
 197         // PMULL (64x64->128) based reduction for CPUs that can do
 198         // it in a single instruction.
 199         //
 200         .macro          __pmull_reduce_p64
 201         pmull           T2.1q, XL.1d, MASK.1d
 202         eor             XM.16b, XM.16b, T1.16b
 203
 204         mov             XH.d[0], XM.d[1]
 205         mov             XM.d[1], XL.d[0]
 206
 207         eor             XL.16b, XM.16b, T2.16b
 208         ext             T2.16b, XL.16b, XL.16b, #8
 209         pmull           XL.1q, XL.1d, MASK.1d
 210         .endm
 211
 212         //
 213         // Alternative reduction for CPUs that lack support for the
 214         // 64x64->128 PMULL instruction
 215         //
 216         .macro          __pmull_reduce_p8
 217         eor             XM.16b, XM.16b, T1.16b
 218
 219         mov             XL.d[1], XM.d[0]
 220         mov             XH.d[0], XM.d[1]
 221
 222         shl             T1.2d, XL.2d, #57
 223         shl             T2.2d, XL.2d, #62
 224         eor             T2.16b, T2.16b, T1.16b
 225         shl             T1.2d, XL.2d, #63
 226         eor             T2.16b, T2.16b, T1.16b
 227         ext             T1.16b, XL.16b, XH.16b, #8
 228         eor             T2.16b, T2.16b, T1.16b
 229
 230         mov             XL.d[1], T2.d[0]
 231         mov             XH.d[0], T2.d[1]
 232
 233         ushr            T2.2d, XL.2d, #1
 234         eor             XH.16b, XH.16b, XL.16b
 235         eor             XL.16b, XL.16b, T2.16b
 236         ushr            T2.2d, T2.2d, #6
 237         ushr            XL.2d, XL.2d, #1
 238         .endm
 239
 240         .macro          __pmull_ghash, pn
 241         ld1             {SHASH.2d}, [x3]
 242         ld1             {XL.2d}, [x1]
 243
 244         __pmull_pre_\pn
 245
 246         /* do the head block first, if supplied */
 247         cbz             x4, 0f
 248         ld1             {T1.2d}, [x4]
 249         mov             x4, xzr
 250         b               3f
 251
 252 0:      .ifc            \pn, p64
 253         tbnz            w0, #0, 2f              // skip until #blocks is a
 254         tbnz            w0, #1, 2f              // round multiple of 4
 255
 256 1:      ld1             {XM3.16b-TT4.16b}, [x2], #64
 257
 258         sub             w0, w0, #4
 259
 260         rev64           T1.16b, XM3.16b
 261         rev64           T2.16b, XH3.16b
 262         rev64           TT4.16b, TT4.16b
 263         rev64           TT3.16b, TT3.16b
 264
 265         ext             IN1.16b, TT4.16b, TT4.16b, #8
 266         ext             XL3.16b, TT3.16b, TT3.16b, #8
 267
 268         eor             TT4.16b, TT4.16b, IN1.16b
 269         pmull2          XH2.1q, SHASH.2d, IN1.2d        // a1 * b1
 270         pmull           XL2.1q, SHASH.1d, IN1.1d        // a0 * b0
 271         pmull           XM2.1q, SHASH2.1d, TT4.1d       // (a1 + a0)(b1 + b0)
 272
 273         eor             TT3.16b, TT3.16b, XL3.16b
 274         pmull2          XH3.1q, HH.2d, XL3.2d           // a1 * b1
 275         pmull           XL3.1q, HH.1d, XL3.1d           // a0 * b0
 276         pmull2          XM3.1q, SHASH2.2d, TT3.2d       // (a1 + a0)(b1 + b0)
 277
 278         ext             IN1.16b, T2.16b, T2.16b, #8
 279         eor             XL2.16b, XL2.16b, XL3.16b
 280         eor             XH2.16b, XH2.16b, XH3.16b
 281         eor             XM2.16b, XM2.16b, XM3.16b
 282
 283         eor             T2.16b, T2.16b, IN1.16b
 284         pmull2          XH3.1q, HH3.2d, IN1.2d          // a1 * b1
 285         pmull           XL3.1q, HH3.1d, IN1.1d          // a0 * b0
 286         pmull           XM3.1q, HH34.1d, T2.1d          // (a1 + a0)(b1 + b0)
 287
 288         eor             XL2.16b, XL2.16b, XL3.16b
 289         eor             XH2.16b, XH2.16b, XH3.16b
 290         eor             XM2.16b, XM2.16b, XM3.16b
 291
 292         ext             IN1.16b, T1.16b, T1.16b, #8
 293         ext             TT3.16b, XL.16b, XL.16b, #8
 294         eor             XL.16b, XL.16b, IN1.16b
 295         eor             T1.16b, T1.16b, TT3.16b
 296
 297         pmull2          XH.1q, HH4.2d, XL.2d            // a1 * b1
 298         eor             T1.16b, T1.16b, XL.16b
 299         pmull           XL.1q, HH4.1d, XL.1d            // a0 * b0
 300         pmull2          XM.1q, HH34.2d, T1.2d           // (a1 + a0)(b1 + b0)
 301
 302         eor             XL.16b, XL.16b, XL2.16b
 303         eor             XH.16b, XH.16b, XH2.16b
 304         eor             XM.16b, XM.16b, XM2.16b
 305
 306         eor             T2.16b, XL.16b, XH.16b
 307         ext             T1.16b, XL.16b, XH.16b, #8
 308         eor             XM.16b, XM.16b, T2.16b
 309
 310         __pmull_reduce_p64
 311
 312         eor             T2.16b, T2.16b, XH.16b
 313         eor             XL.16b, XL.16b, T2.16b
 314
 315         cbz             w0, 5f
 316         b               1b
 317         .endif
 318
 319 2:      ld1             {T1.2d}, [x2], #16
 320         sub             w0, w0, #1
 321
 322 3:      /* multiply XL by SHASH in GF(2^128) */
 323 CPU_LE( rev64           T1.16b, T1.16b  )
 324
 325         ext             T2.16b, XL.16b, XL.16b, #8
 326         ext             IN1.16b, T1.16b, T1.16b, #8
 327         eor             T1.16b, T1.16b, T2.16b
 328         eor             XL.16b, XL.16b, IN1.16b
 329
 330         __pmull2_\pn    XH, XL, SHASH                   // a1 * b1
 331         eor             T1.16b, T1.16b, XL.16b
 332         __pmull_\pn     XL, XL, SHASH                   // a0 * b0
 333         __pmull_\pn     XM, T1, SHASH2                  // (a1 + a0)(b1 + b0)
 334
 335 4:      eor             T2.16b, XL.16b, XH.16b
 336         ext             T1.16b, XL.16b, XH.16b, #8
 337         eor             XM.16b, XM.16b, T2.16b
 338
 339         __pmull_reduce_\pn
 340
 341         eor             T2.16b, T2.16b, XH.16b
 342         eor             XL.16b, XL.16b, T2.16b
 343
 344         cbnz            w0, 0b
 345
 346 5:      st1             {XL.2d}, [x1]
 347         ret
 348         .endm
 349
 350         /*
 351          * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
 352          *                         struct ghash_key const *k, const char *head)
 353          */
 354 SYM_TYPED_FUNC_START(pmull_ghash_update_p64)
 355         __pmull_ghash   p64
 356 SYM_FUNC_END(pmull_ghash_update_p64)
 357
 358 SYM_TYPED_FUNC_START(pmull_ghash_update_p8)
 359         __pmull_ghash   p8
 360 SYM_FUNC_END(pmull_ghash_update_p8)
 361
 362         KS0             .req    v8
 363         KS1             .req    v9
 364         KS2             .req    v10
 365         KS3             .req    v11
 366
 367         INP0            .req    v21
 368         INP1            .req    v22
 369         INP2            .req    v23
 370         INP3            .req    v24
 371
 372         K0              .req    v25
 373         K1              .req    v26
 374         K2              .req    v27
 375         K3              .req    v28
 376         K4              .req    v12
 377         K5              .req    v13
 378         K6              .req    v4
 379         K7              .req    v5
 380         K8              .req    v14
 381         K9              .req    v15
 382         KK              .req    v29
 383         KL              .req    v30
 384         KM              .req    v31
 385
 386         .macro          load_round_keys, rounds, rk, tmp
 387         add             \tmp, \rk, #64
 388         ld1             {K0.4s-K3.4s}, [\rk]
 389         ld1             {K4.4s-K5.4s}, [\tmp]
 390         add             \tmp, \rk, \rounds, lsl #4
 391         sub             \tmp, \tmp, #32
 392         ld1             {KK.4s-KM.4s}, [\tmp]
 393         .endm
 394
 395         .macro          enc_round, state, key
 396         aese            \state\().16b, \key\().16b
 397         aesmc           \state\().16b, \state\().16b
 398         .endm
 399
 400         .macro          enc_qround, s0, s1, s2, s3, key
 401         enc_round       \s0, \key
 402         enc_round       \s1, \key
 403         enc_round       \s2, \key
 404         enc_round       \s3, \key
 405         .endm
 406
 407         .macro          enc_block, state, rounds, rk, tmp
 408         add             \tmp, \rk, #96
 409         ld1             {K6.4s-K7.4s}, [\tmp], #32
 410         .irp            key, K0, K1, K2, K3, K4 K5
 411         enc_round       \state, \key
 412         .endr
 413
 414         tbnz            \rounds, #2, .Lnot128_\@
 415 .Lout256_\@:
 416         enc_round       \state, K6
 417         enc_round       \state, K7
 418
 419 .Lout192_\@:
 420         enc_round       \state, KK
 421         aese            \state\().16b, KL.16b
 422         eor             \state\().16b, \state\().16b, KM.16b
 423
 424         .subsection     1
 425 .Lnot128_\@:
 426         ld1             {K8.4s-K9.4s}, [\tmp], #32
 427         enc_round       \state, K6
 428         enc_round       \state, K7
 429         ld1             {K6.4s-K7.4s}, [\tmp]
 430         enc_round       \state, K8
 431         enc_round       \state, K9
 432         tbz             \rounds, #1, .Lout192_\@
 433         b               .Lout256_\@
 434         .previous
 435         .endm
 436
 437         .align          6
 438         .macro          pmull_gcm_do_crypt, enc
 439         frame_push      1
 440
 441         load_round_keys x7, x6, x8
 442
 443         ld1             {SHASH.2d}, [x3], #16
 444         ld1             {HH.2d-HH4.2d}, [x3]
 445
 446         trn1            SHASH2.2d, SHASH.2d, HH.2d
 447         trn2            T1.2d, SHASH.2d, HH.2d
 448         eor             SHASH2.16b, SHASH2.16b, T1.16b
 449
 450         trn1            HH34.2d, HH3.2d, HH4.2d
 451         trn2            T1.2d, HH3.2d, HH4.2d
 452         eor             HH34.16b, HH34.16b, T1.16b
 453
 454         ld1             {XL.2d}, [x4]
 455
 456         cbz             x0, 3f                          // tag only?
 457
 458         ldr             w8, [x5, #12]                   // load lower counter
 459 CPU_LE( rev             w8, w8          )
 460
 461 0:      mov             w9, #4                          // max blocks per round
 462         add             x10, x0, #0xf
 463         lsr             x10, x10, #4                    // remaining blocks
 464
 465         subs            x0, x0, #64
 466         csel            w9, w10, w9, mi
 467         add             w8, w8, w9
 468
 469         bmi             1f
 470         ld1             {INP0.16b-INP3.16b}, [x2], #64
 471         .subsection     1
 472         /*
 473          * Populate the four input registers right to left with up to 63 bytes
 474          * of data, using overlapping loads to avoid branches.
 475          *
 476          *                INP0     INP1     INP2     INP3
 477          *  1 byte     |        |        |        |x       |
 478          * 16 bytes    |        |        |        |xxxxxxxx|
 479          * 17 bytes    |        |        |xxxxxxxx|x       |
 480          * 47 bytes    |        |xxxxxxxx|xxxxxxxx|xxxxxxx |
 481          * etc etc
 482          *
 483          * Note that this code may read up to 15 bytes before the start of
 484          * the input. It is up to the calling code to ensure this is safe if
 485          * this happens in the first iteration of the loop (i.e., when the
 486          * input size is < 16 bytes)
 487          */
 488 1:      mov             x15, #16
 489         ands            x19, x0, #0xf
 490         csel            x19, x19, x15, ne
 491         adr_l           x17, .Lpermute_table + 16
 492
 493         sub             x11, x15, x19
 494         add             x12, x17, x11
 495         sub             x17, x17, x11
 496         ld1             {T1.16b}, [x12]
 497         sub             x10, x1, x11
 498         sub             x11, x2, x11
 499
 500         cmp             x0, #-16
 501         csel            x14, x15, xzr, gt
 502         cmp             x0, #-32
 503         csel            x15, x15, xzr, gt
 504         cmp             x0, #-48
 505         csel            x16, x19, xzr, gt
 506         csel            x1, x1, x10, gt
 507         csel            x2, x2, x11, gt
 508
 509         ld1             {INP0.16b}, [x2], x14
 510         ld1             {INP1.16b}, [x2], x15
 511         ld1             {INP2.16b}, [x2], x16
 512         ld1             {INP3.16b}, [x2]
 513         tbl             INP3.16b, {INP3.16b}, T1.16b
 514         b               2f
 515         .previous
 516
 517 2:      .if             \enc == 0
 518         bl              pmull_gcm_ghash_4x
 519         .endif
 520
 521         bl              pmull_gcm_enc_4x
 522
 523         tbnz            x0, #63, 6f
 524         st1             {INP0.16b-INP3.16b}, [x1], #64
 525         .if             \enc == 1
 526         bl              pmull_gcm_ghash_4x
 527         .endif
 528         bne             0b
 529
 530 3:      ldr             x10, [sp, #.Lframe_local_offset]
 531         cbz             x10, 5f                         // output tag?
 532
 533         ld1             {INP3.16b}, [x10]               // load lengths[]
 534         mov             w9, #1
 535         bl              pmull_gcm_ghash_4x
 536
 537         mov             w11, #(0x1 << 24)               // BE '1U'
 538         ld1             {KS0.16b}, [x5]
 539         mov             KS0.s[3], w11
 540
 541         enc_block       KS0, x7, x6, x12
 542
 543         ext             XL.16b, XL.16b, XL.16b, #8
 544         rev64           XL.16b, XL.16b
 545         eor             XL.16b, XL.16b, KS0.16b
 546
 547         .if             \enc == 1
 548         st1             {XL.16b}, [x10]                 // store tag
 549         .else
 550         ldp             x11, x12, [sp, #40]             // load tag pointer and authsize
 551         adr_l           x17, .Lpermute_table
 552         ld1             {KS0.16b}, [x11]                // load supplied tag
 553         add             x17, x17, x12
 554         ld1             {KS1.16b}, [x17]                // load permute vector
 555
 556         cmeq            XL.16b, XL.16b, KS0.16b         // compare tags
 557         mvn             XL.16b, XL.16b                  // -1 for fail, 0 for pass
 558         tbl             XL.16b, {XL.16b}, KS1.16b       // keep authsize bytes only
 559         sminv           b0, XL.16b                      // signed minimum across XL
 560         smov            w0, v0.b[0]                     // return b0
 561         .endif
 562
 563 4:      frame_pop
 564         ret
 565
 566 5:
 567 CPU_LE( rev             w8, w8          )
 568         str             w8, [x5, #12]                   // store lower counter
 569         st1             {XL.2d}, [x4]
 570         b               4b
 571
 572 6:      ld1             {T1.16b-T2.16b}, [x17], #32     // permute vectors
 573         sub             x17, x17, x19, lsl #1
 574
 575         cmp             w9, #1
 576         beq             7f
 577         .subsection     1
 578 7:      ld1             {INP2.16b}, [x1]
 579         tbx             INP2.16b, {INP3.16b}, T1.16b
 580         mov             INP3.16b, INP2.16b
 581         b               8f
 582         .previous
 583
 584         st1             {INP0.16b}, [x1], x14
 585         st1             {INP1.16b}, [x1], x15
 586         st1             {INP2.16b}, [x1], x16
 587         tbl             INP3.16b, {INP3.16b}, T1.16b
 588         tbx             INP3.16b, {INP2.16b}, T2.16b
 589 8:      st1             {INP3.16b}, [x1]
 590
 591         .if             \enc == 1
 592         ld1             {T1.16b}, [x17]
 593         tbl             INP3.16b, {INP3.16b}, T1.16b    // clear non-data bits
 594         bl              pmull_gcm_ghash_4x
 595         .endif
 596         b               3b
 597         .endm
 598
 599         /*
 600          * void pmull_gcm_encrypt(int blocks, u8 dst[], const u8 src[],
 601          *                        struct ghash_key const *k, u64 dg[], u8 ctr[],
 602          *                        int rounds, u8 tag)
 603          */
 604 SYM_FUNC_START(pmull_gcm_encrypt)
 605         pmull_gcm_do_crypt      1
 606 SYM_FUNC_END(pmull_gcm_encrypt)
 607
 608         /*
 609          * void pmull_gcm_decrypt(int blocks, u8 dst[], const u8 src[],
 610          *                        struct ghash_key const *k, u64 dg[], u8 ctr[],
 611          *                        int rounds, u8 tag)
 612          */
 613 SYM_FUNC_START(pmull_gcm_decrypt)
 614         pmull_gcm_do_crypt      0
 615 SYM_FUNC_END(pmull_gcm_decrypt)
 616
 617 SYM_FUNC_START_LOCAL(pmull_gcm_ghash_4x)
 618         movi            MASK.16b, #0xe1
 619         shl             MASK.2d, MASK.2d, #57
 620
 621         rev64           T1.16b, INP0.16b
 622         rev64           T2.16b, INP1.16b
 623         rev64           TT3.16b, INP2.16b
 624         rev64           TT4.16b, INP3.16b
 625
 626         ext             XL.16b, XL.16b, XL.16b, #8
 627
 628         tbz             w9, #2, 0f                      // <4 blocks?
 629         .subsection     1
 630 0:      movi            XH2.16b, #0
 631         movi            XM2.16b, #0
 632         movi            XL2.16b, #0
 633
 634         tbz             w9, #0, 1f                      // 2 blocks?
 635         tbz             w9, #1, 2f                      // 1 block?
 636
 637         eor             T2.16b, T2.16b, XL.16b
 638         ext             T1.16b, T2.16b, T2.16b, #8
 639         b               .Lgh3
 640
 641 1:      eor             TT3.16b, TT3.16b, XL.16b
 642         ext             T2.16b, TT3.16b, TT3.16b, #8
 643         b               .Lgh2
 644
 645 2:      eor             TT4.16b, TT4.16b, XL.16b
 646         ext             IN1.16b, TT4.16b, TT4.16b, #8
 647         b               .Lgh1
 648         .previous
 649
 650         eor             T1.16b, T1.16b, XL.16b
 651         ext             IN1.16b, T1.16b, T1.16b, #8
 652
 653         pmull2          XH2.1q, HH4.2d, IN1.2d          // a1 * b1
 654         eor             T1.16b, T1.16b, IN1.16b
 655         pmull           XL2.1q, HH4.1d, IN1.1d          // a0 * b0
 656         pmull2          XM2.1q, HH34.2d, T1.2d          // (a1 + a0)(b1 + b0)
 657
 658         ext             T1.16b, T2.16b, T2.16b, #8
 659 .Lgh3:  eor             T2.16b, T2.16b, T1.16b
 660         pmull2          XH.1q, HH3.2d, T1.2d            // a1 * b1
 661         pmull           XL.1q, HH3.1d, T1.1d            // a0 * b0
 662         pmull           XM.1q, HH34.1d, T2.1d           // (a1 + a0)(b1 + b0)
 663
 664         eor             XH2.16b, XH2.16b, XH.16b
 665         eor             XL2.16b, XL2.16b, XL.16b
 666         eor             XM2.16b, XM2.16b, XM.16b
 667
 668         ext             T2.16b, TT3.16b, TT3.16b, #8
 669 .Lgh2:  eor             TT3.16b, TT3.16b, T2.16b
 670         pmull2          XH.1q, HH.2d, T2.2d             // a1 * b1
 671         pmull           XL.1q, HH.1d, T2.1d             // a0 * b0
 672         pmull2          XM.1q, SHASH2.2d, TT3.2d        // (a1 + a0)(b1 + b0)
 673
 674         eor             XH2.16b, XH2.16b, XH.16b
 675         eor             XL2.16b, XL2.16b, XL.16b
 676         eor             XM2.16b, XM2.16b, XM.16b
 677
 678         ext             IN1.16b, TT4.16b, TT4.16b, #8
 679 .Lgh1:  eor             TT4.16b, TT4.16b, IN1.16b
 680         pmull           XL.1q, SHASH.1d, IN1.1d         // a0 * b0
 681         pmull2          XH.1q, SHASH.2d, IN1.2d         // a1 * b1
 682         pmull           XM.1q, SHASH2.1d, TT4.1d        // (a1 + a0)(b1 + b0)
 683
 684         eor             XH.16b, XH.16b, XH2.16b
 685         eor             XL.16b, XL.16b, XL2.16b
 686         eor             XM.16b, XM.16b, XM2.16b
 687
 688         eor             T2.16b, XL.16b, XH.16b
 689         ext             T1.16b, XL.16b, XH.16b, #8
 690         eor             XM.16b, XM.16b, T2.16b
 691
 692         __pmull_reduce_p64
 693
 694         eor             T2.16b, T2.16b, XH.16b
 695         eor             XL.16b, XL.16b, T2.16b
 696
 697         ret
 698 SYM_FUNC_END(pmull_gcm_ghash_4x)
 699
 700 SYM_FUNC_START_LOCAL(pmull_gcm_enc_4x)
 701         ld1             {KS0.16b}, [x5]                 // load upper counter
 702         sub             w10, w8, #4
 703         sub             w11, w8, #3
 704         sub             w12, w8, #2
 705         sub             w13, w8, #1
 706         rev             w10, w10
 707         rev             w11, w11
 708         rev             w12, w12
 709         rev             w13, w13
 710         mov             KS1.16b, KS0.16b
 711         mov             KS2.16b, KS0.16b
 712         mov             KS3.16b, KS0.16b
 713         ins             KS0.s[3], w10                   // set lower counter
 714         ins             KS1.s[3], w11
 715         ins             KS2.s[3], w12
 716         ins             KS3.s[3], w13
 717
 718         add             x10, x6, #96                    // round key pointer
 719         ld1             {K6.4s-K7.4s}, [x10], #32
 720         .irp            key, K0, K1, K2, K3, K4, K5
 721         enc_qround      KS0, KS1, KS2, KS3, \key
 722         .endr
 723
 724         tbnz            x7, #2, .Lnot128
 725         .subsection     1
 726 .Lnot128:
 727         ld1             {K8.4s-K9.4s}, [x10], #32
 728         .irp            key, K6, K7
 729         enc_qround      KS0, KS1, KS2, KS3, \key
 730         .endr
 731         ld1             {K6.4s-K7.4s}, [x10]
 732         .irp            key, K8, K9
 733         enc_qround      KS0, KS1, KS2, KS3, \key
 734         .endr
 735         tbz             x7, #1, .Lout192
 736         b               .Lout256
 737         .previous
 738
 739 .Lout256:
 740         .irp            key, K6, K7
 741         enc_qround      KS0, KS1, KS2, KS3, \key
 742         .endr
 743
 744 .Lout192:
 745         enc_qround      KS0, KS1, KS2, KS3, KK
 746
 747         aese            KS0.16b, KL.16b
 748         aese            KS1.16b, KL.16b
 749         aese            KS2.16b, KL.16b
 750         aese            KS3.16b, KL.16b
 751
 752         eor             KS0.16b, KS0.16b, KM.16b
 753         eor             KS1.16b, KS1.16b, KM.16b
 754         eor             KS2.16b, KS2.16b, KM.16b
 755         eor             KS3.16b, KS3.16b, KM.16b
 756
 757         eor             INP0.16b, INP0.16b, KS0.16b
 758         eor             INP1.16b, INP1.16b, KS1.16b
 759         eor             INP2.16b, INP2.16b, KS2.16b
 760         eor             INP3.16b, INP3.16b, KS3.16b
 761
 762         ret
 763 SYM_FUNC_END(pmull_gcm_enc_4x)
 764
 765         .section        ".rodata", "a"
 766         .align          6
 767 .Lpermute_table:
 768         .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 769         .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 770         .byte            0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
 771         .byte            0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe,  0xf
 772         .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 773         .byte           0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 774         .byte            0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
 775         .byte            0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe,  0xf
 776         .previous