arch/x86/crypto/glue_helper.c

   1 /*
   2  * Shared glue code for 128bit block ciphers
   3  *
   4  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   5  *
   6  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
   7  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
   8  * CTR part based on code (crypto/ctr.c) by:
   9  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program; if not, write to the Free Software
  23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  24  * USA
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <crypto/b128ops.h>
  30 #include <crypto/gf128mul.h>
  31 #include <crypto/internal/skcipher.h>
  32 #include <crypto/lrw.h>
  33 #include <crypto/xts.h>
  34 #include <asm/crypto/glue_helper.h>
  35
  36 static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
  37                                    struct blkcipher_desc *desc,
  38                                    struct blkcipher_walk *walk)
  39 {
  40         void *ctx = crypto_blkcipher_ctx(desc->tfm);
  41         const unsigned int bsize = 128 / 8;
  42         unsigned int nbytes, i, func_bytes;
  43         bool fpu_enabled = false;
  44         int err;
  45
  46         err = blkcipher_walk_virt(desc, walk);
  47
  48         while ((nbytes = walk->nbytes)) {
  49                 u8 *wsrc = walk->src.virt.addr;
  50                 u8 *wdst = walk->dst.virt.addr;
  51
  52                 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  53                                              desc, fpu_enabled, nbytes);
  54
  55                 for (i = 0; i < gctx->num_funcs; i++) {
  56                         func_bytes = bsize * gctx->funcs[i].num_blocks;
  57
  58                         /* Process multi-block batch */
  59                         if (nbytes >= func_bytes) {
  60                                 do {
  61                                         gctx->funcs[i].fn_u.ecb(ctx, wdst,
  62                                                                 wsrc);
  63
  64                                         wsrc += func_bytes;
  65                                         wdst += func_bytes;
  66                                         nbytes -= func_bytes;
  67                                 } while (nbytes >= func_bytes);
  68
  69                                 if (nbytes < bsize)
  70                                         goto done;
  71                         }
  72                 }
  73
  74 done:
  75                 err = blkcipher_walk_done(desc, walk, nbytes);
  76         }
  77
  78         glue_fpu_end(fpu_enabled);
  79         return err;
  80 }
  81
  82 int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
  83                           struct blkcipher_desc *desc, struct scatterlist *dst,
  84                           struct scatterlist *src, unsigned int nbytes)
  85 {
  86         struct blkcipher_walk walk;
  87
  88         blkcipher_walk_init(&walk, dst, src, nbytes);
  89         return __glue_ecb_crypt_128bit(gctx, desc, &walk);
  90 }
  91 EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit);
  92
  93 static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
  94                                               struct blkcipher_desc *desc,
  95                                               struct blkcipher_walk *walk)
  96 {
  97         void *ctx = crypto_blkcipher_ctx(desc->tfm);
  98         const unsigned int bsize = 128 / 8;
  99         unsigned int nbytes = walk->nbytes;
 100         u128 *src = (u128 *)walk->src.virt.addr;
 101         u128 *dst = (u128 *)walk->dst.virt.addr;
 102         u128 *iv = (u128 *)walk->iv;
 103
 104         do {
 105                 u128_xor(dst, src, iv);
 106                 fn(ctx, (u8 *)dst, (u8 *)dst);
 107                 iv = dst;
 108
 109                 src += 1;
 110                 dst += 1;
 111                 nbytes -= bsize;
 112         } while (nbytes >= bsize);
 113
 114         *(u128 *)walk->iv = *iv;
 115         return nbytes;
 116 }
 117
 118 int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
 119                             struct blkcipher_desc *desc,
 120                             struct scatterlist *dst,
 121                             struct scatterlist *src, unsigned int nbytes)
 122 {
 123         struct blkcipher_walk walk;
 124         int err;
 125
 126         blkcipher_walk_init(&walk, dst, src, nbytes);
 127         err = blkcipher_walk_virt(desc, &walk);
 128
 129         while ((nbytes = walk.nbytes)) {
 130                 nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk);
 131                 err = blkcipher_walk_done(desc, &walk, nbytes);
 132         }
 133
 134         return err;
 135 }
 136 EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit);
 137
 138 static unsigned int
 139 __glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
 140                           struct blkcipher_desc *desc,
 141                           struct blkcipher_walk *walk)
 142 {
 143         void *ctx = crypto_blkcipher_ctx(desc->tfm);
 144         const unsigned int bsize = 128 / 8;
 145         unsigned int nbytes = walk->nbytes;
 146         u128 *src = (u128 *)walk->src.virt.addr;
 147         u128 *dst = (u128 *)walk->dst.virt.addr;
 148         u128 last_iv;
 149         unsigned int num_blocks, func_bytes;
 150         unsigned int i;
 151
 152         /* Start of the last block. */
 153         src += nbytes / bsize - 1;
 154         dst += nbytes / bsize - 1;
 155
 156         last_iv = *src;
 157
 158         for (i = 0; i < gctx->num_funcs; i++) {
 159                 num_blocks = gctx->funcs[i].num_blocks;
 160                 func_bytes = bsize * num_blocks;
 161
 162                 /* Process multi-block batch */
 163                 if (nbytes >= func_bytes) {
 164                         do {
 165                                 nbytes -= func_bytes - bsize;
 166                                 src -= num_blocks - 1;
 167                                 dst -= num_blocks - 1;
 168
 169                                 gctx->funcs[i].fn_u.cbc(ctx, dst, src);
 170
 171                                 nbytes -= bsize;
 172                                 if (nbytes < bsize)
 173                                         goto done;
 174
 175                                 u128_xor(dst, dst, src - 1);
 176                                 src -= 1;
 177                                 dst -= 1;
 178                         } while (nbytes >= func_bytes);
 179                 }
 180         }
 181
 182 done:
 183         u128_xor(dst, dst, (u128 *)walk->iv);
 184         *(u128 *)walk->iv = last_iv;
 185
 186         return nbytes;
 187 }
 188
 189 int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
 190                             struct blkcipher_desc *desc,
 191                             struct scatterlist *dst,
 192                             struct scatterlist *src, unsigned int nbytes)
 193 {
 194         const unsigned int bsize = 128 / 8;
 195         bool fpu_enabled = false;
 196         struct blkcipher_walk walk;
 197         int err;
 198
 199         blkcipher_walk_init(&walk, dst, src, nbytes);
 200         err = blkcipher_walk_virt(desc, &walk);
 201
 202         while ((nbytes = walk.nbytes)) {
 203                 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 204                                              desc, fpu_enabled, nbytes);
 205                 nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
 206                 err = blkcipher_walk_done(desc, &walk, nbytes);
 207         }
 208
 209         glue_fpu_end(fpu_enabled);
 210         return err;
 211 }
 212 EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);
 213
 214 static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
 215                                         struct blkcipher_desc *desc,
 216                                         struct blkcipher_walk *walk)
 217 {
 218         void *ctx = crypto_blkcipher_ctx(desc->tfm);
 219         u8 *src = (u8 *)walk->src.virt.addr;
 220         u8 *dst = (u8 *)walk->dst.virt.addr;
 221         unsigned int nbytes = walk->nbytes;
 222         le128 ctrblk;
 223         u128 tmp;
 224
 225         be128_to_le128(&ctrblk, (be128 *)walk->iv);
 226
 227         memcpy(&tmp, src, nbytes);
 228         fn_ctr(ctx, &tmp, &tmp, &ctrblk);
 229         memcpy(dst, &tmp, nbytes);
 230
 231         le128_to_be128((be128 *)walk->iv, &ctrblk);
 232 }
 233
 234 static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
 235                                             struct blkcipher_desc *desc,
 236                                             struct blkcipher_walk *walk)
 237 {
 238         const unsigned int bsize = 128 / 8;
 239         void *ctx = crypto_blkcipher_ctx(desc->tfm);
 240         unsigned int nbytes = walk->nbytes;
 241         u128 *src = (u128 *)walk->src.virt.addr;
 242         u128 *dst = (u128 *)walk->dst.virt.addr;
 243         le128 ctrblk;
 244         unsigned int num_blocks, func_bytes;
 245         unsigned int i;
 246
 247         be128_to_le128(&ctrblk, (be128 *)walk->iv);
 248
 249         /* Process multi-block batch */
 250         for (i = 0; i < gctx->num_funcs; i++) {
 251                 num_blocks = gctx->funcs[i].num_blocks;
 252                 func_bytes = bsize * num_blocks;
 253
 254                 if (nbytes >= func_bytes) {
 255                         do {
 256                                 gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk);
 257
 258                                 src += num_blocks;
 259                                 dst += num_blocks;
 260                                 nbytes -= func_bytes;
 261                         } while (nbytes >= func_bytes);
 262
 263                         if (nbytes < bsize)
 264                                 goto done;
 265                 }
 266         }
 267
 268 done:
 269         le128_to_be128((be128 *)walk->iv, &ctrblk);
 270         return nbytes;
 271 }
 272
 273 int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
 274                           struct blkcipher_desc *desc, struct scatterlist *dst,
 275                           struct scatterlist *src, unsigned int nbytes)
 276 {
 277         const unsigned int bsize = 128 / 8;
 278         bool fpu_enabled = false;
 279         struct blkcipher_walk walk;
 280         int err;
 281
 282         blkcipher_walk_init(&walk, dst, src, nbytes);
 283         err = blkcipher_walk_virt_block(desc, &walk, bsize);
 284
 285         while ((nbytes = walk.nbytes) >= bsize) {
 286                 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 287                                              desc, fpu_enabled, nbytes);
 288                 nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
 289                 err = blkcipher_walk_done(desc, &walk, nbytes);
 290         }
 291
 292         glue_fpu_end(fpu_enabled);
 293
 294         if (walk.nbytes) {
 295                 glue_ctr_crypt_final_128bit(
 296                         gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
 297                 err = blkcipher_walk_done(desc, &walk, 0);
 298         }
 299
 300         return err;
 301 }
 302 EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit);
 303
 304 static unsigned int __glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 305                                             void *ctx,
 306                                             struct blkcipher_desc *desc,
 307                                             struct blkcipher_walk *walk)
 308 {
 309         const unsigned int bsize = 128 / 8;
 310         unsigned int nbytes = walk->nbytes;
 311         u128 *src = (u128 *)walk->src.virt.addr;
 312         u128 *dst = (u128 *)walk->dst.virt.addr;
 313         unsigned int num_blocks, func_bytes;
 314         unsigned int i;
 315
 316         /* Process multi-block batch */
 317         for (i = 0; i < gctx->num_funcs; i++) {
 318                 num_blocks = gctx->funcs[i].num_blocks;
 319                 func_bytes = bsize * num_blocks;
 320
 321                 if (nbytes >= func_bytes) {
 322                         do {
 323                                 gctx->funcs[i].fn_u.xts(ctx, dst, src,
 324                                                         (le128 *)walk->iv);
 325
 326                                 src += num_blocks;
 327                                 dst += num_blocks;
 328                                 nbytes -= func_bytes;
 329                         } while (nbytes >= func_bytes);
 330
 331                         if (nbytes < bsize)
 332                                 goto done;
 333                 }
 334         }
 335
 336 done:
 337         return nbytes;
 338 }
 339
 340 static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx,
 341                                           void *ctx,
 342                                           struct skcipher_walk *walk)
 343 {
 344         const unsigned int bsize = 128 / 8;
 345         unsigned int nbytes = walk->nbytes;
 346         u128 *src = walk->src.virt.addr;
 347         u128 *dst = walk->dst.virt.addr;
 348         unsigned int num_blocks, func_bytes;
 349         unsigned int i;
 350
 351         /* Process multi-block batch */
 352         for (i = 0; i < gctx->num_funcs; i++) {
 353                 num_blocks = gctx->funcs[i].num_blocks;
 354                 func_bytes = bsize * num_blocks;
 355
 356                 if (nbytes >= func_bytes) {
 357                         do {
 358                                 gctx->funcs[i].fn_u.xts(ctx, dst, src,
 359                                                         walk->iv);
 360
 361                                 src += num_blocks;
 362                                 dst += num_blocks;
 363                                 nbytes -= func_bytes;
 364                         } while (nbytes >= func_bytes);
 365
 366                         if (nbytes < bsize)
 367                                 goto done;
 368                 }
 369         }
 370
 371 done:
 372         return nbytes;
 373 }
 374
 375 /* for implementations implementing faster XTS IV generator */
 376 int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 377                           struct blkcipher_desc *desc, struct scatterlist *dst,
 378                           struct scatterlist *src, unsigned int nbytes,
 379                           void (*tweak_fn)(void *ctx, u8 *dst, const u8 *src),
 380                           void *tweak_ctx, void *crypt_ctx)
 381 {
 382         const unsigned int bsize = 128 / 8;
 383         bool fpu_enabled = false;
 384         struct blkcipher_walk walk;
 385         int err;
 386
 387         blkcipher_walk_init(&walk, dst, src, nbytes);
 388
 389         err = blkcipher_walk_virt(desc, &walk);
 390         nbytes = walk.nbytes;
 391         if (!nbytes)
 392                 return err;
 393
 394         /* set minimum length to bsize, for tweak_fn */
 395         fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 396                                      desc, fpu_enabled,
 397                                      nbytes < bsize ? bsize : nbytes);
 398
 399         /* calculate first value of T */
 400         tweak_fn(tweak_ctx, walk.iv, walk.iv);
 401
 402         while (nbytes) {
 403                 nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk);
 404
 405                 err = blkcipher_walk_done(desc, &walk, nbytes);
 406                 nbytes = walk.nbytes;
 407         }
 408
 409         glue_fpu_end(fpu_enabled);
 410
 411         return err;
 412 }
 413 EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);
 414
 415 int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
 416                         struct skcipher_request *req,
 417                         common_glue_func_t tweak_fn, void *tweak_ctx,
 418                         void *crypt_ctx)
 419 {
 420         const unsigned int bsize = 128 / 8;
 421         struct skcipher_walk walk;
 422         bool fpu_enabled = false;
 423         unsigned int nbytes;
 424         int err;
 425
 426         err = skcipher_walk_virt(&walk, req, false);
 427         nbytes = walk.nbytes;
 428         if (!nbytes)
 429                 return err;
 430
 431         /* set minimum length to bsize, for tweak_fn */
 432         fpu_enabled = glue_skwalk_fpu_begin(bsize, gctx->fpu_blocks_limit,
 433                                             &walk, fpu_enabled,
 434                                             nbytes < bsize ? bsize : nbytes);
 435
 436         /* calculate first value of T */
 437         tweak_fn(tweak_ctx, walk.iv, walk.iv);
 438
 439         while (nbytes) {
 440                 nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk);
 441
 442                 err = skcipher_walk_done(&walk, nbytes);
 443                 nbytes = walk.nbytes;
 444         }
 445
 446         glue_fpu_end(fpu_enabled);
 447
 448         return err;
 449 }
 450 EXPORT_SYMBOL_GPL(glue_xts_req_128bit);
 451
 452 void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
 453                                common_glue_func_t fn)
 454 {
 455         le128 ivblk = *iv;
 456
 457         /* generate next IV */
 458         gf128mul_x_ble(iv, &ivblk);
 459
 460         /* CC <- T xor C */
 461         u128_xor(dst, src, (u128 *)&ivblk);
 462
 463         /* PP <- D(Key2,CC) */
 464         fn(ctx, (u8 *)dst, (u8 *)dst);
 465
 466         /* P <- T xor PP */
 467         u128_xor(dst, dst, (u128 *)&ivblk);
 468 }
 469 EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);
 470
 471 MODULE_LICENSE("GPL");