2 * algif_aead: User-space interface for AEAD algorithms
4 * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
6 * This file provides the user-space API for AEAD ciphers.
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
13 * The following concept of the memory management is used:
15 * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
16 * filled by user space with the data submitted via sendpage/sendmsg. Filling
17 * up the TX SGL does not cause a crypto operation -- the data will only be
18 * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
19 * provide a buffer which is tracked with the RX SGL.
21 * During the processing of the recvmsg operation, the cipher request is
22 * allocated and prepared. As part of the recvmsg operation, the processed
23 * TX buffers are extracted from the TX SGL into a separate SGL.
25 * After the completion of the crypto operation, the RX SGL and the cipher
26 * request is released. The extracted TX SGL parts are released together with
30 #include <crypto/internal/aead.h>
31 #include <crypto/scatterwalk.h>
32 #include <crypto/if_alg.h>
33 #include <crypto/skcipher.h>
34 #include <crypto/null.h>
35 #include <linux/init.h>
36 #include <linux/list.h>
37 #include <linux/kernel.h>
39 #include <linux/module.h>
40 #include <linux/net.h>
44 struct crypto_aead *aead;
46 struct crypto_skcipher *null_tfm;
49 static inline bool aead_sufficient_data(struct sock *sk)
51 struct alg_sock *ask = alg_sk(sk);
52 struct sock *psk = ask->parent;
53 struct alg_sock *pask = alg_sk(psk);
54 struct af_alg_ctx *ctx = ask->private;
55 struct aead_tfm *aeadc = pask->private;
56 struct crypto_aead *tfm = aeadc->aead;
57 unsigned int as = crypto_aead_authsize(tfm);
60 * The minimum amount of memory needed for an AEAD cipher is
61 * the AAD and in case of decryption the tag.
63 return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
66 static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
68 struct sock *sk = sock->sk;
69 struct alg_sock *ask = alg_sk(sk);
70 struct sock *psk = ask->parent;
71 struct alg_sock *pask = alg_sk(psk);
72 struct aead_tfm *aeadc = pask->private;
73 struct crypto_aead *tfm = aeadc->aead;
74 unsigned int ivsize = crypto_aead_ivsize(tfm);
76 return af_alg_sendmsg(sock, msg, size, ivsize);
79 static int crypto_aead_copy_sgl(struct crypto_skcipher *null_tfm,
80 struct scatterlist *src,
81 struct scatterlist *dst, unsigned int len)
83 SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);
85 skcipher_request_set_tfm(skreq, null_tfm);
86 skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_SLEEP,
88 skcipher_request_set_crypt(skreq, src, dst, len, NULL);
90 return crypto_skcipher_encrypt(skreq);
93 static int _aead_recvmsg(struct socket *sock, struct msghdr *msg,
94 size_t ignored, int flags)
96 struct sock *sk = sock->sk;
97 struct alg_sock *ask = alg_sk(sk);
98 struct sock *psk = ask->parent;
99 struct alg_sock *pask = alg_sk(psk);
100 struct af_alg_ctx *ctx = ask->private;
101 struct aead_tfm *aeadc = pask->private;
102 struct crypto_aead *tfm = aeadc->aead;
103 struct crypto_skcipher *null_tfm = aeadc->null_tfm;
104 unsigned int i, as = crypto_aead_authsize(tfm);
105 struct af_alg_async_req *areq;
106 struct af_alg_tsgl *tsgl, *tmp;
107 struct scatterlist *rsgl_src, *tsgl_src = NULL;
109 size_t used = 0; /* [in] TX bufs to be en/decrypted */
110 size_t outlen = 0; /* [out] RX bufs produced by kernel */
111 size_t usedpages = 0; /* [in] RX bufs to be used from user */
112 size_t processed = 0; /* [in] TX bufs to be consumed */
115 err = af_alg_wait_for_data(sk, flags);
121 * Data length provided by caller via sendmsg/sendpage that has not
122 * yet been processed.
127 * Make sure sufficient data is present -- note, the same check is
128 * is also present in sendmsg/sendpage. The checks in sendpage/sendmsg
129 * shall provide an information to the data sender that something is
130 * wrong, but they are irrelevant to maintain the kernel integrity.
131 * We need this check here too in case user space decides to not honor
132 * the error message in sendmsg/sendpage and still call recvmsg. This
133 * check here protects the kernel integrity.
135 if (!aead_sufficient_data(sk))
139 * Calculate the minimum output buffer size holding the result of the
140 * cipher operation. When encrypting data, the receiving buffer is
141 * larger by the tag length compared to the input buffer as the
142 * encryption operation generates the tag. For decryption, the input
143 * buffer provides the tag which is consumed resulting in only the
144 * plaintext without a buffer for the tag returned to the caller.
152 * The cipher operation input data is reduced by the associated data
153 * length as this data is processed separately later on.
155 used -= ctx->aead_assoclen;
157 /* Allocate cipher request for current operation. */
158 areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) +
159 crypto_aead_reqsize(tfm));
161 return PTR_ERR(areq);
163 /* convert iovecs of output buffers into RX SGL */
164 err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages);
169 * Ensure output buffer is sufficiently large. If the caller provides
170 * less buffer space, only use the relative required input size. This
171 * allows AIO operation where the caller sent all data to be processed
172 * and the AIO operation performs the operation on the different chunks
175 if (usedpages < outlen) {
176 size_t less = outlen - usedpages;
186 processed = used + ctx->aead_assoclen;
187 list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) {
188 for (i = 0; i < tsgl->cur; i++) {
189 struct scatterlist *process_sg = tsgl->sg + i;
191 if (!(process_sg->length) || !sg_page(process_sg))
193 tsgl_src = process_sg;
199 if (processed && !tsgl_src) {
205 * Copy of AAD from source to destination
207 * The AAD is copied to the destination buffer without change. Even
208 * when user space uses an in-place cipher operation, the kernel
209 * will copy the data as it does not see whether such in-place operation
212 * To ensure efficiency, the following implementation ensure that the
213 * ciphers are invoked to perform a crypto operation in-place. This
214 * is achieved by memory management specified as follows.
217 /* Use the RX SGL as source (and destination) for crypto op. */
218 rsgl_src = areq->first_rsgl.sgl.sg;
222 * Encryption operation - The in-place cipher operation is
223 * achieved by the following operation:
229 * RX SGL: AAD || PT || Tag
231 err = crypto_aead_copy_sgl(null_tfm, tsgl_src,
232 areq->first_rsgl.sgl.sg, processed);
235 af_alg_pull_tsgl(sk, processed, NULL, 0);
238 * Decryption operation - To achieve an in-place cipher
239 * operation, the following SGL structure is used:
241 * TX SGL: AAD || CT || Tag
243 * | copy | | Create SGL link.
245 * RX SGL: AAD || CT ----+
248 /* Copy AAD || CT to RX SGL buffer for in-place operation. */
249 err = crypto_aead_copy_sgl(null_tfm, tsgl_src,
250 areq->first_rsgl.sgl.sg, outlen);
254 /* Create TX SGL for tag and chain it to RX SGL. */
255 areq->tsgl_entries = af_alg_count_tsgl(sk, processed,
257 if (!areq->tsgl_entries)
258 areq->tsgl_entries = 1;
259 areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) *
266 sg_init_table(areq->tsgl, areq->tsgl_entries);
268 /* Release TX SGL, except for tag data and reassign tag data. */
269 af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as);
271 /* chain the areq TX SGL holding the tag with RX SGL */
274 struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl;
276 sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1);
277 sg_chain(sgl_prev->sg, sgl_prev->npages + 1,
280 /* no RX SGL present (e.g. authentication only) */
281 rsgl_src = areq->tsgl;
284 /* Initialize the crypto operation */
285 aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src,
286 areq->first_rsgl.sgl.sg, used, ctx->iv);
287 aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen);
288 aead_request_set_tfm(&areq->cra_u.aead_req, tfm);
290 if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
293 areq->iocb = msg->msg_iocb;
295 /* Remember output size that will be generated. */
296 areq->outlen = outlen;
298 aead_request_set_callback(&areq->cra_u.aead_req,
299 CRYPTO_TFM_REQ_MAY_SLEEP,
300 af_alg_async_cb, areq);
301 err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) :
302 crypto_aead_decrypt(&areq->cra_u.aead_req);
304 /* AIO operation in progress */
305 if (err == -EINPROGRESS)
310 /* Synchronous operation */
311 aead_request_set_callback(&areq->cra_u.aead_req,
312 CRYPTO_TFM_REQ_MAY_SLEEP |
313 CRYPTO_TFM_REQ_MAY_BACKLOG,
314 af_alg_complete, &ctx->completion);
315 err = af_alg_wait_for_completion(ctx->enc ?
316 crypto_aead_encrypt(&areq->cra_u.aead_req) :
317 crypto_aead_decrypt(&areq->cra_u.aead_req),
323 af_alg_free_resources(areq);
325 return err ? err : outlen;
328 static int aead_recvmsg(struct socket *sock, struct msghdr *msg,
329 size_t ignored, int flags)
331 struct sock *sk = sock->sk;
335 while (msg_data_left(msg)) {
336 int err = _aead_recvmsg(sock, msg, ignored, flags);
339 * This error covers -EIOCBQUEUED which implies that we can
340 * only handle one AIO request. If the caller wants to have
341 * multiple AIO requests in parallel, he must make multiple
342 * separate AIO calls.
344 * Also return the error if no data has been processed so far.
347 if (err == -EIOCBQUEUED || err == -EBADMSG || !ret)
356 af_alg_wmem_wakeup(sk);
361 static struct proto_ops algif_aead_ops = {
364 .connect = sock_no_connect,
365 .socketpair = sock_no_socketpair,
366 .getname = sock_no_getname,
367 .ioctl = sock_no_ioctl,
368 .listen = sock_no_listen,
369 .shutdown = sock_no_shutdown,
370 .getsockopt = sock_no_getsockopt,
371 .mmap = sock_no_mmap,
372 .bind = sock_no_bind,
373 .accept = sock_no_accept,
374 .setsockopt = sock_no_setsockopt,
376 .release = af_alg_release,
377 .sendmsg = aead_sendmsg,
378 .sendpage = af_alg_sendpage,
379 .recvmsg = aead_recvmsg,
383 static int aead_check_key(struct socket *sock)
387 struct alg_sock *pask;
388 struct aead_tfm *tfm;
389 struct sock *sk = sock->sk;
390 struct alg_sock *ask = alg_sk(sk);
393 if (!atomic_read(&ask->nokey_refcnt))
397 pask = alg_sk(ask->parent);
401 lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
405 atomic_dec(&pask->nokey_refcnt);
406 atomic_set(&ask->nokey_refcnt, 0);
418 static int aead_sendmsg_nokey(struct socket *sock, struct msghdr *msg,
423 err = aead_check_key(sock);
427 return aead_sendmsg(sock, msg, size);
430 static ssize_t aead_sendpage_nokey(struct socket *sock, struct page *page,
431 int offset, size_t size, int flags)
435 err = aead_check_key(sock);
439 return af_alg_sendpage(sock, page, offset, size, flags);
442 static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
443 size_t ignored, int flags)
447 err = aead_check_key(sock);
451 return aead_recvmsg(sock, msg, ignored, flags);
454 static struct proto_ops algif_aead_ops_nokey = {
457 .connect = sock_no_connect,
458 .socketpair = sock_no_socketpair,
459 .getname = sock_no_getname,
460 .ioctl = sock_no_ioctl,
461 .listen = sock_no_listen,
462 .shutdown = sock_no_shutdown,
463 .getsockopt = sock_no_getsockopt,
464 .mmap = sock_no_mmap,
465 .bind = sock_no_bind,
466 .accept = sock_no_accept,
467 .setsockopt = sock_no_setsockopt,
469 .release = af_alg_release,
470 .sendmsg = aead_sendmsg_nokey,
471 .sendpage = aead_sendpage_nokey,
472 .recvmsg = aead_recvmsg_nokey,
476 static void *aead_bind(const char *name, u32 type, u32 mask)
478 struct aead_tfm *tfm;
479 struct crypto_aead *aead;
480 struct crypto_skcipher *null_tfm;
482 tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
484 return ERR_PTR(-ENOMEM);
486 aead = crypto_alloc_aead(name, type, mask);
489 return ERR_CAST(aead);
492 null_tfm = crypto_get_default_null_skcipher2();
493 if (IS_ERR(null_tfm)) {
494 crypto_free_aead(aead);
496 return ERR_CAST(null_tfm);
500 tfm->null_tfm = null_tfm;
505 static void aead_release(void *private)
507 struct aead_tfm *tfm = private;
509 crypto_free_aead(tfm->aead);
510 crypto_put_default_null_skcipher2();
514 static int aead_setauthsize(void *private, unsigned int authsize)
516 struct aead_tfm *tfm = private;
518 return crypto_aead_setauthsize(tfm->aead, authsize);
521 static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
523 struct aead_tfm *tfm = private;
526 err = crypto_aead_setkey(tfm->aead, key, keylen);
532 static void aead_sock_destruct(struct sock *sk)
534 struct alg_sock *ask = alg_sk(sk);
535 struct af_alg_ctx *ctx = ask->private;
536 struct sock *psk = ask->parent;
537 struct alg_sock *pask = alg_sk(psk);
538 struct aead_tfm *aeadc = pask->private;
539 struct crypto_aead *tfm = aeadc->aead;
540 unsigned int ivlen = crypto_aead_ivsize(tfm);
542 af_alg_pull_tsgl(sk, ctx->used, NULL, 0);
543 sock_kzfree_s(sk, ctx->iv, ivlen);
544 sock_kfree_s(sk, ctx, ctx->len);
545 af_alg_release_parent(sk);
548 static int aead_accept_parent_nokey(void *private, struct sock *sk)
550 struct af_alg_ctx *ctx;
551 struct alg_sock *ask = alg_sk(sk);
552 struct aead_tfm *tfm = private;
553 struct crypto_aead *aead = tfm->aead;
554 unsigned int len = sizeof(*ctx);
555 unsigned int ivlen = crypto_aead_ivsize(aead);
557 ctx = sock_kmalloc(sk, len, GFP_KERNEL);
562 ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);
564 sock_kfree_s(sk, ctx, len);
567 memset(ctx->iv, 0, ivlen);
569 INIT_LIST_HEAD(&ctx->tsgl_list);
572 atomic_set(&ctx->rcvused, 0);
576 ctx->aead_assoclen = 0;
577 af_alg_init_completion(&ctx->completion);
581 sk->sk_destruct = aead_sock_destruct;
586 static int aead_accept_parent(void *private, struct sock *sk)
588 struct aead_tfm *tfm = private;
593 return aead_accept_parent_nokey(private, sk);
596 static const struct af_alg_type algif_type_aead = {
598 .release = aead_release,
599 .setkey = aead_setkey,
600 .setauthsize = aead_setauthsize,
601 .accept = aead_accept_parent,
602 .accept_nokey = aead_accept_parent_nokey,
603 .ops = &algif_aead_ops,
604 .ops_nokey = &algif_aead_ops_nokey,
609 static int __init algif_aead_init(void)
611 return af_alg_register_type(&algif_type_aead);
614 static void __exit algif_aead_exit(void)
616 int err = af_alg_unregister_type(&algif_type_aead);
620 module_init(algif_aead_init);
621 module_exit(algif_aead_exit);
622 MODULE_LICENSE("GPL");
623 MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
624 MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");