1 // SPDX-License-Identifier: GPL-2.0-or-later
3 #include <crypto/hash.h>
5 #include <linux/kref.h>
6 #include <linux/module.h>
7 #include <linux/mutex.h>
8 #include <linux/percpu.h>
9 #include <linux/workqueue.h>
12 static size_t __scratch_size;
13 static DEFINE_PER_CPU(void __rcu *, sigpool_scratch);
15 struct sigpool_entry {
16 struct crypto_ahash *hash;
23 #define CPOOL_SIZE (PAGE_SIZE / sizeof(struct sigpool_entry))
24 static struct sigpool_entry cpool[CPOOL_SIZE];
25 static unsigned int cpool_populated;
26 static DEFINE_MUTEX(cpool_mutex);
29 struct scratches_to_free {
35 static void free_old_scratches(struct rcu_head *head)
37 struct scratches_to_free *stf;
39 stf = container_of(head, struct scratches_to_free, rcu);
41 kfree(stf->scratches[stf->cnt]);
46 * sigpool_reserve_scratch - re-allocates scratch buffer, slow-path
47 * @size: request size for the scratch/temp buffer
49 static int sigpool_reserve_scratch(size_t size)
51 struct scratches_to_free *stf;
52 size_t stf_sz = struct_size(stf, scratches, num_possible_cpus());
55 lockdep_assert_held(&cpool_mutex);
56 if (__scratch_size >= size)
59 stf = kmalloc(stf_sz, GFP_KERNEL);
64 size = max(size, __scratch_size);
66 for_each_possible_cpu(cpu) {
67 void *scratch, *old_scratch;
69 scratch = kmalloc_node(size, GFP_KERNEL, cpu_to_node(cpu));
75 old_scratch = rcu_replace_pointer(per_cpu(sigpool_scratch, cpu),
76 scratch, lockdep_is_held(&cpool_mutex));
77 if (!cpu_online(cpu) || !old_scratch) {
81 stf->scratches[stf->cnt++] = old_scratch;
85 __scratch_size = size;
87 call_rcu(&stf->rcu, free_old_scratches);
91 static void sigpool_scratch_free(void)
95 for_each_possible_cpu(cpu)
96 kfree(rcu_replace_pointer(per_cpu(sigpool_scratch, cpu),
97 NULL, lockdep_is_held(&cpool_mutex)));
101 static int __cpool_try_clone(struct crypto_ahash *hash)
103 struct crypto_ahash *tmp;
105 tmp = crypto_clone_ahash(hash);
109 crypto_free_ahash(tmp);
113 static int __cpool_alloc_ahash(struct sigpool_entry *e, const char *alg)
115 struct crypto_ahash *cpu0_hash;
118 e->alg = kstrdup(alg, GFP_KERNEL);
122 cpu0_hash = crypto_alloc_ahash(alg, 0, CRYPTO_ALG_ASYNC);
123 if (IS_ERR(cpu0_hash)) {
124 ret = PTR_ERR(cpu0_hash);
128 e->needs_key = crypto_ahash_get_flags(cpu0_hash) & CRYPTO_TFM_NEED_KEY;
130 ret = __cpool_try_clone(cpu0_hash);
132 goto out_free_cpu0_hash;
138 crypto_free_ahash(cpu0_hash);
146 * tcp_sigpool_alloc_ahash - allocates pool for ahash requests
147 * @alg: name of async hash algorithm
148 * @scratch_size: reserve a tcp_sigpool::scratch buffer of this size
150 int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size)
155 mutex_lock(&cpool_mutex);
156 ret = sigpool_reserve_scratch(scratch_size);
159 for (i = 0; i < cpool_populated; i++) {
162 if (strcmp(cpool[i].alg, alg))
165 if (kref_read(&cpool[i].kref) > 0)
166 kref_get(&cpool[i].kref);
168 kref_init(&cpool[i].kref);
173 for (i = 0; i < cpool_populated; i++) {
177 if (i >= CPOOL_SIZE) {
182 ret = __cpool_alloc_ahash(&cpool[i], alg);
185 if (i == cpool_populated)
189 mutex_unlock(&cpool_mutex);
192 EXPORT_SYMBOL_GPL(tcp_sigpool_alloc_ahash);
194 static void __cpool_free_entry(struct sigpool_entry *e)
196 crypto_free_ahash(e->hash);
198 memset(e, 0, sizeof(*e));
201 static void cpool_cleanup_work_cb(struct work_struct *work)
203 bool free_scratch = true;
206 mutex_lock(&cpool_mutex);
207 for (i = 0; i < cpool_populated; i++) {
208 if (kref_read(&cpool[i].kref) > 0) {
209 free_scratch = false;
214 __cpool_free_entry(&cpool[i]);
217 sigpool_scratch_free();
218 mutex_unlock(&cpool_mutex);
221 static DECLARE_WORK(cpool_cleanup_work, cpool_cleanup_work_cb);
222 static void cpool_schedule_cleanup(struct kref *kref)
224 schedule_work(&cpool_cleanup_work);
228 * tcp_sigpool_release - decreases number of users for a pool. If it was
229 * the last user of the pool, releases any memory that was consumed.
230 * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
232 void tcp_sigpool_release(unsigned int id)
234 if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
238 kref_put(&cpool[id].kref, cpool_schedule_cleanup);
240 EXPORT_SYMBOL_GPL(tcp_sigpool_release);
243 * tcp_sigpool_get - increases number of users (refcounter) for a pool
244 * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
246 void tcp_sigpool_get(unsigned int id)
248 if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
250 kref_get(&cpool[id].kref);
252 EXPORT_SYMBOL_GPL(tcp_sigpool_get);
254 int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c) __cond_acquires(RCU_BH)
256 struct crypto_ahash *hash;
259 if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg)) {
260 rcu_read_unlock_bh();
264 hash = crypto_clone_ahash(cpool[id].hash);
266 rcu_read_unlock_bh();
267 return PTR_ERR(hash);
270 c->req = ahash_request_alloc(hash, GFP_ATOMIC);
272 crypto_free_ahash(hash);
273 rcu_read_unlock_bh();
276 ahash_request_set_callback(c->req, 0, NULL, NULL);
278 /* Pairs with tcp_sigpool_reserve_scratch(), scratch area is
279 * valid (allocated) until tcp_sigpool_end().
281 c->scratch = rcu_dereference_bh(*this_cpu_ptr(&sigpool_scratch));
284 EXPORT_SYMBOL_GPL(tcp_sigpool_start);
286 void tcp_sigpool_end(struct tcp_sigpool *c) __releases(RCU_BH)
288 struct crypto_ahash *hash = crypto_ahash_reqtfm(c->req);
290 rcu_read_unlock_bh();
291 ahash_request_free(c->req);
292 crypto_free_ahash(hash);
294 EXPORT_SYMBOL_GPL(tcp_sigpool_end);
297 * tcp_sigpool_algo - return algorithm of tcp_sigpool
298 * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
299 * @buf: buffer to return name of algorithm
300 * @buf_len: size of @buf
302 size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len)
304 if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
307 return strscpy(buf, cpool[id].alg, buf_len);
309 EXPORT_SYMBOL_GPL(tcp_sigpool_algo);
312 * tcp_sigpool_hash_skb_data - hash data in skb with initialized tcp_sigpool
313 * @hp: tcp_sigpool pointer
314 * @skb: buffer to add sign for
315 * @header_len: TCP header length for this segment
317 int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp,
318 const struct sk_buff *skb,
319 unsigned int header_len)
321 const unsigned int head_data_len = skb_headlen(skb) > header_len ?
322 skb_headlen(skb) - header_len : 0;
323 const struct skb_shared_info *shi = skb_shinfo(skb);
324 const struct tcphdr *tp = tcp_hdr(skb);
325 struct ahash_request *req = hp->req;
326 struct sk_buff *frag_iter;
327 struct scatterlist sg;
330 sg_init_table(&sg, 1);
332 sg_set_buf(&sg, ((u8 *)tp) + header_len, head_data_len);
333 ahash_request_set_crypt(req, &sg, NULL, head_data_len);
334 if (crypto_ahash_update(req))
337 for (i = 0; i < shi->nr_frags; ++i) {
338 const skb_frag_t *f = &shi->frags[i];
339 unsigned int offset = skb_frag_off(f);
342 page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
343 sg_set_page(&sg, page, skb_frag_size(f), offset_in_page(offset));
344 ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f));
345 if (crypto_ahash_update(req))
349 skb_walk_frags(skb, frag_iter)
350 if (tcp_sigpool_hash_skb_data(hp, frag_iter, 0))
355 EXPORT_SYMBOL(tcp_sigpool_hash_skb_data);
357 MODULE_LICENSE("GPL");
358 MODULE_DESCRIPTION("Per-CPU pool of crypto requests");