arm64: dts: qcom: sm8550: add TRNG node
[linux-modified.git] / fs / cachefiles / io.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* kiocb-using read/write
3  *
4  * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5  * Written by David Howells (dhowells@redhat.com)
6  */
7
8 #include <linux/mount.h>
9 #include <linux/slab.h>
10 #include <linux/file.h>
11 #include <linux/uio.h>
12 #include <linux/falloc.h>
13 #include <linux/sched/mm.h>
14 #include <trace/events/fscache.h>
15 #include "internal.h"
16
17 struct cachefiles_kiocb {
18         struct kiocb            iocb;
19         refcount_t              ki_refcnt;
20         loff_t                  start;
21         union {
22                 size_t          skipped;
23                 size_t          len;
24         };
25         struct cachefiles_object *object;
26         netfs_io_terminated_t   term_func;
27         void                    *term_func_priv;
28         bool                    was_async;
29         unsigned int            inval_counter;  /* Copy of cookie->inval_counter */
30         u64                     b_writing;
31 };
32
33 static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
34 {
35         if (refcount_dec_and_test(&ki->ki_refcnt)) {
36                 cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq);
37                 fput(ki->iocb.ki_filp);
38                 kfree(ki);
39         }
40 }
41
42 /*
43  * Handle completion of a read from the cache.
44  */
45 static void cachefiles_read_complete(struct kiocb *iocb, long ret)
46 {
47         struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
48         struct inode *inode = file_inode(ki->iocb.ki_filp);
49
50         _enter("%ld", ret);
51
52         if (ret < 0)
53                 trace_cachefiles_io_error(ki->object, inode, ret,
54                                           cachefiles_trace_read_error);
55
56         if (ki->term_func) {
57                 if (ret >= 0) {
58                         if (ki->object->cookie->inval_counter == ki->inval_counter)
59                                 ki->skipped += ret;
60                         else
61                                 ret = -ESTALE;
62                 }
63
64                 ki->term_func(ki->term_func_priv, ret, ki->was_async);
65         }
66
67         cachefiles_put_kiocb(ki);
68 }
69
70 /*
71  * Initiate a read from the cache.
72  */
73 static int cachefiles_read(struct netfs_cache_resources *cres,
74                            loff_t start_pos,
75                            struct iov_iter *iter,
76                            enum netfs_read_from_hole read_hole,
77                            netfs_io_terminated_t term_func,
78                            void *term_func_priv)
79 {
80         struct cachefiles_object *object;
81         struct cachefiles_kiocb *ki;
82         struct file *file;
83         unsigned int old_nofs;
84         ssize_t ret = -ENOBUFS;
85         size_t len = iov_iter_count(iter), skipped = 0;
86
87         if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
88                 goto presubmission_error;
89
90         fscache_count_read();
91         object = cachefiles_cres_object(cres);
92         file = cachefiles_cres_file(cres);
93
94         _enter("%pD,%li,%llx,%zx/%llx",
95                file, file_inode(file)->i_ino, start_pos, len,
96                i_size_read(file_inode(file)));
97
98         /* If the caller asked us to seek for data before doing the read, then
99          * we should do that now.  If we find a gap, we fill it with zeros.
100          */
101         if (read_hole != NETFS_READ_HOLE_IGNORE) {
102                 loff_t off = start_pos, off2;
103
104                 off2 = cachefiles_inject_read_error();
105                 if (off2 == 0)
106                         off2 = vfs_llseek(file, off, SEEK_DATA);
107                 if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) {
108                         skipped = 0;
109                         ret = off2;
110                         goto presubmission_error;
111                 }
112
113                 if (off2 == -ENXIO || off2 >= start_pos + len) {
114                         /* The region is beyond the EOF or there's no more data
115                          * in the region, so clear the rest of the buffer and
116                          * return success.
117                          */
118                         ret = -ENODATA;
119                         if (read_hole == NETFS_READ_HOLE_FAIL)
120                                 goto presubmission_error;
121
122                         iov_iter_zero(len, iter);
123                         skipped = len;
124                         ret = 0;
125                         goto presubmission_error;
126                 }
127
128                 skipped = off2 - off;
129                 iov_iter_zero(skipped, iter);
130         }
131
132         ret = -ENOMEM;
133         ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
134         if (!ki)
135                 goto presubmission_error;
136
137         refcount_set(&ki->ki_refcnt, 2);
138         ki->iocb.ki_filp        = file;
139         ki->iocb.ki_pos         = start_pos + skipped;
140         ki->iocb.ki_flags       = IOCB_DIRECT;
141         ki->iocb.ki_ioprio      = get_current_ioprio();
142         ki->skipped             = skipped;
143         ki->object              = object;
144         ki->inval_counter       = cres->inval_counter;
145         ki->term_func           = term_func;
146         ki->term_func_priv      = term_func_priv;
147         ki->was_async           = true;
148
149         if (ki->term_func)
150                 ki->iocb.ki_complete = cachefiles_read_complete;
151
152         get_file(ki->iocb.ki_filp);
153         cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
154
155         trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped);
156         old_nofs = memalloc_nofs_save();
157         ret = cachefiles_inject_read_error();
158         if (ret == 0)
159                 ret = vfs_iocb_iter_read(file, &ki->iocb, iter);
160         memalloc_nofs_restore(old_nofs);
161         switch (ret) {
162         case -EIOCBQUEUED:
163                 goto in_progress;
164
165         case -ERESTARTSYS:
166         case -ERESTARTNOINTR:
167         case -ERESTARTNOHAND:
168         case -ERESTART_RESTARTBLOCK:
169                 /* There's no easy way to restart the syscall since other AIO's
170                  * may be already running. Just fail this IO with EINTR.
171                  */
172                 ret = -EINTR;
173                 fallthrough;
174         default:
175                 ki->was_async = false;
176                 cachefiles_read_complete(&ki->iocb, ret);
177                 if (ret > 0)
178                         ret = 0;
179                 break;
180         }
181
182 in_progress:
183         cachefiles_put_kiocb(ki);
184         _leave(" = %zd", ret);
185         return ret;
186
187 presubmission_error:
188         if (term_func)
189                 term_func(term_func_priv, ret < 0 ? ret : skipped, false);
190         return ret;
191 }
192
193 /*
194  * Query the occupancy of the cache in a region, returning where the next chunk
195  * of data starts and how long it is.
196  */
197 static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
198                                       loff_t start, size_t len, size_t granularity,
199                                       loff_t *_data_start, size_t *_data_len)
200 {
201         struct cachefiles_object *object;
202         struct file *file;
203         loff_t off, off2;
204
205         *_data_start = -1;
206         *_data_len = 0;
207
208         if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
209                 return -ENOBUFS;
210
211         object = cachefiles_cres_object(cres);
212         file = cachefiles_cres_file(cres);
213         granularity = max_t(size_t, object->volume->cache->bsize, granularity);
214
215         _enter("%pD,%li,%llx,%zx/%llx",
216                file, file_inode(file)->i_ino, start, len,
217                i_size_read(file_inode(file)));
218
219         off = cachefiles_inject_read_error();
220         if (off == 0)
221                 off = vfs_llseek(file, start, SEEK_DATA);
222         if (off == -ENXIO)
223                 return -ENODATA; /* Beyond EOF */
224         if (off < 0 && off >= (loff_t)-MAX_ERRNO)
225                 return -ENOBUFS; /* Error. */
226         if (round_up(off, granularity) >= start + len)
227                 return -ENODATA; /* No data in range */
228
229         off2 = cachefiles_inject_read_error();
230         if (off2 == 0)
231                 off2 = vfs_llseek(file, off, SEEK_HOLE);
232         if (off2 == -ENXIO)
233                 return -ENODATA; /* Beyond EOF */
234         if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO)
235                 return -ENOBUFS; /* Error. */
236
237         /* Round away partial blocks */
238         off = round_up(off, granularity);
239         off2 = round_down(off2, granularity);
240         if (off2 <= off)
241                 return -ENODATA;
242
243         *_data_start = off;
244         if (off2 > start + len)
245                 *_data_len = len;
246         else
247                 *_data_len = off2 - off;
248         return 0;
249 }
250
251 /*
252  * Handle completion of a write to the cache.
253  */
254 static void cachefiles_write_complete(struct kiocb *iocb, long ret)
255 {
256         struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
257         struct cachefiles_object *object = ki->object;
258         struct inode *inode = file_inode(ki->iocb.ki_filp);
259
260         _enter("%ld", ret);
261
262         kiocb_end_write(iocb);
263
264         if (ret < 0)
265                 trace_cachefiles_io_error(object, inode, ret,
266                                           cachefiles_trace_write_error);
267
268         atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing);
269         set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags);
270         if (ki->term_func)
271                 ki->term_func(ki->term_func_priv, ret, ki->was_async);
272         cachefiles_put_kiocb(ki);
273 }
274
275 /*
276  * Initiate a write to the cache.
277  */
278 int __cachefiles_write(struct cachefiles_object *object,
279                        struct file *file,
280                        loff_t start_pos,
281                        struct iov_iter *iter,
282                        netfs_io_terminated_t term_func,
283                        void *term_func_priv)
284 {
285         struct cachefiles_cache *cache;
286         struct cachefiles_kiocb *ki;
287         unsigned int old_nofs;
288         ssize_t ret;
289         size_t len = iov_iter_count(iter);
290
291         fscache_count_write();
292         cache = object->volume->cache;
293
294         _enter("%pD,%li,%llx,%zx/%llx",
295                file, file_inode(file)->i_ino, start_pos, len,
296                i_size_read(file_inode(file)));
297
298         ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
299         if (!ki) {
300                 if (term_func)
301                         term_func(term_func_priv, -ENOMEM, false);
302                 return -ENOMEM;
303         }
304
305         refcount_set(&ki->ki_refcnt, 2);
306         ki->iocb.ki_filp        = file;
307         ki->iocb.ki_pos         = start_pos;
308         ki->iocb.ki_flags       = IOCB_DIRECT | IOCB_WRITE;
309         ki->iocb.ki_ioprio      = get_current_ioprio();
310         ki->object              = object;
311         ki->start               = start_pos;
312         ki->len                 = len;
313         ki->term_func           = term_func;
314         ki->term_func_priv      = term_func_priv;
315         ki->was_async           = true;
316         ki->b_writing           = (len + (1 << cache->bshift) - 1) >> cache->bshift;
317
318         if (ki->term_func)
319                 ki->iocb.ki_complete = cachefiles_write_complete;
320         atomic_long_add(ki->b_writing, &cache->b_writing);
321
322         kiocb_start_write(&ki->iocb);
323
324         get_file(ki->iocb.ki_filp);
325         cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
326
327         trace_cachefiles_write(object, file_inode(file), ki->iocb.ki_pos, len);
328         old_nofs = memalloc_nofs_save();
329         ret = cachefiles_inject_write_error();
330         if (ret == 0)
331                 ret = vfs_iocb_iter_write(file, &ki->iocb, iter);
332         memalloc_nofs_restore(old_nofs);
333         switch (ret) {
334         case -EIOCBQUEUED:
335                 goto in_progress;
336
337         case -ERESTARTSYS:
338         case -ERESTARTNOINTR:
339         case -ERESTARTNOHAND:
340         case -ERESTART_RESTARTBLOCK:
341                 /* There's no easy way to restart the syscall since other AIO's
342                  * may be already running. Just fail this IO with EINTR.
343                  */
344                 ret = -EINTR;
345                 fallthrough;
346         default:
347                 ki->was_async = false;
348                 cachefiles_write_complete(&ki->iocb, ret);
349                 if (ret > 0)
350                         ret = 0;
351                 break;
352         }
353
354 in_progress:
355         cachefiles_put_kiocb(ki);
356         _leave(" = %zd", ret);
357         return ret;
358 }
359
360 static int cachefiles_write(struct netfs_cache_resources *cres,
361                             loff_t start_pos,
362                             struct iov_iter *iter,
363                             netfs_io_terminated_t term_func,
364                             void *term_func_priv)
365 {
366         if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) {
367                 if (term_func)
368                         term_func(term_func_priv, -ENOBUFS, false);
369                 return -ENOBUFS;
370         }
371
372         return __cachefiles_write(cachefiles_cres_object(cres),
373                                   cachefiles_cres_file(cres),
374                                   start_pos, iter,
375                                   term_func, term_func_priv);
376 }
377
378 static inline enum netfs_io_source
379 cachefiles_do_prepare_read(struct netfs_cache_resources *cres,
380                            loff_t start, size_t *_len, loff_t i_size,
381                            unsigned long *_flags, ino_t netfs_ino)
382 {
383         enum cachefiles_prepare_read_trace why;
384         struct cachefiles_object *object = NULL;
385         struct cachefiles_cache *cache;
386         struct fscache_cookie *cookie = fscache_cres_cookie(cres);
387         const struct cred *saved_cred;
388         struct file *file = cachefiles_cres_file(cres);
389         enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER;
390         size_t len = *_len;
391         loff_t off, to;
392         ino_t ino = file ? file_inode(file)->i_ino : 0;
393         int rc;
394
395         _enter("%zx @%llx/%llx", len, start, i_size);
396
397         if (start >= i_size) {
398                 ret = NETFS_FILL_WITH_ZEROES;
399                 why = cachefiles_trace_read_after_eof;
400                 goto out_no_object;
401         }
402
403         if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) {
404                 __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
405                 why = cachefiles_trace_read_no_data;
406                 if (!test_bit(NETFS_SREQ_ONDEMAND, _flags))
407                         goto out_no_object;
408         }
409
410         /* The object and the file may be being created in the background. */
411         if (!file) {
412                 why = cachefiles_trace_read_no_file;
413                 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
414                         goto out_no_object;
415                 file = cachefiles_cres_file(cres);
416                 if (!file)
417                         goto out_no_object;
418                 ino = file_inode(file)->i_ino;
419         }
420
421         object = cachefiles_cres_object(cres);
422         cache = object->volume->cache;
423         cachefiles_begin_secure(cache, &saved_cred);
424 retry:
425         off = cachefiles_inject_read_error();
426         if (off == 0)
427                 off = vfs_llseek(file, start, SEEK_DATA);
428         if (off < 0 && off >= (loff_t)-MAX_ERRNO) {
429                 if (off == (loff_t)-ENXIO) {
430                         why = cachefiles_trace_read_seek_nxio;
431                         goto download_and_store;
432                 }
433                 trace_cachefiles_io_error(object, file_inode(file), off,
434                                           cachefiles_trace_seek_error);
435                 why = cachefiles_trace_read_seek_error;
436                 goto out;
437         }
438
439         if (off >= start + len) {
440                 why = cachefiles_trace_read_found_hole;
441                 goto download_and_store;
442         }
443
444         if (off > start) {
445                 off = round_up(off, cache->bsize);
446                 len = off - start;
447                 *_len = len;
448                 why = cachefiles_trace_read_found_part;
449                 goto download_and_store;
450         }
451
452         to = cachefiles_inject_read_error();
453         if (to == 0)
454                 to = vfs_llseek(file, start, SEEK_HOLE);
455         if (to < 0 && to >= (loff_t)-MAX_ERRNO) {
456                 trace_cachefiles_io_error(object, file_inode(file), to,
457                                           cachefiles_trace_seek_error);
458                 why = cachefiles_trace_read_seek_error;
459                 goto out;
460         }
461
462         if (to < start + len) {
463                 if (start + len >= i_size)
464                         to = round_up(to, cache->bsize);
465                 else
466                         to = round_down(to, cache->bsize);
467                 len = to - start;
468                 *_len = len;
469         }
470
471         why = cachefiles_trace_read_have_data;
472         ret = NETFS_READ_FROM_CACHE;
473         goto out;
474
475 download_and_store:
476         __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
477         if (test_bit(NETFS_SREQ_ONDEMAND, _flags)) {
478                 rc = cachefiles_ondemand_read(object, start, len);
479                 if (!rc) {
480                         __clear_bit(NETFS_SREQ_ONDEMAND, _flags);
481                         goto retry;
482                 }
483                 ret = NETFS_INVALID_READ;
484         }
485 out:
486         cachefiles_end_secure(cache, saved_cred);
487 out_no_object:
488         trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino);
489         return ret;
490 }
491
492 /*
493  * Prepare a read operation, shortening it to a cached/uncached
494  * boundary as appropriate.
495  */
496 static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq,
497                                                     loff_t i_size)
498 {
499         return cachefiles_do_prepare_read(&subreq->rreq->cache_resources,
500                                           subreq->start, &subreq->len, i_size,
501                                           &subreq->flags, subreq->rreq->inode->i_ino);
502 }
503
504 /*
505  * Prepare an on-demand read operation, shortening it to a cached/uncached
506  * boundary as appropriate.
507  */
508 static enum netfs_io_source
509 cachefiles_prepare_ondemand_read(struct netfs_cache_resources *cres,
510                                  loff_t start, size_t *_len, loff_t i_size,
511                                  unsigned long *_flags, ino_t ino)
512 {
513         return cachefiles_do_prepare_read(cres, start, _len, i_size, _flags, ino);
514 }
515
516 /*
517  * Prepare for a write to occur.
518  */
519 int __cachefiles_prepare_write(struct cachefiles_object *object,
520                                struct file *file,
521                                loff_t *_start, size_t *_len,
522                                bool no_space_allocated_yet)
523 {
524         struct cachefiles_cache *cache = object->volume->cache;
525         loff_t start = *_start, pos;
526         size_t len = *_len, down;
527         int ret;
528
529         /* Round to DIO size */
530         down = start - round_down(start, PAGE_SIZE);
531         *_start = start - down;
532         *_len = round_up(down + len, PAGE_SIZE);
533
534         /* We need to work out whether there's sufficient disk space to perform
535          * the write - but we can skip that check if we have space already
536          * allocated.
537          */
538         if (no_space_allocated_yet)
539                 goto check_space;
540
541         pos = cachefiles_inject_read_error();
542         if (pos == 0)
543                 pos = vfs_llseek(file, *_start, SEEK_DATA);
544         if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
545                 if (pos == -ENXIO)
546                         goto check_space; /* Unallocated tail */
547                 trace_cachefiles_io_error(object, file_inode(file), pos,
548                                           cachefiles_trace_seek_error);
549                 return pos;
550         }
551         if ((u64)pos >= (u64)*_start + *_len)
552                 goto check_space; /* Unallocated region */
553
554         /* We have a block that's at least partially filled - if we're low on
555          * space, we need to see if it's fully allocated.  If it's not, we may
556          * want to cull it.
557          */
558         if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
559                                  cachefiles_has_space_check) == 0)
560                 return 0; /* Enough space to simply overwrite the whole block */
561
562         pos = cachefiles_inject_read_error();
563         if (pos == 0)
564                 pos = vfs_llseek(file, *_start, SEEK_HOLE);
565         if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
566                 trace_cachefiles_io_error(object, file_inode(file), pos,
567                                           cachefiles_trace_seek_error);
568                 return pos;
569         }
570         if ((u64)pos >= (u64)*_start + *_len)
571                 return 0; /* Fully allocated */
572
573         /* Partially allocated, but insufficient space: cull. */
574         fscache_count_no_write_space();
575         ret = cachefiles_inject_remove_error();
576         if (ret == 0)
577                 ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
578                                     *_start, *_len);
579         if (ret < 0) {
580                 trace_cachefiles_io_error(object, file_inode(file), ret,
581                                           cachefiles_trace_fallocate_error);
582                 cachefiles_io_error_obj(object,
583                                         "CacheFiles: fallocate failed (%d)\n", ret);
584                 ret = -EIO;
585         }
586
587         return ret;
588
589 check_space:
590         return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
591                                     cachefiles_has_space_for_write);
592 }
593
594 static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
595                                     loff_t *_start, size_t *_len, loff_t i_size,
596                                     bool no_space_allocated_yet)
597 {
598         struct cachefiles_object *object = cachefiles_cres_object(cres);
599         struct cachefiles_cache *cache = object->volume->cache;
600         const struct cred *saved_cred;
601         int ret;
602
603         if (!cachefiles_cres_file(cres)) {
604                 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
605                         return -ENOBUFS;
606                 if (!cachefiles_cres_file(cres))
607                         return -ENOBUFS;
608         }
609
610         cachefiles_begin_secure(cache, &saved_cred);
611         ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
612                                          _start, _len,
613                                          no_space_allocated_yet);
614         cachefiles_end_secure(cache, saved_cred);
615         return ret;
616 }
617
618 /*
619  * Clean up an operation.
620  */
621 static void cachefiles_end_operation(struct netfs_cache_resources *cres)
622 {
623         struct file *file = cachefiles_cres_file(cres);
624
625         if (file)
626                 fput(file);
627         fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end);
628 }
629
630 static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
631         .end_operation          = cachefiles_end_operation,
632         .read                   = cachefiles_read,
633         .write                  = cachefiles_write,
634         .prepare_read           = cachefiles_prepare_read,
635         .prepare_write          = cachefiles_prepare_write,
636         .prepare_ondemand_read  = cachefiles_prepare_ondemand_read,
637         .query_occupancy        = cachefiles_query_occupancy,
638 };
639
640 /*
641  * Open the cache file when beginning a cache operation.
642  */
643 bool cachefiles_begin_operation(struct netfs_cache_resources *cres,
644                                 enum fscache_want_state want_state)
645 {
646         struct cachefiles_object *object = cachefiles_cres_object(cres);
647
648         if (!cachefiles_cres_file(cres)) {
649                 cres->ops = &cachefiles_netfs_cache_ops;
650                 if (object->file) {
651                         spin_lock(&object->lock);
652                         if (!cres->cache_priv2 && object->file)
653                                 cres->cache_priv2 = get_file(object->file);
654                         spin_unlock(&object->lock);
655                 }
656         }
657
658         if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) {
659                 pr_err("failed to get cres->file\n");
660                 return false;
661         }
662
663         return true;
664 }