GNU Linux-libre 5.10.219-gnu1
[releases.git] / fs / nfs / read.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * linux/fs/nfs/read.c
4  *
5  * Block I/O for NFS
6  *
7  * Partial copy of Linus' read cache modifications to fs/nfs/file.c
8  * modified for async RPC by okir@monad.swb.de
9  */
10
11 #include <linux/time.h>
12 #include <linux/kernel.h>
13 #include <linux/errno.h>
14 #include <linux/fcntl.h>
15 #include <linux/stat.h>
16 #include <linux/mm.h>
17 #include <linux/slab.h>
18 #include <linux/pagemap.h>
19 #include <linux/sunrpc/clnt.h>
20 #include <linux/nfs_fs.h>
21 #include <linux/nfs_page.h>
22 #include <linux/module.h>
23
24 #include "nfs4_fs.h"
25 #include "internal.h"
26 #include "iostat.h"
27 #include "fscache.h"
28 #include "pnfs.h"
29 #include "nfstrace.h"
30
31 #define NFSDBG_FACILITY         NFSDBG_PAGECACHE
32
33 static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops;
34 static const struct nfs_rw_ops nfs_rw_read_ops;
35
36 static struct kmem_cache *nfs_rdata_cachep;
37
38 static struct nfs_pgio_header *nfs_readhdr_alloc(void)
39 {
40         struct nfs_pgio_header *p = kmem_cache_zalloc(nfs_rdata_cachep, GFP_KERNEL);
41
42         if (p)
43                 p->rw_mode = FMODE_READ;
44         return p;
45 }
46
47 static void nfs_readhdr_free(struct nfs_pgio_header *rhdr)
48 {
49         kmem_cache_free(nfs_rdata_cachep, rhdr);
50 }
51
52 static
53 int nfs_return_empty_page(struct page *page)
54 {
55         zero_user(page, 0, PAGE_SIZE);
56         SetPageUptodate(page);
57         unlock_page(page);
58         return 0;
59 }
60
61 void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio,
62                               struct inode *inode, bool force_mds,
63                               const struct nfs_pgio_completion_ops *compl_ops)
64 {
65         struct nfs_server *server = NFS_SERVER(inode);
66         const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
67
68 #ifdef CONFIG_NFS_V4_1
69         if (server->pnfs_curr_ld && !force_mds)
70                 pg_ops = server->pnfs_curr_ld->pg_read_ops;
71 #endif
72         nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_read_ops,
73                         server->rsize, 0);
74 }
75 EXPORT_SYMBOL_GPL(nfs_pageio_init_read);
76
77 void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
78 {
79         struct nfs_pgio_mirror *mirror;
80
81         if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
82                 pgio->pg_ops->pg_cleanup(pgio);
83
84         pgio->pg_ops = &nfs_pgio_rw_ops;
85
86         /* read path should never have more than one mirror */
87         WARN_ON_ONCE(pgio->pg_mirror_count != 1);
88
89         mirror = &pgio->pg_mirrors[0];
90         mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->rsize;
91 }
92 EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds);
93
94 static void nfs_readpage_release(struct nfs_page *req, int error)
95 {
96         struct inode *inode = d_inode(nfs_req_openctx(req)->dentry);
97         struct page *page = req->wb_page;
98
99         dprintk("NFS: read done (%s/%llu %d@%lld)\n", inode->i_sb->s_id,
100                 (unsigned long long)NFS_FILEID(inode), req->wb_bytes,
101                 (long long)req_offset(req));
102
103         if (nfs_error_is_fatal_on_server(error) && error != -ETIMEDOUT)
104                 SetPageError(page);
105         if (nfs_page_group_sync_on_bit(req, PG_UNLOCKPAGE)) {
106                 struct address_space *mapping = page_file_mapping(page);
107
108                 if (PageUptodate(page))
109                         nfs_readpage_to_fscache(inode, page, 0);
110                 else if (!PageError(page) && !PagePrivate(page))
111                         generic_error_remove_page(mapping, page);
112                 unlock_page(page);
113         }
114         nfs_release_request(req);
115 }
116
117 int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode,
118                        struct page *page)
119 {
120         struct nfs_page *new;
121         unsigned int len;
122         struct nfs_pageio_descriptor pgio;
123         struct nfs_pgio_mirror *pgm;
124
125         len = nfs_page_length(page);
126         if (len == 0)
127                 return nfs_return_empty_page(page);
128         new = nfs_create_request(ctx, page, 0, len);
129         if (IS_ERR(new)) {
130                 unlock_page(page);
131                 return PTR_ERR(new);
132         }
133         if (len < PAGE_SIZE)
134                 zero_user_segment(page, len, PAGE_SIZE);
135
136         nfs_pageio_init_read(&pgio, inode, false,
137                              &nfs_async_read_completion_ops);
138         if (!nfs_pageio_add_request(&pgio, new)) {
139                 nfs_list_remove_request(new);
140                 nfs_readpage_release(new, pgio.pg_error);
141         }
142         nfs_pageio_complete(&pgio);
143
144         /* It doesn't make sense to do mirrored reads! */
145         WARN_ON_ONCE(pgio.pg_mirror_count != 1);
146
147         pgm = &pgio.pg_mirrors[0];
148         NFS_I(inode)->read_io += pgm->pg_bytes_written;
149
150         return pgio.pg_error < 0 ? pgio.pg_error : 0;
151 }
152
153 static void nfs_page_group_set_uptodate(struct nfs_page *req)
154 {
155         if (nfs_page_group_sync_on_bit(req, PG_UPTODATE))
156                 SetPageUptodate(req->wb_page);
157 }
158
159 static void nfs_read_completion(struct nfs_pgio_header *hdr)
160 {
161         unsigned long bytes = 0;
162         int error;
163
164         if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
165                 goto out;
166         while (!list_empty(&hdr->pages)) {
167                 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
168                 struct page *page = req->wb_page;
169                 unsigned long start = req->wb_pgbase;
170                 unsigned long end = req->wb_pgbase + req->wb_bytes;
171
172                 if (test_bit(NFS_IOHDR_EOF, &hdr->flags)) {
173                         /* note: regions of the page not covered by a
174                          * request are zeroed in nfs_readpage_async /
175                          * readpage_async_filler */
176                         if (bytes > hdr->good_bytes) {
177                                 /* nothing in this request was good, so zero
178                                  * the full extent of the request */
179                                 zero_user_segment(page, start, end);
180
181                         } else if (hdr->good_bytes - bytes < req->wb_bytes) {
182                                 /* part of this request has good bytes, but
183                                  * not all. zero the bad bytes */
184                                 start += hdr->good_bytes - bytes;
185                                 WARN_ON(start < req->wb_pgbase);
186                                 zero_user_segment(page, start, end);
187                         }
188                 }
189                 error = 0;
190                 bytes += req->wb_bytes;
191                 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
192                         if (bytes <= hdr->good_bytes)
193                                 nfs_page_group_set_uptodate(req);
194                         else {
195                                 error = hdr->error;
196                                 xchg(&nfs_req_openctx(req)->error, error);
197                         }
198                 } else
199                         nfs_page_group_set_uptodate(req);
200                 nfs_list_remove_request(req);
201                 nfs_readpage_release(req, error);
202         }
203 out:
204         hdr->release(hdr);
205 }
206
207 static void nfs_initiate_read(struct nfs_pgio_header *hdr,
208                               struct rpc_message *msg,
209                               const struct nfs_rpc_ops *rpc_ops,
210                               struct rpc_task_setup *task_setup_data, int how)
211 {
212         struct inode *inode = hdr->inode;
213         int swap_flags = IS_SWAPFILE(inode) ? NFS_RPC_SWAPFLAGS : 0;
214
215         task_setup_data->flags |= swap_flags;
216         rpc_ops->read_setup(hdr, msg);
217         trace_nfs_initiate_read(hdr);
218 }
219
220 static void
221 nfs_async_read_error(struct list_head *head, int error)
222 {
223         struct nfs_page *req;
224
225         while (!list_empty(head)) {
226                 req = nfs_list_entry(head->next);
227                 nfs_list_remove_request(req);
228                 nfs_readpage_release(req, error);
229         }
230 }
231
232 static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops = {
233         .error_cleanup = nfs_async_read_error,
234         .completion = nfs_read_completion,
235 };
236
237 /*
238  * This is the callback from RPC telling us whether a reply was
239  * received or some error occurred (timeout or socket shutdown).
240  */
241 static int nfs_readpage_done(struct rpc_task *task,
242                              struct nfs_pgio_header *hdr,
243                              struct inode *inode)
244 {
245         int status = NFS_PROTO(inode)->read_done(task, hdr);
246         if (status != 0)
247                 return status;
248
249         nfs_add_stats(inode, NFSIOS_SERVERREADBYTES, hdr->res.count);
250         trace_nfs_readpage_done(task, hdr);
251
252         if (task->tk_status == -ESTALE) {
253                 nfs_set_inode_stale(inode);
254                 nfs_mark_for_revalidate(inode);
255         }
256         return 0;
257 }
258
259 static void nfs_readpage_retry(struct rpc_task *task,
260                                struct nfs_pgio_header *hdr)
261 {
262         struct nfs_pgio_args *argp = &hdr->args;
263         struct nfs_pgio_res  *resp = &hdr->res;
264
265         /* This is a short read! */
266         nfs_inc_stats(hdr->inode, NFSIOS_SHORTREAD);
267         trace_nfs_readpage_short(task, hdr);
268
269         /* Has the server at least made some progress? */
270         if (resp->count == 0) {
271                 nfs_set_pgio_error(hdr, -EIO, argp->offset);
272                 return;
273         }
274
275         /* For non rpc-based layout drivers, retry-through-MDS */
276         if (!task->tk_ops) {
277                 hdr->pnfs_error = -EAGAIN;
278                 return;
279         }
280
281         /* Yes, so retry the read at the end of the hdr */
282         hdr->mds_offset += resp->count;
283         argp->offset += resp->count;
284         argp->pgbase += resp->count;
285         argp->count -= resp->count;
286         resp->count = 0;
287         resp->eof = 0;
288         rpc_restart_call_prepare(task);
289 }
290
291 static void nfs_readpage_result(struct rpc_task *task,
292                                 struct nfs_pgio_header *hdr)
293 {
294         if (hdr->res.eof) {
295                 loff_t pos = hdr->args.offset + hdr->res.count;
296                 unsigned int new = pos - hdr->io_start;
297
298                 if (hdr->good_bytes > new) {
299                         hdr->good_bytes = new;
300                         set_bit(NFS_IOHDR_EOF, &hdr->flags);
301                         clear_bit(NFS_IOHDR_ERROR, &hdr->flags);
302                 }
303         } else if (hdr->res.count < hdr->args.count)
304                 nfs_readpage_retry(task, hdr);
305 }
306
307 /*
308  * Read a page over NFS.
309  * We read the page synchronously in the following case:
310  *  -   The error flag is set for this page. This happens only when a
311  *      previous async read operation failed.
312  */
313 int nfs_readpage(struct file *file, struct page *page)
314 {
315         struct nfs_open_context *ctx;
316         struct inode *inode = page_file_mapping(page)->host;
317         int             error;
318
319         dprintk("NFS: nfs_readpage (%p %ld@%lu)\n",
320                 page, PAGE_SIZE, page_index(page));
321         nfs_inc_stats(inode, NFSIOS_VFSREADPAGE);
322         nfs_add_stats(inode, NFSIOS_READPAGES, 1);
323
324         /*
325          * Try to flush any pending writes to the file..
326          *
327          * NOTE! Because we own the page lock, there cannot
328          * be any new pending writes generated at this point
329          * for this page (other pages can be written to).
330          */
331         error = nfs_wb_page(inode, page);
332         if (error)
333                 goto out_unlock;
334         if (PageUptodate(page))
335                 goto out_unlock;
336
337         error = -ESTALE;
338         if (NFS_STALE(inode))
339                 goto out_unlock;
340
341         if (file == NULL) {
342                 error = -EBADF;
343                 ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
344                 if (ctx == NULL)
345                         goto out_unlock;
346         } else
347                 ctx = get_nfs_open_context(nfs_file_open_context(file));
348
349         if (!IS_SYNC(inode)) {
350                 error = nfs_readpage_from_fscache(ctx, inode, page);
351                 if (error == 0)
352                         goto out;
353         }
354
355         xchg(&ctx->error, 0);
356         error = nfs_readpage_async(ctx, inode, page);
357         if (!error) {
358                 error = wait_on_page_locked_killable(page);
359                 if (!PageUptodate(page) && !error)
360                         error = xchg(&ctx->error, 0);
361         }
362 out:
363         put_nfs_open_context(ctx);
364         return error;
365 out_unlock:
366         unlock_page(page);
367         return error;
368 }
369
370 struct nfs_readdesc {
371         struct nfs_pageio_descriptor *pgio;
372         struct nfs_open_context *ctx;
373 };
374
375 static int
376 readpage_async_filler(void *data, struct page *page)
377 {
378         struct nfs_readdesc *desc = (struct nfs_readdesc *)data;
379         struct nfs_page *new;
380         unsigned int len;
381         int error;
382
383         len = nfs_page_length(page);
384         if (len == 0)
385                 return nfs_return_empty_page(page);
386
387         new = nfs_create_request(desc->ctx, page, 0, len);
388         if (IS_ERR(new))
389                 goto out_error;
390
391         if (len < PAGE_SIZE)
392                 zero_user_segment(page, len, PAGE_SIZE);
393         if (!nfs_pageio_add_request(desc->pgio, new)) {
394                 nfs_list_remove_request(new);
395                 error = desc->pgio->pg_error;
396                 nfs_readpage_release(new, error);
397                 goto out;
398         }
399         return 0;
400 out_error:
401         error = PTR_ERR(new);
402         unlock_page(page);
403 out:
404         return error;
405 }
406
407 int nfs_readpages(struct file *filp, struct address_space *mapping,
408                 struct list_head *pages, unsigned nr_pages)
409 {
410         struct nfs_pageio_descriptor pgio;
411         struct nfs_pgio_mirror *pgm;
412         struct nfs_readdesc desc = {
413                 .pgio = &pgio,
414         };
415         struct inode *inode = mapping->host;
416         unsigned long npages;
417         int ret = -ESTALE;
418
419         dprintk("NFS: nfs_readpages (%s/%Lu %d)\n",
420                         inode->i_sb->s_id,
421                         (unsigned long long)NFS_FILEID(inode),
422                         nr_pages);
423         nfs_inc_stats(inode, NFSIOS_VFSREADPAGES);
424
425         if (NFS_STALE(inode))
426                 goto out;
427
428         if (filp == NULL) {
429                 desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
430                 if (desc.ctx == NULL)
431                         return -EBADF;
432         } else
433                 desc.ctx = get_nfs_open_context(nfs_file_open_context(filp));
434
435         /* attempt to read as many of the pages as possible from the cache
436          * - this returns -ENOBUFS immediately if the cookie is negative
437          */
438         ret = nfs_readpages_from_fscache(desc.ctx, inode, mapping,
439                                          pages, &nr_pages);
440         if (ret == 0)
441                 goto read_complete; /* all pages were read */
442
443         nfs_pageio_init_read(&pgio, inode, false,
444                              &nfs_async_read_completion_ops);
445
446         ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc);
447         nfs_pageio_complete(&pgio);
448
449         /* It doesn't make sense to do mirrored reads! */
450         WARN_ON_ONCE(pgio.pg_mirror_count != 1);
451
452         pgm = &pgio.pg_mirrors[0];
453         NFS_I(inode)->read_io += pgm->pg_bytes_written;
454         npages = (pgm->pg_bytes_written + PAGE_SIZE - 1) >>
455                  PAGE_SHIFT;
456         nfs_add_stats(inode, NFSIOS_READPAGES, npages);
457 read_complete:
458         put_nfs_open_context(desc.ctx);
459 out:
460         return ret;
461 }
462
463 int __init nfs_init_readpagecache(void)
464 {
465         nfs_rdata_cachep = kmem_cache_create("nfs_read_data",
466                                              sizeof(struct nfs_pgio_header),
467                                              0, SLAB_HWCACHE_ALIGN,
468                                              NULL);
469         if (nfs_rdata_cachep == NULL)
470                 return -ENOMEM;
471
472         return 0;
473 }
474
475 void nfs_destroy_readpagecache(void)
476 {
477         kmem_cache_destroy(nfs_rdata_cachep);
478 }
479
480 static const struct nfs_rw_ops nfs_rw_read_ops = {
481         .rw_alloc_header        = nfs_readhdr_alloc,
482         .rw_free_header         = nfs_readhdr_free,
483         .rw_done                = nfs_readpage_done,
484         .rw_result              = nfs_readpage_result,
485         .rw_initiate            = nfs_initiate_read,
486 };