1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/fs/nfs/direct.c
5 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
7 * High-performance uncached I/O for the Linux NFS client
9 * There are important applications whose performance or correctness
10 * depends on uncached access to file data. Database clusters
11 * (multiple copies of the same instance running on separate hosts)
12 * implement their own cache coherency protocol that subsumes file
13 * system cache protocols. Applications that process datasets
14 * considerably larger than the client's memory do not always benefit
15 * from a local cache. A streaming video server, for instance, has no
16 * need to cache the contents of a file.
18 * When an application requests uncached I/O, all read and write requests
19 * are made directly to the server; data stored or fetched via these
20 * requests is not cached in the Linux page cache. The client does not
21 * correct unaligned requests from applications. All requested bytes are
22 * held on permanent storage before a direct write system call returns to
25 * Solaris implements an uncached I/O facility called directio() that
26 * is used for backups and sequential I/O to very large files. Solaris
27 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
28 * an undocumented mount option.
30 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
31 * help from Andrew Morton.
33 * 18 Dec 2001 Initial implementation for 2.4 --cel
34 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
35 * 08 Jun 2003 Port to 2.5 APIs --cel
36 * 31 Mar 2004 Handle direct I/O without VFS support --cel
37 * 15 Sep 2004 Parallel async reads --cel
38 * 04 May 2005 support O_DIRECT with aio --cel
42 #include <linux/errno.h>
43 #include <linux/sched.h>
44 #include <linux/kernel.h>
45 #include <linux/file.h>
46 #include <linux/pagemap.h>
47 #include <linux/kref.h>
48 #include <linux/slab.h>
49 #include <linux/task_io_accounting_ops.h>
50 #include <linux/module.h>
52 #include <linux/nfs_fs.h>
53 #include <linux/nfs_page.h>
54 #include <linux/sunrpc/clnt.h>
56 #include <linux/uaccess.h>
57 #include <linux/atomic.h>
65 #define NFSDBG_FACILITY NFSDBG_VFS
67 static struct kmem_cache *nfs_direct_cachep;
69 static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
70 static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
71 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
72 static void nfs_direct_write_schedule_work(struct work_struct *work);
74 static inline void get_dreq(struct nfs_direct_req *dreq)
76 atomic_inc(&dreq->io_count);
79 static inline int put_dreq(struct nfs_direct_req *dreq)
81 return atomic_dec_and_test(&dreq->io_count);
85 nfs_direct_handle_truncated(struct nfs_direct_req *dreq,
86 const struct nfs_pgio_header *hdr,
89 if (!(test_bit(NFS_IOHDR_ERROR, &hdr->flags) ||
90 test_bit(NFS_IOHDR_EOF, &hdr->flags)))
92 if (dreq->max_count >= dreq_len) {
93 dreq->max_count = dreq_len;
94 if (dreq->count > dreq_len)
95 dreq->count = dreq_len;
98 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && !dreq->error)
99 dreq->error = hdr->error;
103 nfs_direct_count_bytes(struct nfs_direct_req *dreq,
104 const struct nfs_pgio_header *hdr)
106 loff_t hdr_end = hdr->io_start + hdr->good_bytes;
107 ssize_t dreq_len = 0;
109 if (hdr_end > dreq->io_start)
110 dreq_len = hdr_end - dreq->io_start;
112 nfs_direct_handle_truncated(dreq, hdr, dreq_len);
114 if (dreq_len > dreq->max_count)
115 dreq_len = dreq->max_count;
117 if (dreq->count < dreq_len)
118 dreq->count = dreq_len;
121 static void nfs_direct_truncate_request(struct nfs_direct_req *dreq,
122 struct nfs_page *req)
124 loff_t offs = req_offset(req);
125 size_t req_start = (size_t)(offs - dreq->io_start);
127 if (req_start < dreq->max_count)
128 dreq->max_count = req_start;
129 if (req_start < dreq->count)
130 dreq->count = req_start;
134 * nfs_swap_rw - NFS address space operation for swap I/O
135 * @iocb: target I/O control block
138 * Perform IO to the swap-file. This is much like direct IO.
140 int nfs_swap_rw(struct kiocb *iocb, struct iov_iter *iter)
144 VM_BUG_ON(iov_iter_count(iter) != PAGE_SIZE);
146 if (iov_iter_rw(iter) == READ)
147 ret = nfs_file_direct_read(iocb, iter, true);
149 ret = nfs_file_direct_write(iocb, iter, true);
155 static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
158 for (i = 0; i < npages; i++)
162 void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
163 struct nfs_direct_req *dreq)
165 cinfo->inode = dreq->inode;
166 cinfo->mds = &dreq->mds_cinfo;
167 cinfo->ds = &dreq->ds_cinfo;
169 cinfo->completion_ops = &nfs_direct_commit_completion_ops;
172 static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
174 struct nfs_direct_req *dreq;
176 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
180 kref_init(&dreq->kref);
181 kref_get(&dreq->kref);
182 init_completion(&dreq->completion);
183 INIT_LIST_HEAD(&dreq->mds_cinfo.list);
184 pnfs_init_ds_commit_info(&dreq->ds_cinfo);
185 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
186 spin_lock_init(&dreq->lock);
191 static void nfs_direct_req_free(struct kref *kref)
193 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
195 pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode);
196 if (dreq->l_ctx != NULL)
197 nfs_put_lock_context(dreq->l_ctx);
198 if (dreq->ctx != NULL)
199 put_nfs_open_context(dreq->ctx);
200 kmem_cache_free(nfs_direct_cachep, dreq);
203 static void nfs_direct_req_release(struct nfs_direct_req *dreq)
205 kref_put(&dreq->kref, nfs_direct_req_free);
208 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq, loff_t offset)
210 loff_t start = offset - dreq->io_start;
211 return dreq->max_count - start;
213 EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
216 * Collects and returns the final error value/byte-count.
218 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
220 ssize_t result = -EIOCBQUEUED;
222 /* Async requests don't wait here */
226 result = wait_for_completion_killable(&dreq->completion);
229 result = dreq->count;
230 WARN_ON_ONCE(dreq->count < 0);
233 result = dreq->error;
236 return (ssize_t) result;
240 * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
241 * the iocb is still valid here if this is a synchronous request.
243 static void nfs_direct_complete(struct nfs_direct_req *dreq)
245 struct inode *inode = dreq->inode;
247 inode_dio_end(inode);
250 long res = (long) dreq->error;
251 if (dreq->count != 0) {
252 res = (long) dreq->count;
253 WARN_ON_ONCE(dreq->count < 0);
255 dreq->iocb->ki_complete(dreq->iocb, res);
258 complete(&dreq->completion);
260 nfs_direct_req_release(dreq);
263 static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
265 unsigned long bytes = 0;
266 struct nfs_direct_req *dreq = hdr->dreq;
268 spin_lock(&dreq->lock);
269 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) {
270 spin_unlock(&dreq->lock);
274 nfs_direct_count_bytes(dreq, hdr);
275 spin_unlock(&dreq->lock);
277 while (!list_empty(&hdr->pages)) {
278 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
279 struct page *page = req->wb_page;
281 if (!PageCompound(page) && bytes < hdr->good_bytes &&
282 (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY))
283 set_page_dirty(page);
284 bytes += req->wb_bytes;
285 nfs_list_remove_request(req);
286 nfs_release_request(req);
290 nfs_direct_complete(dreq);
294 static void nfs_read_sync_pgio_error(struct list_head *head, int error)
296 struct nfs_page *req;
298 while (!list_empty(head)) {
299 req = nfs_list_entry(head->next);
300 nfs_list_remove_request(req);
301 nfs_release_request(req);
305 static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
310 static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
311 .error_cleanup = nfs_read_sync_pgio_error,
312 .init_hdr = nfs_direct_pgio_init,
313 .completion = nfs_direct_read_completion,
317 * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
318 * operation. If nfs_readdata_alloc() or get_user_pages() fails,
319 * bail and stop sending more reads. Read length accounting is
320 * handled automatically by nfs_direct_read_result(). Otherwise, if
321 * no requests have been sent, just return an error.
324 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
325 struct iov_iter *iter,
328 struct nfs_pageio_descriptor desc;
329 struct inode *inode = dreq->inode;
330 ssize_t result = -EINVAL;
331 size_t requested_bytes = 0;
332 size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
334 nfs_pageio_init_read(&desc, dreq->inode, false,
335 &nfs_direct_read_completion_ops);
338 inode_dio_begin(inode);
340 while (iov_iter_count(iter)) {
341 struct page **pagevec;
346 result = iov_iter_get_pages_alloc2(iter, &pagevec,
352 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
353 for (i = 0; i < npages; i++) {
354 struct nfs_page *req;
355 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
356 /* XXX do we need to do the eof zeroing found in async_filler? */
357 req = nfs_page_create_from_page(dreq->ctx, pagevec[i],
358 pgbase, pos, req_len);
360 result = PTR_ERR(req);
363 if (!nfs_pageio_add_request(&desc, req)) {
364 result = desc.pg_error;
365 nfs_release_request(req);
370 requested_bytes += req_len;
372 dreq->bytes_left -= req_len;
374 nfs_direct_release_pages(pagevec, npages);
380 nfs_pageio_complete(&desc);
383 * If no bytes were started, return the error, and let the
384 * generic layer handle the completion.
386 if (requested_bytes == 0) {
387 inode_dio_end(inode);
388 nfs_direct_req_release(dreq);
389 return result < 0 ? result : -EIO;
393 nfs_direct_complete(dreq);
394 return requested_bytes;
398 * nfs_file_direct_read - file direct read operation for NFS files
399 * @iocb: target I/O control block
400 * @iter: vector of user buffers into which to read data
401 * @swap: flag indicating this is swap IO, not O_DIRECT IO
403 * We use this function for direct reads instead of calling
404 * generic_file_aio_read() in order to avoid gfar's check to see if
405 * the request starts before the end of the file. For that check
406 * to work, we must generate a GETATTR before each direct read, and
407 * even then there is a window between the GETATTR and the subsequent
408 * READ where the file size could change. Our preference is simply
409 * to do all reads the application wants, and the server will take
410 * care of managing the end of file boundary.
412 * This function also eliminates unnecessarily updating the file's
413 * atime locally, as the NFS server sets the file's atime, and this
414 * client must read the updated atime from the server back into its
417 ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter,
420 struct file *file = iocb->ki_filp;
421 struct address_space *mapping = file->f_mapping;
422 struct inode *inode = mapping->host;
423 struct nfs_direct_req *dreq;
424 struct nfs_lock_context *l_ctx;
425 ssize_t result, requested;
426 size_t count = iov_iter_count(iter);
427 nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
429 dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
430 file, count, (long long) iocb->ki_pos);
436 task_io_account_read(count);
439 dreq = nfs_direct_req_alloc();
444 dreq->bytes_left = dreq->max_count = count;
445 dreq->io_start = iocb->ki_pos;
446 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
447 l_ctx = nfs_get_lock_context(dreq->ctx);
449 result = PTR_ERR(l_ctx);
450 nfs_direct_req_release(dreq);
454 if (!is_sync_kiocb(iocb))
457 if (user_backed_iter(iter))
458 dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;
461 nfs_start_io_direct(inode);
463 NFS_I(inode)->read_io += count;
464 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
467 nfs_end_io_direct(inode);
470 result = nfs_direct_wait(dreq);
473 iocb->ki_pos += result;
475 iov_iter_revert(iter, requested);
481 nfs_direct_req_release(dreq);
486 static void nfs_direct_add_page_head(struct list_head *list,
487 struct nfs_page *req)
489 struct nfs_page *head = req->wb_head;
491 if (!list_empty(&head->wb_list) || !nfs_lock_request(head))
493 if (!list_empty(&head->wb_list)) {
494 nfs_unlock_request(head);
497 list_add(&head->wb_list, list);
498 kref_get(&head->wb_kref);
499 kref_get(&head->wb_kref);
502 static void nfs_direct_join_group(struct list_head *list,
503 struct nfs_commit_info *cinfo,
506 struct nfs_page *req, *subreq;
508 list_for_each_entry(req, list, wb_list) {
509 if (req->wb_head != req) {
510 nfs_direct_add_page_head(&req->wb_list, req);
513 subreq = req->wb_this_page;
518 * Remove subrequests from this list before freeing
519 * them in the call to nfs_join_page_group().
521 if (!list_empty(&subreq->wb_list)) {
522 nfs_list_remove_request(subreq);
523 nfs_release_request(subreq);
525 } while ((subreq = subreq->wb_this_page) != req);
526 nfs_join_page_group(req, cinfo, inode);
531 nfs_direct_write_scan_commit_list(struct inode *inode,
532 struct list_head *list,
533 struct nfs_commit_info *cinfo)
535 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
536 pnfs_recover_commit_reqs(list, cinfo);
537 nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
538 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
541 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
543 struct nfs_pageio_descriptor desc;
544 struct nfs_page *req;
546 struct nfs_commit_info cinfo;
548 nfs_init_cinfo_from_dreq(&cinfo, dreq);
549 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
551 nfs_direct_join_group(&reqs, &cinfo, dreq->inode);
553 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);
556 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
557 &nfs_direct_write_completion_ops);
560 while (!list_empty(&reqs)) {
561 req = nfs_list_entry(reqs.next);
562 /* Bump the transmission count */
564 if (!nfs_pageio_add_request(&desc, req)) {
565 spin_lock(&dreq->lock);
566 if (dreq->error < 0) {
567 desc.pg_error = dreq->error;
568 } else if (desc.pg_error != -EAGAIN) {
571 desc.pg_error = -EIO;
572 dreq->error = desc.pg_error;
574 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
575 spin_unlock(&dreq->lock);
578 nfs_release_request(req);
580 nfs_pageio_complete(&desc);
582 while (!list_empty(&reqs)) {
583 req = nfs_list_entry(reqs.next);
584 nfs_list_remove_request(req);
585 nfs_unlock_and_release_request(req);
586 if (desc.pg_error == -EAGAIN) {
587 nfs_mark_request_commit(req, NULL, &cinfo, 0);
589 spin_lock(&dreq->lock);
590 nfs_direct_truncate_request(dreq, req);
591 spin_unlock(&dreq->lock);
592 nfs_release_request(req);
597 nfs_direct_write_complete(dreq);
600 static void nfs_direct_commit_complete(struct nfs_commit_data *data)
602 const struct nfs_writeverf *verf = data->res.verf;
603 struct nfs_direct_req *dreq = data->dreq;
604 struct nfs_commit_info cinfo;
605 struct nfs_page *req;
606 int status = data->task.tk_status;
608 trace_nfs_direct_commit_complete(dreq);
611 /* Errors in commit are fatal */
612 dreq->error = status;
613 dreq->flags = NFS_ODIRECT_DONE;
615 status = dreq->error;
618 nfs_init_cinfo_from_dreq(&cinfo, dreq);
620 while (!list_empty(&data->pages)) {
621 req = nfs_list_entry(data->pages.next);
622 nfs_list_remove_request(req);
624 spin_lock(&dreq->lock);
625 nfs_direct_truncate_request(dreq, req);
626 spin_unlock(&dreq->lock);
627 nfs_release_request(req);
628 } else if (!nfs_write_match_verf(verf, req)) {
629 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
631 * Despite the reboot, the write was successful,
635 nfs_mark_request_commit(req, NULL, &cinfo, 0);
637 nfs_release_request(req);
638 nfs_unlock_and_release_request(req);
641 if (nfs_commit_end(cinfo.mds))
642 nfs_direct_write_complete(dreq);
645 static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
646 struct nfs_page *req)
648 struct nfs_direct_req *dreq = cinfo->dreq;
650 trace_nfs_direct_resched_write(dreq);
652 spin_lock(&dreq->lock);
653 if (dreq->flags != NFS_ODIRECT_DONE)
654 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
655 spin_unlock(&dreq->lock);
656 nfs_mark_request_commit(req, NULL, cinfo, 0);
659 static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
660 .completion = nfs_direct_commit_complete,
661 .resched_write = nfs_direct_resched_write,
664 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
667 struct nfs_commit_info cinfo;
670 nfs_init_cinfo_from_dreq(&cinfo, dreq);
671 nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
672 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
673 if (res < 0) /* res == -ENOMEM */
674 nfs_direct_write_reschedule(dreq);
677 static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq)
679 struct nfs_commit_info cinfo;
680 struct nfs_page *req;
683 nfs_init_cinfo_from_dreq(&cinfo, dreq);
684 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
686 while (!list_empty(&reqs)) {
687 req = nfs_list_entry(reqs.next);
688 nfs_list_remove_request(req);
689 nfs_direct_truncate_request(dreq, req);
690 nfs_release_request(req);
691 nfs_unlock_and_release_request(req);
695 static void nfs_direct_write_schedule_work(struct work_struct *work)
697 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
698 int flags = dreq->flags;
702 case NFS_ODIRECT_DO_COMMIT:
703 nfs_direct_commit_schedule(dreq);
705 case NFS_ODIRECT_RESCHED_WRITES:
706 nfs_direct_write_reschedule(dreq);
709 nfs_direct_write_clear_reqs(dreq);
710 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);
711 nfs_direct_complete(dreq);
715 static void nfs_direct_write_complete(struct nfs_direct_req *dreq)
717 trace_nfs_direct_write_complete(dreq);
718 queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */
721 static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
723 struct nfs_direct_req *dreq = hdr->dreq;
724 struct nfs_commit_info cinfo;
725 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
726 int flags = NFS_ODIRECT_DONE;
728 trace_nfs_direct_write_completion(dreq);
730 nfs_init_cinfo_from_dreq(&cinfo, dreq);
732 spin_lock(&dreq->lock);
733 if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) {
734 spin_unlock(&dreq->lock);
738 nfs_direct_count_bytes(dreq, hdr);
739 if (test_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags) &&
740 !test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
742 dreq->flags = NFS_ODIRECT_DO_COMMIT;
745 spin_unlock(&dreq->lock);
747 while (!list_empty(&hdr->pages)) {
749 req = nfs_list_entry(hdr->pages.next);
750 nfs_list_remove_request(req);
751 if (flags == NFS_ODIRECT_DO_COMMIT) {
752 kref_get(&req->wb_kref);
753 memcpy(&req->wb_verf, &hdr->verf.verifier,
754 sizeof(req->wb_verf));
755 nfs_mark_request_commit(req, hdr->lseg, &cinfo,
757 } else if (flags == NFS_ODIRECT_RESCHED_WRITES) {
758 kref_get(&req->wb_kref);
759 nfs_mark_request_commit(req, NULL, &cinfo, 0);
761 nfs_unlock_and_release_request(req);
766 nfs_direct_write_complete(dreq);
770 static void nfs_write_sync_pgio_error(struct list_head *head, int error)
772 struct nfs_page *req;
774 while (!list_empty(head)) {
775 req = nfs_list_entry(head->next);
776 nfs_list_remove_request(req);
777 nfs_unlock_and_release_request(req);
781 static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr)
783 struct nfs_direct_req *dreq = hdr->dreq;
784 struct nfs_page *req;
785 struct nfs_commit_info cinfo;
787 trace_nfs_direct_write_reschedule_io(dreq);
789 nfs_init_cinfo_from_dreq(&cinfo, dreq);
790 spin_lock(&dreq->lock);
791 if (dreq->error == 0)
792 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
793 set_bit(NFS_IOHDR_REDO, &hdr->flags);
794 spin_unlock(&dreq->lock);
795 while (!list_empty(&hdr->pages)) {
796 req = nfs_list_entry(hdr->pages.next);
797 nfs_list_remove_request(req);
798 nfs_unlock_request(req);
799 nfs_mark_request_commit(req, NULL, &cinfo, 0);
803 static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
804 .error_cleanup = nfs_write_sync_pgio_error,
805 .init_hdr = nfs_direct_pgio_init,
806 .completion = nfs_direct_write_completion,
807 .reschedule_io = nfs_direct_write_reschedule_io,
812 * NB: Return the value of the first error return code. Subsequent
813 * errors after the first one are ignored.
816 * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
817 * operation. If nfs_writedata_alloc() or get_user_pages() fails,
818 * bail and stop sending more writes. Write length accounting is
819 * handled automatically by nfs_direct_write_result(). Otherwise, if
820 * no requests have been sent, just return an error.
822 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
823 struct iov_iter *iter,
824 loff_t pos, int ioflags)
826 struct nfs_pageio_descriptor desc;
827 struct inode *inode = dreq->inode;
828 struct nfs_commit_info cinfo;
830 size_t requested_bytes = 0;
831 size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
834 trace_nfs_direct_write_schedule_iovec(dreq);
836 nfs_pageio_init_write(&desc, inode, ioflags, false,
837 &nfs_direct_write_completion_ops);
840 inode_dio_begin(inode);
842 NFS_I(inode)->write_io += iov_iter_count(iter);
843 while (iov_iter_count(iter)) {
844 struct page **pagevec;
849 result = iov_iter_get_pages_alloc2(iter, &pagevec,
855 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
856 for (i = 0; i < npages; i++) {
857 struct nfs_page *req;
858 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
860 req = nfs_page_create_from_page(dreq->ctx, pagevec[i],
861 pgbase, pos, req_len);
863 result = PTR_ERR(req);
867 if (desc.pg_error < 0) {
868 nfs_free_request(req);
869 result = desc.pg_error;
875 requested_bytes += req_len;
877 dreq->bytes_left -= req_len;
880 nfs_mark_request_commit(req, NULL, &cinfo, 0);
884 nfs_lock_request(req);
885 if (nfs_pageio_add_request(&desc, req))
888 /* Exit on hard errors */
889 if (desc.pg_error < 0 && desc.pg_error != -EAGAIN) {
890 result = desc.pg_error;
891 nfs_unlock_and_release_request(req);
895 /* If the error is soft, defer remaining requests */
896 nfs_init_cinfo_from_dreq(&cinfo, dreq);
897 spin_lock(&dreq->lock);
898 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
899 spin_unlock(&dreq->lock);
900 nfs_unlock_request(req);
901 nfs_mark_request_commit(req, NULL, &cinfo, 0);
905 nfs_direct_release_pages(pagevec, npages);
910 nfs_pageio_complete(&desc);
913 * If no bytes were started, return the error, and let the
914 * generic layer handle the completion.
916 if (requested_bytes == 0) {
917 inode_dio_end(inode);
918 nfs_direct_req_release(dreq);
919 return result < 0 ? result : -EIO;
923 nfs_direct_write_complete(dreq);
924 return requested_bytes;
928 * nfs_file_direct_write - file direct write operation for NFS files
929 * @iocb: target I/O control block
930 * @iter: vector of user buffers from which to write data
931 * @swap: flag indicating this is swap IO, not O_DIRECT IO
933 * We use this function for direct writes instead of calling
934 * generic_file_aio_write() in order to avoid taking the inode
935 * semaphore and updating the i_size. The NFS server will set
936 * the new i_size and this client must read the updated size
937 * back into its cache. We let the server do generic write
938 * parameter checking and report problems.
940 * We eliminate local atime updates, see direct read above.
942 * We avoid unnecessary page cache invalidations for normal cached
943 * readers of this file.
945 * Note that O_APPEND is not supported for NFS direct writes, as there
946 * is no atomic O_APPEND write facility in the NFS protocol.
948 ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter,
951 ssize_t result, requested;
953 struct file *file = iocb->ki_filp;
954 struct address_space *mapping = file->f_mapping;
955 struct inode *inode = mapping->host;
956 struct nfs_direct_req *dreq;
957 struct nfs_lock_context *l_ctx;
960 dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
961 file, iov_iter_count(iter), (long long) iocb->ki_pos);
964 /* bypass generic checks */
965 result = iov_iter_count(iter);
967 result = generic_write_checks(iocb, iter);
971 nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
974 end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT;
976 task_io_account_write(count);
979 dreq = nfs_direct_req_alloc();
984 dreq->bytes_left = dreq->max_count = count;
985 dreq->io_start = pos;
986 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
987 l_ctx = nfs_get_lock_context(dreq->ctx);
989 result = PTR_ERR(l_ctx);
990 nfs_direct_req_release(dreq);
994 if (!is_sync_kiocb(iocb))
996 pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);
999 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
1002 nfs_start_io_direct(inode);
1004 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos,
1007 if (mapping->nrpages) {
1008 invalidate_inode_pages2_range(mapping,
1009 pos >> PAGE_SHIFT, end);
1012 nfs_end_io_direct(inode);
1015 if (requested > 0) {
1016 result = nfs_direct_wait(dreq);
1018 requested -= result;
1019 iocb->ki_pos = pos + result;
1020 /* XXX: should check the generic_write_sync retval */
1021 generic_write_sync(iocb, result);
1023 iov_iter_revert(iter, requested);
1027 nfs_fscache_invalidate(inode, FSCACHE_INVAL_DIO_WRITE);
1029 nfs_direct_req_release(dreq);
1035 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
1038 int __init nfs_init_directcache(void)
1040 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
1041 sizeof(struct nfs_direct_req),
1042 0, (SLAB_RECLAIM_ACCOUNT|
1045 if (nfs_direct_cachep == NULL)
1052 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
1055 void nfs_destroy_directcache(void)
1057 kmem_cache_destroy(nfs_direct_cachep);