]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - fs/nfs/direct.c
Merge commit 'v2.6.37-rc7' into x86/security
[mirror_ubuntu-artful-kernel.git] / fs / nfs / direct.c
CommitLineData
1da177e4
LT
1/*
2 * linux/fs/nfs/direct.c
3 *
4 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
5 *
6 * High-performance uncached I/O for the Linux NFS client
7 *
8 * There are important applications whose performance or correctness
9 * depends on uncached access to file data. Database clusters
88467055 10 * (multiple copies of the same instance running on separate hosts)
1da177e4 11 * implement their own cache coherency protocol that subsumes file
88467055
CL
12 * system cache protocols. Applications that process datasets
13 * considerably larger than the client's memory do not always benefit
14 * from a local cache. A streaming video server, for instance, has no
1da177e4
LT
15 * need to cache the contents of a file.
16 *
17 * When an application requests uncached I/O, all read and write requests
18 * are made directly to the server; data stored or fetched via these
19 * requests is not cached in the Linux page cache. The client does not
20 * correct unaligned requests from applications. All requested bytes are
21 * held on permanent storage before a direct write system call returns to
22 * an application.
23 *
24 * Solaris implements an uncached I/O facility called directio() that
25 * is used for backups and sequential I/O to very large files. Solaris
26 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
27 * an undocumented mount option.
28 *
29 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
30 * help from Andrew Morton.
31 *
32 * 18 Dec 2001 Initial implementation for 2.4 --cel
33 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
34 * 08 Jun 2003 Port to 2.5 APIs --cel
35 * 31 Mar 2004 Handle direct I/O without VFS support --cel
36 * 15 Sep 2004 Parallel async reads --cel
88467055 37 * 04 May 2005 support O_DIRECT with aio --cel
1da177e4
LT
38 *
39 */
40
1da177e4
LT
41#include <linux/errno.h>
42#include <linux/sched.h>
43#include <linux/kernel.h>
1da177e4
LT
44#include <linux/file.h>
45#include <linux/pagemap.h>
46#include <linux/kref.h>
5a0e3ad6 47#include <linux/slab.h>
1da177e4
LT
48
49#include <linux/nfs_fs.h>
50#include <linux/nfs_page.h>
51#include <linux/sunrpc/clnt.h>
52
53#include <asm/system.h>
54#include <asm/uaccess.h>
55#include <asm/atomic.h>
56
8d5658c9 57#include "internal.h"
91d5b470
CL
58#include "iostat.h"
59
1da177e4 60#define NFSDBG_FACILITY NFSDBG_VFS
1da177e4 61
e18b890b 62static struct kmem_cache *nfs_direct_cachep;
1da177e4
LT
63
64/*
65 * This represents a set of asynchronous requests that we're waiting on
66 */
67struct nfs_direct_req {
68 struct kref kref; /* release manager */
15ce4a0c
CL
69
70 /* I/O parameters */
a8881f5a 71 struct nfs_open_context *ctx; /* file open context info */
f11ac8db 72 struct nfs_lock_context *l_ctx; /* Lock context info */
99514f8f 73 struct kiocb * iocb; /* controlling i/o request */
88467055 74 struct inode * inode; /* target file of i/o */
15ce4a0c
CL
75
76 /* completion state */
607f31e8 77 atomic_t io_count; /* i/os we're waiting for */
15ce4a0c 78 spinlock_t lock; /* protect completion state */
15ce4a0c 79 ssize_t count, /* bytes actually processed */
1da177e4 80 error; /* any reported error */
d72b7a6b 81 struct completion completion; /* wait for i/o completion */
fad61490
TM
82
83 /* commit state */
607f31e8 84 struct list_head rewrite_list; /* saved nfs_write_data structs */
fad61490
TM
85 struct nfs_write_data * commit_data; /* special write_data for commits */
86 int flags;
87#define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
88#define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
89 struct nfs_writeverf verf; /* unstable write verifier */
1da177e4
LT
90};
91
fad61490 92static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
607f31e8
TM
93static const struct rpc_call_ops nfs_write_direct_ops;
94
95static inline void get_dreq(struct nfs_direct_req *dreq)
96{
97 atomic_inc(&dreq->io_count);
98}
99
100static inline int put_dreq(struct nfs_direct_req *dreq)
101{
102 return atomic_dec_and_test(&dreq->io_count);
103}
104
1da177e4 105/**
b8a32e2b
CL
106 * nfs_direct_IO - NFS address space operation for direct I/O
107 * @rw: direction (read or write)
108 * @iocb: target I/O control block
109 * @iov: array of vectors that define I/O buffer
110 * @pos: offset in file to begin the operation
111 * @nr_segs: size of iovec array
112 *
113 * The presence of this routine in the address space ops vector means
114 * the NFS client supports direct I/O. However, we shunt off direct
115 * read and write requests before the VFS gets them, so this method
116 * should never be called.
1da177e4 117 */
b8a32e2b
CL
118ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
119{
b8a32e2b 120 dprintk("NFS: nfs_direct_IO (%s) off/no(%Ld/%lu) EINVAL\n",
01cce933 121 iocb->ki_filp->f_path.dentry->d_name.name,
e99170ff 122 (long long) pos, nr_segs);
b8a32e2b
CL
123
124 return -EINVAL;
125}
126
d4a8f367 127static void nfs_direct_dirty_pages(struct page **pages, unsigned int pgbase, size_t count)
6b45d858 128{
d4a8f367 129 unsigned int npages;
749e146e 130 unsigned int i;
d4a8f367
TM
131
132 if (count == 0)
133 return;
134 pages += (pgbase >> PAGE_SHIFT);
135 npages = (count + (pgbase & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT;
6b45d858
TM
136 for (i = 0; i < npages; i++) {
137 struct page *page = pages[i];
607f31e8 138 if (!PageCompound(page))
d4a8f367 139 set_page_dirty(page);
6b45d858 140 }
9c93ab7d
CL
141}
142
749e146e 143static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
9c93ab7d 144{
749e146e 145 unsigned int i;
607f31e8
TM
146 for (i = 0; i < npages; i++)
147 page_cache_release(pages[i]);
6b45d858
TM
148}
149
93619e59 150static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
1da177e4 151{
93619e59
CL
152 struct nfs_direct_req *dreq;
153
e94b1766 154 dreq = kmem_cache_alloc(nfs_direct_cachep, GFP_KERNEL);
93619e59
CL
155 if (!dreq)
156 return NULL;
157
158 kref_init(&dreq->kref);
607f31e8 159 kref_get(&dreq->kref);
d72b7a6b 160 init_completion(&dreq->completion);
fad61490 161 INIT_LIST_HEAD(&dreq->rewrite_list);
93619e59 162 dreq->iocb = NULL;
a8881f5a 163 dreq->ctx = NULL;
f11ac8db 164 dreq->l_ctx = NULL;
15ce4a0c 165 spin_lock_init(&dreq->lock);
607f31e8 166 atomic_set(&dreq->io_count, 0);
15ce4a0c
CL
167 dreq->count = 0;
168 dreq->error = 0;
fad61490 169 dreq->flags = 0;
93619e59
CL
170
171 return dreq;
1da177e4
LT
172}
173
b4946ffb 174static void nfs_direct_req_free(struct kref *kref)
1da177e4
LT
175{
176 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
a8881f5a 177
f11ac8db
TM
178 if (dreq->l_ctx != NULL)
179 nfs_put_lock_context(dreq->l_ctx);
a8881f5a
TM
180 if (dreq->ctx != NULL)
181 put_nfs_open_context(dreq->ctx);
1da177e4
LT
182 kmem_cache_free(nfs_direct_cachep, dreq);
183}
184
b4946ffb
TM
185static void nfs_direct_req_release(struct nfs_direct_req *dreq)
186{
187 kref_put(&dreq->kref, nfs_direct_req_free);
188}
189
bc0fb201
CL
190/*
191 * Collects and returns the final error value/byte-count.
192 */
193static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
194{
15ce4a0c 195 ssize_t result = -EIOCBQUEUED;
bc0fb201
CL
196
197 /* Async requests don't wait here */
198 if (dreq->iocb)
199 goto out;
200
150030b7 201 result = wait_for_completion_killable(&dreq->completion);
bc0fb201
CL
202
203 if (!result)
15ce4a0c 204 result = dreq->error;
bc0fb201 205 if (!result)
15ce4a0c 206 result = dreq->count;
bc0fb201
CL
207
208out:
bc0fb201
CL
209 return (ssize_t) result;
210}
211
63ab46ab 212/*
607f31e8
TM
213 * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
214 * the iocb is still valid here if this is a synchronous request.
63ab46ab
CL
215 */
216static void nfs_direct_complete(struct nfs_direct_req *dreq)
217{
63ab46ab 218 if (dreq->iocb) {
15ce4a0c 219 long res = (long) dreq->error;
63ab46ab 220 if (!res)
15ce4a0c 221 res = (long) dreq->count;
63ab46ab 222 aio_complete(dreq->iocb, res, 0);
d72b7a6b
TM
223 }
224 complete_all(&dreq->completion);
63ab46ab 225
b4946ffb 226 nfs_direct_req_release(dreq);
63ab46ab
CL
227}
228
06cf6f2e 229/*
607f31e8
TM
230 * We must hold a reference to all the pages in this direct read request
231 * until the RPCs complete. This could be long *after* we are woken up in
232 * nfs_direct_wait (for instance, if someone hits ^C on a slow server).
06cf6f2e 233 */
ec06c096 234static void nfs_direct_read_result(struct rpc_task *task, void *calldata)
1da177e4 235{
ec06c096 236 struct nfs_read_data *data = calldata;
1da177e4 237
fdd1e74c
TM
238 nfs_readpage_result(task, data);
239}
240
241static void nfs_direct_read_release(void *calldata)
242{
243
244 struct nfs_read_data *data = calldata;
245 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
246 int status = data->task.tk_status;
15ce4a0c
CL
247
248 spin_lock(&dreq->lock);
fdd1e74c
TM
249 if (unlikely(status < 0)) {
250 dreq->error = status;
d4a8f367
TM
251 spin_unlock(&dreq->lock);
252 } else {
253 dreq->count += data->res.count;
254 spin_unlock(&dreq->lock);
255 nfs_direct_dirty_pages(data->pagevec,
256 data->args.pgbase,
257 data->res.count);
258 }
259 nfs_direct_release_pages(data->pagevec, data->npages);
607f31e8
TM
260
261 if (put_dreq(dreq))
262 nfs_direct_complete(dreq);
1ae88b2e 263 nfs_readdata_free(data);
1da177e4
LT
264}
265
ec06c096 266static const struct rpc_call_ops nfs_read_direct_ops = {
f11c88af
AA
267#if defined(CONFIG_NFS_V4_1)
268 .rpc_call_prepare = nfs_read_prepare,
269#endif /* CONFIG_NFS_V4_1 */
ec06c096 270 .rpc_call_done = nfs_direct_read_result,
fdd1e74c 271 .rpc_release = nfs_direct_read_release,
ec06c096
TM
272};
273
d4cc948b 274/*
607f31e8
TM
275 * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
276 * operation. If nfs_readdata_alloc() or get_user_pages() fails,
277 * bail and stop sending more reads. Read length accounting is
278 * handled automatically by nfs_direct_read_result(). Otherwise, if
279 * no requests have been sent, just return an error.
1da177e4 280 */
02fe4946
CL
281static ssize_t nfs_direct_read_schedule_segment(struct nfs_direct_req *dreq,
282 const struct iovec *iov,
283 loff_t pos)
1da177e4 284{
a8881f5a 285 struct nfs_open_context *ctx = dreq->ctx;
88be9f99 286 struct inode *inode = ctx->path.dentry->d_inode;
02fe4946
CL
287 unsigned long user_addr = (unsigned long)iov->iov_base;
288 size_t count = iov->iov_len;
5dd602f2 289 size_t rsize = NFS_SERVER(inode)->rsize;
07737691 290 struct rpc_task *task;
bdc7f021
TM
291 struct rpc_message msg = {
292 .rpc_cred = ctx->cred,
293 };
84115e1c
TM
294 struct rpc_task_setup task_setup_data = {
295 .rpc_client = NFS_CLIENT(inode),
bdc7f021 296 .rpc_message = &msg,
84115e1c 297 .callback_ops = &nfs_read_direct_ops,
101070ca 298 .workqueue = nfsiod_workqueue,
84115e1c
TM
299 .flags = RPC_TASK_ASYNC,
300 };
607f31e8
TM
301 unsigned int pgbase;
302 int result;
303 ssize_t started = 0;
304
1da177e4 305 do {
82b145c5 306 struct nfs_read_data *data;
5dd602f2 307 size_t bytes;
1da177e4 308
e9f7bee1
TM
309 pgbase = user_addr & ~PAGE_MASK;
310 bytes = min(rsize,count);
311
607f31e8 312 result = -ENOMEM;
8d5658c9 313 data = nfs_readdata_alloc(nfs_page_array_len(pgbase, bytes));
607f31e8
TM
314 if (unlikely(!data))
315 break;
316
607f31e8
TM
317 down_read(&current->mm->mmap_sem);
318 result = get_user_pages(current, current->mm, user_addr,
319 data->npages, 1, 0, data->pagevec, NULL);
320 up_read(&current->mm->mmap_sem);
749e146e 321 if (result < 0) {
1ae88b2e 322 nfs_readdata_free(data);
749e146e
CL
323 break;
324 }
325 if ((unsigned)result < data->npages) {
d9df8d6b
TM
326 bytes = result * PAGE_SIZE;
327 if (bytes <= pgbase) {
328 nfs_direct_release_pages(data->pagevec, result);
1ae88b2e 329 nfs_readdata_free(data);
d9df8d6b
TM
330 break;
331 }
332 bytes -= pgbase;
333 data->npages = result;
607f31e8
TM
334 }
335
336 get_dreq(dreq);
82b145c5 337
607f31e8 338 data->req = (struct nfs_page *) dreq;
1da177e4 339 data->inode = inode;
bdc7f021 340 data->cred = msg.rpc_cred;
1da177e4 341 data->args.fh = NFS_FH(inode);
1ae88b2e 342 data->args.context = ctx;
f11ac8db 343 data->args.lock_context = dreq->l_ctx;
88467055 344 data->args.offset = pos;
1da177e4 345 data->args.pgbase = pgbase;
607f31e8 346 data->args.pages = data->pagevec;
1da177e4
LT
347 data->args.count = bytes;
348 data->res.fattr = &data->fattr;
349 data->res.eof = 0;
350 data->res.count = bytes;
65d26953 351 nfs_fattr_init(&data->fattr);
bdc7f021
TM
352 msg.rpc_argp = &data->args;
353 msg.rpc_resp = &data->res;
1da177e4 354
07737691 355 task_setup_data.task = &data->task;
84115e1c 356 task_setup_data.callback_data = data;
bdc7f021 357 NFS_PROTO(inode)->read_setup(data, &msg);
1da177e4 358
07737691 359 task = rpc_run_task(&task_setup_data);
dbae4c73
TM
360 if (IS_ERR(task))
361 break;
362 rpc_put_task(task);
1da177e4 363
a3f565b1
CL
364 dprintk("NFS: %5u initiated direct read call "
365 "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
1da177e4
LT
366 data->task.tk_pid,
367 inode->i_sb->s_id,
368 (long long)NFS_FILEID(inode),
369 bytes,
370 (unsigned long long)data->args.offset);
371
607f31e8
TM
372 started += bytes;
373 user_addr += bytes;
88467055 374 pos += bytes;
e9f7bee1 375 /* FIXME: Remove this unnecessary math from final patch */
1da177e4 376 pgbase += bytes;
1da177e4 377 pgbase &= ~PAGE_MASK;
e9f7bee1 378 BUG_ON(pgbase != (user_addr & ~PAGE_MASK));
1da177e4
LT
379
380 count -= bytes;
381 } while (count != 0);
607f31e8 382
607f31e8 383 if (started)
c216fd70 384 return started;
607f31e8 385 return result < 0 ? (ssize_t) result : -EFAULT;
1da177e4
LT
386}
387
19f73787
CL
388static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
389 const struct iovec *iov,
390 unsigned long nr_segs,
391 loff_t pos)
392{
393 ssize_t result = -EINVAL;
394 size_t requested_bytes = 0;
395 unsigned long seg;
396
397 get_dreq(dreq);
398
399 for (seg = 0; seg < nr_segs; seg++) {
400 const struct iovec *vec = &iov[seg];
02fe4946 401 result = nfs_direct_read_schedule_segment(dreq, vec, pos);
19f73787
CL
402 if (result < 0)
403 break;
404 requested_bytes += result;
405 if ((size_t)result < vec->iov_len)
406 break;
407 pos += vec->iov_len;
408 }
409
410 if (put_dreq(dreq))
411 nfs_direct_complete(dreq);
412
413 if (requested_bytes != 0)
414 return 0;
415
416 if (result < 0)
417 return result;
418 return -EIO;
419}
420
c216fd70
CL
421static ssize_t nfs_direct_read(struct kiocb *iocb, const struct iovec *iov,
422 unsigned long nr_segs, loff_t pos)
1da177e4 423{
f11ac8db 424 ssize_t result = -ENOMEM;
99514f8f 425 struct inode *inode = iocb->ki_filp->f_mapping->host;
1da177e4
LT
426 struct nfs_direct_req *dreq;
427
607f31e8 428 dreq = nfs_direct_req_alloc();
f11ac8db
TM
429 if (dreq == NULL)
430 goto out;
1da177e4 431
91d5b470 432 dreq->inode = inode;
cd3758e3 433 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
f11ac8db
TM
434 dreq->l_ctx = nfs_get_lock_context(dreq->ctx);
435 if (dreq->l_ctx == NULL)
436 goto out_release;
487b8372
CL
437 if (!is_sync_kiocb(iocb))
438 dreq->iocb = iocb;
1da177e4 439
c216fd70 440 result = nfs_direct_read_schedule_iovec(dreq, iov, nr_segs, pos);
607f31e8
TM
441 if (!result)
442 result = nfs_direct_wait(dreq);
f11ac8db 443out_release:
b4946ffb 444 nfs_direct_req_release(dreq);
f11ac8db 445out:
1da177e4
LT
446 return result;
447}
448
fad61490 449static void nfs_direct_free_writedata(struct nfs_direct_req *dreq)
1da177e4 450{
607f31e8
TM
451 while (!list_empty(&dreq->rewrite_list)) {
452 struct nfs_write_data *data = list_entry(dreq->rewrite_list.next, struct nfs_write_data, pages);
fad61490 453 list_del(&data->pages);
607f31e8 454 nfs_direct_release_pages(data->pagevec, data->npages);
1ae88b2e 455 nfs_writedata_free(data);
fad61490
TM
456 }
457}
1da177e4 458
fad61490
TM
459#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
460static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
461{
607f31e8
TM
462 struct inode *inode = dreq->inode;
463 struct list_head *p;
464 struct nfs_write_data *data;
07737691 465 struct rpc_task *task;
bdc7f021
TM
466 struct rpc_message msg = {
467 .rpc_cred = dreq->ctx->cred,
468 };
84115e1c
TM
469 struct rpc_task_setup task_setup_data = {
470 .rpc_client = NFS_CLIENT(inode),
a8b40bc7 471 .rpc_message = &msg,
84115e1c 472 .callback_ops = &nfs_write_direct_ops,
101070ca 473 .workqueue = nfsiod_workqueue,
84115e1c
TM
474 .flags = RPC_TASK_ASYNC,
475 };
1da177e4 476
fad61490 477 dreq->count = 0;
607f31e8
TM
478 get_dreq(dreq);
479
480 list_for_each(p, &dreq->rewrite_list) {
481 data = list_entry(p, struct nfs_write_data, pages);
482
483 get_dreq(dreq);
484
bdc7f021
TM
485 /* Use stable writes */
486 data->args.stable = NFS_FILE_SYNC;
487
607f31e8
TM
488 /*
489 * Reset data->res.
490 */
491 nfs_fattr_init(&data->fattr);
492 data->res.count = data->args.count;
493 memset(&data->verf, 0, sizeof(data->verf));
494
495 /*
496 * Reuse data->task; data->args should not have changed
497 * since the original request was sent.
498 */
07737691 499 task_setup_data.task = &data->task;
84115e1c 500 task_setup_data.callback_data = data;
bdc7f021
TM
501 msg.rpc_argp = &data->args;
502 msg.rpc_resp = &data->res;
503 NFS_PROTO(inode)->write_setup(data, &msg);
607f31e8 504
607f31e8
TM
505 /*
506 * We're called via an RPC callback, so BKL is already held.
507 */
07737691
TM
508 task = rpc_run_task(&task_setup_data);
509 if (!IS_ERR(task))
510 rpc_put_task(task);
607f31e8
TM
511
512 dprintk("NFS: %5u rescheduled direct write call (req %s/%Ld, %u bytes @ offset %Lu)\n",
513 data->task.tk_pid,
514 inode->i_sb->s_id,
515 (long long)NFS_FILEID(inode),
516 data->args.count,
517 (unsigned long long)data->args.offset);
518 }
fedb595c 519
607f31e8
TM
520 if (put_dreq(dreq))
521 nfs_direct_write_complete(dreq, inode);
fad61490
TM
522}
523
524static void nfs_direct_commit_result(struct rpc_task *task, void *calldata)
525{
526 struct nfs_write_data *data = calldata;
fad61490
TM
527
528 /* Call the NFS version-specific code */
c9d8f89d
TM
529 NFS_PROTO(data->inode)->commit_done(task, data);
530}
531
532static void nfs_direct_commit_release(void *calldata)
533{
534 struct nfs_write_data *data = calldata;
535 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
536 int status = data->task.tk_status;
537
538 if (status < 0) {
60fa3f76 539 dprintk("NFS: %5u commit failed with error %d.\n",
c9d8f89d 540 data->task.tk_pid, status);
fad61490 541 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
60fa3f76 542 } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) {
c9d8f89d 543 dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid);
fad61490 544 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
1da177e4
LT
545 }
546
c9d8f89d 547 dprintk("NFS: %5u commit returned %d\n", data->task.tk_pid, status);
fad61490 548 nfs_direct_write_complete(dreq, data->inode);
1ae88b2e 549 nfs_commit_free(data);
1da177e4
LT
550}
551
fad61490 552static const struct rpc_call_ops nfs_commit_direct_ops = {
21d9a851
AA
553#if defined(CONFIG_NFS_V4_1)
554 .rpc_call_prepare = nfs_write_prepare,
555#endif /* CONFIG_NFS_V4_1 */
fad61490 556 .rpc_call_done = nfs_direct_commit_result,
c9d8f89d 557 .rpc_release = nfs_direct_commit_release,
fad61490
TM
558};
559
560static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
1da177e4 561{
fad61490 562 struct nfs_write_data *data = dreq->commit_data;
07737691 563 struct rpc_task *task;
bdc7f021
TM
564 struct rpc_message msg = {
565 .rpc_argp = &data->args,
566 .rpc_resp = &data->res,
567 .rpc_cred = dreq->ctx->cred,
568 };
84115e1c 569 struct rpc_task_setup task_setup_data = {
07737691 570 .task = &data->task,
84115e1c 571 .rpc_client = NFS_CLIENT(dreq->inode),
bdc7f021 572 .rpc_message = &msg,
84115e1c
TM
573 .callback_ops = &nfs_commit_direct_ops,
574 .callback_data = data,
101070ca 575 .workqueue = nfsiod_workqueue,
84115e1c
TM
576 .flags = RPC_TASK_ASYNC,
577 };
1da177e4 578
fad61490 579 data->inode = dreq->inode;
bdc7f021 580 data->cred = msg.rpc_cred;
1da177e4 581
fad61490 582 data->args.fh = NFS_FH(data->inode);
607f31e8
TM
583 data->args.offset = 0;
584 data->args.count = 0;
1ae88b2e 585 data->args.context = dreq->ctx;
f11ac8db 586 data->args.lock_context = dreq->l_ctx;
fad61490
TM
587 data->res.count = 0;
588 data->res.fattr = &data->fattr;
589 data->res.verf = &data->verf;
65d26953 590 nfs_fattr_init(&data->fattr);
1da177e4 591
bdc7f021 592 NFS_PROTO(data->inode)->commit_setup(data, &msg);
1da177e4 593
fad61490
TM
594 /* Note: task.tk_ops->rpc_release will free dreq->commit_data */
595 dreq->commit_data = NULL;
1da177e4 596
e99170ff 597 dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
1da177e4 598
07737691
TM
599 task = rpc_run_task(&task_setup_data);
600 if (!IS_ERR(task))
601 rpc_put_task(task);
fad61490 602}
1da177e4 603
fad61490
TM
604static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
605{
606 int flags = dreq->flags;
1da177e4 607
fad61490
TM
608 dreq->flags = 0;
609 switch (flags) {
610 case NFS_ODIRECT_DO_COMMIT:
611 nfs_direct_commit_schedule(dreq);
1da177e4 612 break;
fad61490
TM
613 case NFS_ODIRECT_RESCHED_WRITES:
614 nfs_direct_write_reschedule(dreq);
615 break;
616 default:
fad61490
TM
617 if (dreq->commit_data != NULL)
618 nfs_commit_free(dreq->commit_data);
619 nfs_direct_free_writedata(dreq);
cd9ae2b6 620 nfs_zap_mapping(inode, inode->i_mapping);
fad61490
TM
621 nfs_direct_complete(dreq);
622 }
623}
1da177e4 624
fad61490
TM
625static void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
626{
c9d8f89d 627 dreq->commit_data = nfs_commitdata_alloc();
fad61490
TM
628 if (dreq->commit_data != NULL)
629 dreq->commit_data->req = (struct nfs_page *) dreq;
630}
631#else
632static inline void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
633{
634 dreq->commit_data = NULL;
635}
1da177e4 636
fad61490
TM
637static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
638{
fad61490 639 nfs_direct_free_writedata(dreq);
cd9ae2b6 640 nfs_zap_mapping(inode, inode->i_mapping);
fad61490
TM
641 nfs_direct_complete(dreq);
642}
643#endif
1da177e4 644
462d5b32 645static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
1da177e4 646{
462d5b32 647 struct nfs_write_data *data = calldata;
462d5b32
CL
648
649 if (nfs_writeback_done(task, data) != 0)
650 return;
c9d8f89d
TM
651}
652
653/*
654 * NB: Return the value of the first error return code. Subsequent
655 * errors after the first one are ignored.
656 */
657static void nfs_direct_write_release(void *calldata)
658{
659 struct nfs_write_data *data = calldata;
660 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
661 int status = data->task.tk_status;
462d5b32 662
15ce4a0c 663 spin_lock(&dreq->lock);
1da177e4 664
eda3cef8 665 if (unlikely(status < 0)) {
432409ee 666 /* An error has occurred, so we should not commit */
60fa3f76 667 dreq->flags = 0;
eda3cef8 668 dreq->error = status;
eda3cef8 669 }
432409ee
NB
670 if (unlikely(dreq->error != 0))
671 goto out_unlock;
eda3cef8
TM
672
673 dreq->count += data->res.count;
1da177e4 674
fad61490
TM
675 if (data->res.verf->committed != NFS_FILE_SYNC) {
676 switch (dreq->flags) {
677 case 0:
678 memcpy(&dreq->verf, &data->verf, sizeof(dreq->verf));
679 dreq->flags = NFS_ODIRECT_DO_COMMIT;
1da177e4 680 break;
fad61490
TM
681 case NFS_ODIRECT_DO_COMMIT:
682 if (memcmp(&dreq->verf, &data->verf, sizeof(dreq->verf))) {
c9d8f89d 683 dprintk("NFS: %5u write verify failed\n", data->task.tk_pid);
fad61490
TM
684 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
685 }
1da177e4 686 }
1da177e4 687 }
eda3cef8 688out_unlock:
fad61490 689 spin_unlock(&dreq->lock);
1da177e4 690
607f31e8
TM
691 if (put_dreq(dreq))
692 nfs_direct_write_complete(dreq, data->inode);
462d5b32
CL
693}
694
695static const struct rpc_call_ops nfs_write_direct_ops = {
def6ed7e
AA
696#if defined(CONFIG_NFS_V4_1)
697 .rpc_call_prepare = nfs_write_prepare,
698#endif /* CONFIG_NFS_V4_1 */
462d5b32 699 .rpc_call_done = nfs_direct_write_result,
fad61490 700 .rpc_release = nfs_direct_write_release,
462d5b32
CL
701};
702
703/*
607f31e8
TM
704 * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
705 * operation. If nfs_writedata_alloc() or get_user_pages() fails,
706 * bail and stop sending more writes. Write length accounting is
707 * handled automatically by nfs_direct_write_result(). Otherwise, if
708 * no requests have been sent, just return an error.
462d5b32 709 */
02fe4946
CL
710static ssize_t nfs_direct_write_schedule_segment(struct nfs_direct_req *dreq,
711 const struct iovec *iov,
712 loff_t pos, int sync)
462d5b32 713{
a8881f5a 714 struct nfs_open_context *ctx = dreq->ctx;
88be9f99 715 struct inode *inode = ctx->path.dentry->d_inode;
02fe4946
CL
716 unsigned long user_addr = (unsigned long)iov->iov_base;
717 size_t count = iov->iov_len;
07737691 718 struct rpc_task *task;
bdc7f021
TM
719 struct rpc_message msg = {
720 .rpc_cred = ctx->cred,
721 };
84115e1c
TM
722 struct rpc_task_setup task_setup_data = {
723 .rpc_client = NFS_CLIENT(inode),
bdc7f021 724 .rpc_message = &msg,
84115e1c 725 .callback_ops = &nfs_write_direct_ops,
101070ca 726 .workqueue = nfsiod_workqueue,
84115e1c
TM
727 .flags = RPC_TASK_ASYNC,
728 };
462d5b32 729 size_t wsize = NFS_SERVER(inode)->wsize;
607f31e8
TM
730 unsigned int pgbase;
731 int result;
732 ssize_t started = 0;
82b145c5 733
1da177e4 734 do {
82b145c5 735 struct nfs_write_data *data;
462d5b32
CL
736 size_t bytes;
737
e9f7bee1
TM
738 pgbase = user_addr & ~PAGE_MASK;
739 bytes = min(wsize,count);
740
607f31e8 741 result = -ENOMEM;
8d5658c9 742 data = nfs_writedata_alloc(nfs_page_array_len(pgbase, bytes));
607f31e8
TM
743 if (unlikely(!data))
744 break;
745
607f31e8
TM
746 down_read(&current->mm->mmap_sem);
747 result = get_user_pages(current, current->mm, user_addr,
748 data->npages, 0, 0, data->pagevec, NULL);
749 up_read(&current->mm->mmap_sem);
749e146e 750 if (result < 0) {
1ae88b2e 751 nfs_writedata_free(data);
749e146e
CL
752 break;
753 }
754 if ((unsigned)result < data->npages) {
d9df8d6b
TM
755 bytes = result * PAGE_SIZE;
756 if (bytes <= pgbase) {
757 nfs_direct_release_pages(data->pagevec, result);
1ae88b2e 758 nfs_writedata_free(data);
d9df8d6b
TM
759 break;
760 }
761 bytes -= pgbase;
762 data->npages = result;
607f31e8
TM
763 }
764
765 get_dreq(dreq);
766
fad61490 767 list_move_tail(&data->pages, &dreq->rewrite_list);
462d5b32 768
607f31e8 769 data->req = (struct nfs_page *) dreq;
462d5b32 770 data->inode = inode;
bdc7f021 771 data->cred = msg.rpc_cred;
462d5b32 772 data->args.fh = NFS_FH(inode);
1ae88b2e 773 data->args.context = ctx;
f11ac8db 774 data->args.lock_context = dreq->l_ctx;
88467055 775 data->args.offset = pos;
462d5b32 776 data->args.pgbase = pgbase;
607f31e8 777 data->args.pages = data->pagevec;
462d5b32 778 data->args.count = bytes;
bdc7f021 779 data->args.stable = sync;
462d5b32
CL
780 data->res.fattr = &data->fattr;
781 data->res.count = bytes;
47989d74 782 data->res.verf = &data->verf;
65d26953 783 nfs_fattr_init(&data->fattr);
462d5b32 784
07737691 785 task_setup_data.task = &data->task;
84115e1c 786 task_setup_data.callback_data = data;
bdc7f021
TM
787 msg.rpc_argp = &data->args;
788 msg.rpc_resp = &data->res;
789 NFS_PROTO(inode)->write_setup(data, &msg);
1da177e4 790
07737691 791 task = rpc_run_task(&task_setup_data);
dbae4c73
TM
792 if (IS_ERR(task))
793 break;
794 rpc_put_task(task);
1da177e4 795
a3f565b1
CL
796 dprintk("NFS: %5u initiated direct write call "
797 "(req %s/%Ld, %zu bytes @ offset %Lu)\n",
462d5b32
CL
798 data->task.tk_pid,
799 inode->i_sb->s_id,
800 (long long)NFS_FILEID(inode),
801 bytes,
802 (unsigned long long)data->args.offset);
1da177e4 803
607f31e8
TM
804 started += bytes;
805 user_addr += bytes;
88467055 806 pos += bytes;
e9f7bee1
TM
807
808 /* FIXME: Remove this useless math from the final patch */
462d5b32 809 pgbase += bytes;
462d5b32 810 pgbase &= ~PAGE_MASK;
e9f7bee1 811 BUG_ON(pgbase != (user_addr & ~PAGE_MASK));
1da177e4 812
462d5b32
CL
813 count -= bytes;
814 } while (count != 0);
607f31e8 815
607f31e8 816 if (started)
c216fd70 817 return started;
607f31e8 818 return result < 0 ? (ssize_t) result : -EFAULT;
462d5b32 819}
1da177e4 820
19f73787
CL
821static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
822 const struct iovec *iov,
823 unsigned long nr_segs,
824 loff_t pos, int sync)
825{
826 ssize_t result = 0;
827 size_t requested_bytes = 0;
828 unsigned long seg;
829
830 get_dreq(dreq);
831
832 for (seg = 0; seg < nr_segs; seg++) {
833 const struct iovec *vec = &iov[seg];
02fe4946
CL
834 result = nfs_direct_write_schedule_segment(dreq, vec,
835 pos, sync);
19f73787
CL
836 if (result < 0)
837 break;
838 requested_bytes += result;
839 if ((size_t)result < vec->iov_len)
840 break;
841 pos += vec->iov_len;
842 }
843
844 if (put_dreq(dreq))
845 nfs_direct_write_complete(dreq, dreq->inode);
846
847 if (requested_bytes != 0)
848 return 0;
849
850 if (result < 0)
851 return result;
852 return -EIO;
853}
854
c216fd70
CL
855static ssize_t nfs_direct_write(struct kiocb *iocb, const struct iovec *iov,
856 unsigned long nr_segs, loff_t pos,
857 size_t count)
462d5b32 858{
f11ac8db 859 ssize_t result = -ENOMEM;
c89f2ee5 860 struct inode *inode = iocb->ki_filp->f_mapping->host;
462d5b32 861 struct nfs_direct_req *dreq;
fad61490 862 size_t wsize = NFS_SERVER(inode)->wsize;
bdc7f021 863 int sync = NFS_UNSTABLE;
1da177e4 864
607f31e8 865 dreq = nfs_direct_req_alloc();
462d5b32 866 if (!dreq)
f11ac8db 867 goto out;
607f31e8
TM
868 nfs_alloc_commit_data(dreq);
869
b47d19de 870 if (dreq->commit_data == NULL || count <= wsize)
bdc7f021 871 sync = NFS_FILE_SYNC;
1da177e4 872
c89f2ee5 873 dreq->inode = inode;
cd3758e3 874 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
f11ac8db 875 dreq->l_ctx = nfs_get_lock_context(dreq->ctx);
568a810d 876 if (dreq->l_ctx == NULL)
f11ac8db 877 goto out_release;
c89f2ee5
CL
878 if (!is_sync_kiocb(iocb))
879 dreq->iocb = iocb;
1da177e4 880
c216fd70 881 result = nfs_direct_write_schedule_iovec(dreq, iov, nr_segs, pos, sync);
607f31e8
TM
882 if (!result)
883 result = nfs_direct_wait(dreq);
f11ac8db 884out_release:
b4946ffb 885 nfs_direct_req_release(dreq);
f11ac8db 886out:
1da177e4
LT
887 return result;
888}
889
890/**
891 * nfs_file_direct_read - file direct read operation for NFS files
892 * @iocb: target I/O control block
027445c3
BP
893 * @iov: vector of user buffers into which to read data
894 * @nr_segs: size of iov vector
88467055 895 * @pos: byte offset in file where reading starts
1da177e4
LT
896 *
897 * We use this function for direct reads instead of calling
898 * generic_file_aio_read() in order to avoid gfar's check to see if
899 * the request starts before the end of the file. For that check
900 * to work, we must generate a GETATTR before each direct read, and
901 * even then there is a window between the GETATTR and the subsequent
88467055 902 * READ where the file size could change. Our preference is simply
1da177e4
LT
903 * to do all reads the application wants, and the server will take
904 * care of managing the end of file boundary.
88467055 905 *
1da177e4
LT
906 * This function also eliminates unnecessarily updating the file's
907 * atime locally, as the NFS server sets the file's atime, and this
908 * client must read the updated atime from the server back into its
909 * cache.
910 */
027445c3
BP
911ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov,
912 unsigned long nr_segs, loff_t pos)
1da177e4
LT
913{
914 ssize_t retval = -EINVAL;
1da177e4 915 struct file *file = iocb->ki_filp;
1da177e4 916 struct address_space *mapping = file->f_mapping;
c216fd70
CL
917 size_t count;
918
919 count = iov_length(iov, nr_segs);
920 nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
1da177e4 921
6da24bc9 922 dfprintk(FILE, "NFS: direct read(%s/%s, %zd@%Ld)\n",
01cce933
JJS
923 file->f_path.dentry->d_parent->d_name.name,
924 file->f_path.dentry->d_name.name,
c216fd70 925 count, (long long) pos);
1da177e4 926
1da177e4
LT
927 retval = 0;
928 if (!count)
929 goto out;
930
29884df0
TM
931 retval = nfs_sync_mapping(mapping);
932 if (retval)
933 goto out;
1da177e4 934
c216fd70 935 retval = nfs_direct_read(iocb, iov, nr_segs, pos);
1da177e4 936 if (retval > 0)
0cdd80d0 937 iocb->ki_pos = pos + retval;
1da177e4
LT
938
939out:
940 return retval;
941}
942
943/**
944 * nfs_file_direct_write - file direct write operation for NFS files
945 * @iocb: target I/O control block
027445c3
BP
946 * @iov: vector of user buffers from which to write data
947 * @nr_segs: size of iov vector
88467055 948 * @pos: byte offset in file where writing starts
1da177e4
LT
949 *
950 * We use this function for direct writes instead of calling
951 * generic_file_aio_write() in order to avoid taking the inode
952 * semaphore and updating the i_size. The NFS server will set
953 * the new i_size and this client must read the updated size
954 * back into its cache. We let the server do generic write
955 * parameter checking and report problems.
956 *
1da177e4
LT
957 * We eliminate local atime updates, see direct read above.
958 *
959 * We avoid unnecessary page cache invalidations for normal cached
960 * readers of this file.
961 *
962 * Note that O_APPEND is not supported for NFS direct writes, as there
963 * is no atomic O_APPEND write facility in the NFS protocol.
964 */
027445c3
BP
965ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
966 unsigned long nr_segs, loff_t pos)
1da177e4 967{
070ea602 968 ssize_t retval = -EINVAL;
1da177e4 969 struct file *file = iocb->ki_filp;
1da177e4 970 struct address_space *mapping = file->f_mapping;
c216fd70 971 size_t count;
1da177e4 972
c216fd70
CL
973 count = iov_length(iov, nr_segs);
974 nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
975
6da24bc9 976 dfprintk(FILE, "NFS: direct write(%s/%s, %zd@%Ld)\n",
01cce933
JJS
977 file->f_path.dentry->d_parent->d_name.name,
978 file->f_path.dentry->d_name.name,
c216fd70 979 count, (long long) pos);
027445c3 980
ce1a8e67
CL
981 retval = generic_write_checks(file, &pos, &count, 0);
982 if (retval)
1da177e4 983 goto out;
ce1a8e67
CL
984
985 retval = -EINVAL;
986 if ((ssize_t) count < 0)
1da177e4 987 goto out;
1da177e4
LT
988 retval = 0;
989 if (!count)
990 goto out;
ce1a8e67 991
29884df0
TM
992 retval = nfs_sync_mapping(mapping);
993 if (retval)
994 goto out;
1da177e4 995
c216fd70 996 retval = nfs_direct_write(iocb, iov, nr_segs, pos, count);
9eafa8cc 997
1da177e4 998 if (retval > 0)
ce1a8e67 999 iocb->ki_pos = pos + retval;
1da177e4
LT
1000
1001out:
1002 return retval;
1003}
1004
88467055
CL
1005/**
1006 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
1007 *
1008 */
f7b422b1 1009int __init nfs_init_directcache(void)
1da177e4
LT
1010{
1011 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
1012 sizeof(struct nfs_direct_req),
fffb60f9
PJ
1013 0, (SLAB_RECLAIM_ACCOUNT|
1014 SLAB_MEM_SPREAD),
20c2df83 1015 NULL);
1da177e4
LT
1016 if (nfs_direct_cachep == NULL)
1017 return -ENOMEM;
1018
1019 return 0;
1020}
1021
88467055 1022/**
f7b422b1 1023 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
88467055
CL
1024 *
1025 */
266bee88 1026void nfs_destroy_directcache(void)
1da177e4 1027{
1a1d92c1 1028 kmem_cache_destroy(nfs_direct_cachep);
1da177e4 1029}