]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - fs/nfs/write.c
nfs: merge nfs_pgio_data into _header
[mirror_ubuntu-bionic-kernel.git] / fs / nfs / write.c
CommitLineData
1da177e4
LT
1/*
2 * linux/fs/nfs/write.c
3 *
7c85d900 4 * Write file data over NFS.
1da177e4
LT
5 *
6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
7 */
8
1da177e4
LT
9#include <linux/types.h>
10#include <linux/slab.h>
11#include <linux/mm.h>
12#include <linux/pagemap.h>
13#include <linux/file.h>
1da177e4 14#include <linux/writeback.h>
89a09141 15#include <linux/swap.h>
074cc1de 16#include <linux/migrate.h>
1da177e4
LT
17
18#include <linux/sunrpc/clnt.h>
19#include <linux/nfs_fs.h>
20#include <linux/nfs_mount.h>
21#include <linux/nfs_page.h>
3fcfab16 22#include <linux/backing-dev.h>
afeacc8c 23#include <linux/export.h>
3fcfab16 24
1da177e4 25#include <asm/uaccess.h>
1da177e4
LT
26
27#include "delegation.h"
49a70f27 28#include "internal.h"
91d5b470 29#include "iostat.h"
def6ed7e 30#include "nfs4_fs.h"
074cc1de 31#include "fscache.h"
94ad1c80 32#include "pnfs.h"
1da177e4 33
f4ce1299
TM
34#include "nfstrace.h"
35
1da177e4
LT
36#define NFSDBG_FACILITY NFSDBG_PAGECACHE
37
38#define MIN_POOL_WRITE (32)
39#define MIN_POOL_COMMIT (4)
40
41/*
42 * Local function declarations
43 */
f8512ad0 44static void nfs_redirty_request(struct nfs_page *req);
788e7a89 45static const struct rpc_call_ops nfs_commit_ops;
061ae2ed 46static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
f453a54a 47static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
4a0de55c 48static const struct nfs_rw_ops nfs_rw_write_ops;
1da177e4 49
e18b890b 50static struct kmem_cache *nfs_wdata_cachep;
3feb2d49 51static mempool_t *nfs_wdata_mempool;
0b7c0153 52static struct kmem_cache *nfs_cdata_cachep;
1da177e4
LT
53static mempool_t *nfs_commit_mempool;
54
0b7c0153 55struct nfs_commit_data *nfs_commitdata_alloc(void)
1da177e4 56{
192e501b 57 struct nfs_commit_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
40859d7e 58
1da177e4
LT
59 if (p) {
60 memset(p, 0, sizeof(*p));
61 INIT_LIST_HEAD(&p->pages);
62 }
63 return p;
64}
e0c2b380 65EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
1da177e4 66
0b7c0153 67void nfs_commit_free(struct nfs_commit_data *p)
1da177e4
LT
68{
69 mempool_free(p, nfs_commit_mempool);
70}
e0c2b380 71EXPORT_SYMBOL_GPL(nfs_commit_free);
1da177e4 72
1e7f3a48 73static struct nfs_pgio_header *nfs_writehdr_alloc(void)
3feb2d49 74{
1e7f3a48 75 struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
cd841605 76
4a0de55c 77 if (p)
3feb2d49 78 memset(p, 0, sizeof(*p));
3feb2d49
TM
79 return p;
80}
6c75dc0d 81
1e7f3a48 82static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
3feb2d49 83{
1e7f3a48 84 mempool_free(hdr, nfs_wdata_mempool);
3feb2d49 85}
1da177e4 86
7b159fc1
TM
87static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
88{
89 ctx->error = error;
90 smp_wmb();
91 set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
92}
93
29418aa4
MG
94static struct nfs_page *
95nfs_page_find_request_locked(struct nfs_inode *nfsi, struct page *page)
277459d2
TM
96{
97 struct nfs_page *req = NULL;
98
29418aa4 99 if (PagePrivate(page))
277459d2 100 req = (struct nfs_page *)page_private(page);
29418aa4
MG
101 else if (unlikely(PageSwapCache(page))) {
102 struct nfs_page *freq, *t;
103
104 /* Linearly search the commit list for the correct req */
105 list_for_each_entry_safe(freq, t, &nfsi->commit_info.list, wb_list) {
106 if (freq->wb_page == page) {
107 req = freq;
108 break;
109 }
110 }
277459d2 111 }
29418aa4
MG
112
113 if (req)
114 kref_get(&req->wb_kref);
115
277459d2
TM
116 return req;
117}
118
119static struct nfs_page *nfs_page_find_request(struct page *page)
120{
d56b4ddf 121 struct inode *inode = page_file_mapping(page)->host;
277459d2 122 struct nfs_page *req = NULL;
277459d2 123
587142f8 124 spin_lock(&inode->i_lock);
29418aa4 125 req = nfs_page_find_request_locked(NFS_I(inode), page);
587142f8 126 spin_unlock(&inode->i_lock);
277459d2
TM
127 return req;
128}
129
1da177e4
LT
130/* Adjust the file length if we're writing beyond the end */
131static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
132{
d56b4ddf 133 struct inode *inode = page_file_mapping(page)->host;
a3d01454
TM
134 loff_t end, i_size;
135 pgoff_t end_index;
1da177e4 136
a3d01454
TM
137 spin_lock(&inode->i_lock);
138 i_size = i_size_read(inode);
139 end_index = (i_size - 1) >> PAGE_CACHE_SHIFT;
d56b4ddf 140 if (i_size > 0 && page_file_index(page) < end_index)
a3d01454 141 goto out;
d56b4ddf 142 end = page_file_offset(page) + ((loff_t)offset+count);
1da177e4 143 if (i_size >= end)
a3d01454 144 goto out;
1da177e4 145 i_size_write(inode, end);
a3d01454
TM
146 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
147out:
148 spin_unlock(&inode->i_lock);
1da177e4
LT
149}
150
a301b777
TM
151/* A writeback failed: mark the page as bad, and invalidate the page cache */
152static void nfs_set_pageerror(struct page *page)
153{
d56b4ddf 154 nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page));
a301b777
TM
155}
156
d72ddcba
WAA
157/*
158 * nfs_page_group_search_locked
159 * @head - head request of page group
160 * @page_offset - offset into page
161 *
162 * Search page group with head @head to find a request that contains the
163 * page offset @page_offset.
164 *
165 * Returns a pointer to the first matching nfs request, or NULL if no
166 * match is found.
167 *
168 * Must be called with the page group lock held
169 */
170static struct nfs_page *
171nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
172{
173 struct nfs_page *req;
174
175 WARN_ON_ONCE(head != head->wb_head);
176 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_head->wb_flags));
177
178 req = head;
179 do {
180 if (page_offset >= req->wb_pgbase &&
181 page_offset < (req->wb_pgbase + req->wb_bytes))
182 return req;
183
184 req = req->wb_this_page;
185 } while (req != head);
186
187 return NULL;
188}
189
190/*
191 * nfs_page_group_covers_page
192 * @head - head request of page group
193 *
194 * Return true if the page group with head @head covers the whole page,
195 * returns false otherwise
196 */
197static bool nfs_page_group_covers_page(struct nfs_page *req)
198{
199 struct nfs_page *tmp;
200 unsigned int pos = 0;
201 unsigned int len = nfs_page_length(req->wb_page);
202
203 nfs_page_group_lock(req);
204
205 do {
206 tmp = nfs_page_group_search_locked(req->wb_head, pos);
207 if (tmp) {
208 /* no way this should happen */
209 WARN_ON_ONCE(tmp->wb_pgbase != pos);
210 pos += tmp->wb_bytes - (pos - tmp->wb_pgbase);
211 }
212 } while (tmp && pos < len);
213
214 nfs_page_group_unlock(req);
215 WARN_ON_ONCE(pos > len);
216 return pos == len;
217}
218
1da177e4
LT
219/* We can set the PG_uptodate flag if we see that a write request
220 * covers the full page.
221 */
d72ddcba 222static void nfs_mark_uptodate(struct nfs_page *req)
1da177e4 223{
d72ddcba 224 if (PageUptodate(req->wb_page))
1da177e4 225 return;
d72ddcba 226 if (!nfs_page_group_covers_page(req))
1da177e4 227 return;
d72ddcba 228 SetPageUptodate(req->wb_page);
1da177e4
LT
229}
230
1da177e4
LT
231static int wb_priority(struct writeback_control *wbc)
232{
233 if (wbc->for_reclaim)
c63c7b05 234 return FLUSH_HIGHPRI | FLUSH_STABLE;
b17621fe 235 if (wbc->for_kupdate || wbc->for_background)
b31268ac
TM
236 return FLUSH_LOWPRI | FLUSH_COND_STABLE;
237 return FLUSH_COND_STABLE;
1da177e4
LT
238}
239
89a09141
PZ
240/*
241 * NFS congestion control
242 */
243
244int nfs_congestion_kb;
245
246#define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))
247#define NFS_CONGESTION_OFF_THRESH \
248 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
249
deed85e7 250static void nfs_set_page_writeback(struct page *page)
89a09141 251{
deed85e7 252 struct nfs_server *nfss = NFS_SERVER(page_file_mapping(page)->host);
5a6d41b3
TM
253 int ret = test_set_page_writeback(page);
254
deed85e7 255 WARN_ON_ONCE(ret != 0);
89a09141 256
deed85e7
TM
257 if (atomic_long_inc_return(&nfss->writeback) >
258 NFS_CONGESTION_ON_THRESH) {
259 set_bdi_congested(&nfss->backing_dev_info,
260 BLK_RW_ASYNC);
89a09141
PZ
261 }
262}
263
20633f04 264static void nfs_end_page_writeback(struct nfs_page *req)
89a09141 265{
20633f04 266 struct inode *inode = page_file_mapping(req->wb_page)->host;
89a09141
PZ
267 struct nfs_server *nfss = NFS_SERVER(inode);
268
20633f04
WAA
269 if (!nfs_page_group_sync_on_bit(req, PG_WB_END))
270 return;
271
272 end_page_writeback(req->wb_page);
c4dc4bee 273 if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
8aa7e847 274 clear_bdi_congested(&nfss->backing_dev_info, BLK_RW_ASYNC);
89a09141
PZ
275}
276
cfb506e1 277static struct nfs_page *nfs_find_and_lock_request(struct page *page, bool nonblock)
e261f51f 278{
d56b4ddf 279 struct inode *inode = page_file_mapping(page)->host;
e261f51f 280 struct nfs_page *req;
e261f51f
TM
281 int ret;
282
587142f8 283 spin_lock(&inode->i_lock);
074cc1de 284 for (;;) {
29418aa4 285 req = nfs_page_find_request_locked(NFS_I(inode), page);
074cc1de
TM
286 if (req == NULL)
287 break;
7ad84aa9 288 if (nfs_lock_request(req))
e261f51f
TM
289 break;
290 /* Note: If we hold the page lock, as is the case in nfs_writepage,
7ad84aa9 291 * then the call to nfs_lock_request() will always
e261f51f
TM
292 * succeed provided that someone hasn't already marked the
293 * request as dirty (in which case we don't care).
294 */
587142f8 295 spin_unlock(&inode->i_lock);
cfb506e1
TM
296 if (!nonblock)
297 ret = nfs_wait_on_request(req);
298 else
299 ret = -EAGAIN;
e261f51f
TM
300 nfs_release_request(req);
301 if (ret != 0)
074cc1de 302 return ERR_PTR(ret);
587142f8 303 spin_lock(&inode->i_lock);
e261f51f 304 }
587142f8 305 spin_unlock(&inode->i_lock);
074cc1de
TM
306 return req;
307}
308
309/*
310 * Find an associated nfs write request, and prepare to flush it out
311 * May return an error if the user signalled nfs_wait_on_request().
312 */
313static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
cfb506e1 314 struct page *page, bool nonblock)
074cc1de
TM
315{
316 struct nfs_page *req;
317 int ret = 0;
318
cfb506e1 319 req = nfs_find_and_lock_request(page, nonblock);
074cc1de
TM
320 if (!req)
321 goto out;
322 ret = PTR_ERR(req);
323 if (IS_ERR(req))
324 goto out;
325
deed85e7
TM
326 nfs_set_page_writeback(page);
327 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
074cc1de 328
deed85e7 329 ret = 0;
f8512ad0
FI
330 if (!nfs_pageio_add_request(pgio, req)) {
331 nfs_redirty_request(req);
074cc1de 332 ret = pgio->pg_error;
f8512ad0 333 }
074cc1de
TM
334out:
335 return ret;
e261f51f
TM
336}
337
f758c885 338static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
1da177e4 339{
d56b4ddf 340 struct inode *inode = page_file_mapping(page)->host;
cfb506e1 341 int ret;
1da177e4 342
91d5b470
CL
343 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
344 nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
345
d56b4ddf 346 nfs_pageio_cond_complete(pgio, page_file_index(page));
1b430bee 347 ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE);
cfb506e1
TM
348 if (ret == -EAGAIN) {
349 redirty_page_for_writepage(wbc, page);
350 ret = 0;
351 }
352 return ret;
f758c885 353}
7fe7f848 354
f758c885
TM
355/*
356 * Write an mmapped page to the server.
357 */
358static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc)
359{
360 struct nfs_pageio_descriptor pgio;
361 int err;
49a70f27 362
a20c93e3
CH
363 nfs_pageio_init_write(&pgio, page->mapping->host, wb_priority(wbc),
364 false, &nfs_async_write_completion_ops);
f758c885
TM
365 err = nfs_do_writepage(page, wbc, &pgio);
366 nfs_pageio_complete(&pgio);
367 if (err < 0)
368 return err;
369 if (pgio.pg_error < 0)
370 return pgio.pg_error;
371 return 0;
4d770ccf
TM
372}
373
374int nfs_writepage(struct page *page, struct writeback_control *wbc)
375{
f758c885 376 int ret;
4d770ccf 377
f758c885 378 ret = nfs_writepage_locked(page, wbc);
1da177e4 379 unlock_page(page);
f758c885
TM
380 return ret;
381}
382
383static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
384{
385 int ret;
386
387 ret = nfs_do_writepage(page, wbc, data);
388 unlock_page(page);
389 return ret;
1da177e4
LT
390}
391
1da177e4
LT
392int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
393{
1da177e4 394 struct inode *inode = mapping->host;
72cb77f4 395 unsigned long *bitlock = &NFS_I(inode)->flags;
c63c7b05 396 struct nfs_pageio_descriptor pgio;
1da177e4
LT
397 int err;
398
72cb77f4
TM
399 /* Stop dirtying of new pages while we sync */
400 err = wait_on_bit_lock(bitlock, NFS_INO_FLUSHING,
401 nfs_wait_bit_killable, TASK_KILLABLE);
402 if (err)
403 goto out_err;
404
91d5b470
CL
405 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
406
a20c93e3
CH
407 nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
408 &nfs_async_write_completion_ops);
f758c885 409 err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
c63c7b05 410 nfs_pageio_complete(&pgio);
72cb77f4
TM
411
412 clear_bit_unlock(NFS_INO_FLUSHING, bitlock);
4e857c58 413 smp_mb__after_atomic();
72cb77f4
TM
414 wake_up_bit(bitlock, NFS_INO_FLUSHING);
415
f758c885 416 if (err < 0)
72cb77f4
TM
417 goto out_err;
418 err = pgio.pg_error;
419 if (err < 0)
420 goto out_err;
c63c7b05 421 return 0;
72cb77f4
TM
422out_err:
423 return err;
1da177e4
LT
424}
425
426/*
427 * Insert a write request into an inode
428 */
d6d6dc7c 429static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
1da177e4
LT
430{
431 struct nfs_inode *nfsi = NFS_I(inode);
e7d39069 432
2bfc6e56
WAA
433 WARN_ON_ONCE(req->wb_this_page != req);
434
e7d39069 435 /* Lock the request! */
7ad84aa9 436 nfs_lock_request(req);
e7d39069
TM
437
438 spin_lock(&inode->i_lock);
011e2a7f 439 if (!nfsi->npages && NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
a9a4a87a 440 inode->i_version++;
29418aa4
MG
441 /*
442 * Swap-space should not get truncated. Hence no need to plug the race
443 * with invalidate/truncate.
444 */
445 if (likely(!PageSwapCache(req->wb_page))) {
446 set_bit(PG_MAPPED, &req->wb_flags);
447 SetPagePrivate(req->wb_page);
448 set_page_private(req->wb_page, (unsigned long)req);
449 }
1da177e4 450 nfsi->npages++;
2bfc6e56 451 set_bit(PG_INODE_REF, &req->wb_flags);
c03b4024 452 kref_get(&req->wb_kref);
e7d39069 453 spin_unlock(&inode->i_lock);
1da177e4
LT
454}
455
456/*
89a09141 457 * Remove a write request from an inode
1da177e4
LT
458 */
459static void nfs_inode_remove_request(struct nfs_page *req)
460{
3d4ff43d 461 struct inode *inode = req->wb_context->dentry->d_inode;
1da177e4 462 struct nfs_inode *nfsi = NFS_I(inode);
20633f04 463 struct nfs_page *head;
1da177e4 464
20633f04
WAA
465 if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
466 head = req->wb_head;
467
468 spin_lock(&inode->i_lock);
469 if (likely(!PageSwapCache(head->wb_page))) {
470 set_page_private(head->wb_page, 0);
471 ClearPagePrivate(head->wb_page);
472 clear_bit(PG_MAPPED, &head->wb_flags);
473 }
474 nfsi->npages--;
475 spin_unlock(&inode->i_lock);
29418aa4 476 }
1da177e4
LT
477 nfs_release_request(req);
478}
479
61822ab5 480static void
6d884e8f 481nfs_mark_request_dirty(struct nfs_page *req)
61822ab5 482{
61822ab5
TM
483 __set_page_dirty_nobuffers(req->wb_page);
484}
485
89d77c8f 486#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
8dd37758
TM
487/**
488 * nfs_request_add_commit_list - add request to a commit list
489 * @req: pointer to a struct nfs_page
ea2cf228
FI
490 * @dst: commit list head
491 * @cinfo: holds list lock and accounting info
8dd37758 492 *
ea2cf228 493 * This sets the PG_CLEAN bit, updates the cinfo count of
8dd37758
TM
494 * number of outstanding requests requiring a commit as well as
495 * the MM page stats.
496 *
ea2cf228 497 * The caller must _not_ hold the cinfo->lock, but must be
8dd37758 498 * holding the nfs_page lock.
1da177e4 499 */
8dd37758 500void
ea2cf228
FI
501nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst,
502 struct nfs_commit_info *cinfo)
1da177e4 503{
e468bae9 504 set_bit(PG_CLEAN, &(req)->wb_flags);
ea2cf228
FI
505 spin_lock(cinfo->lock);
506 nfs_list_add_request(req, dst);
507 cinfo->mds->ncommit++;
508 spin_unlock(cinfo->lock);
56f9cd68
FI
509 if (!cinfo->dreq) {
510 inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
d56b4ddf 511 inc_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
56f9cd68
FI
512 BDI_RECLAIMABLE);
513 __mark_inode_dirty(req->wb_context->dentry->d_inode,
514 I_DIRTY_DATASYNC);
515 }
1da177e4 516}
8dd37758
TM
517EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);
518
519/**
520 * nfs_request_remove_commit_list - Remove request from a commit list
521 * @req: pointer to a nfs_page
ea2cf228 522 * @cinfo: holds list lock and accounting info
8dd37758 523 *
ea2cf228 524 * This clears the PG_CLEAN bit, and updates the cinfo's count of
8dd37758
TM
525 * number of outstanding requests requiring a commit
526 * It does not update the MM page stats.
527 *
ea2cf228 528 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
8dd37758
TM
529 */
530void
ea2cf228
FI
531nfs_request_remove_commit_list(struct nfs_page *req,
532 struct nfs_commit_info *cinfo)
8dd37758 533{
8dd37758
TM
534 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
535 return;
536 nfs_list_remove_request(req);
ea2cf228 537 cinfo->mds->ncommit--;
8dd37758
TM
538}
539EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);
540
ea2cf228
FI
541static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
542 struct inode *inode)
543{
544 cinfo->lock = &inode->i_lock;
545 cinfo->mds = &NFS_I(inode)->commit_info;
546 cinfo->ds = pnfs_get_ds_info(inode);
b359f9d0 547 cinfo->dreq = NULL;
f453a54a 548 cinfo->completion_ops = &nfs_commit_completion_ops;
ea2cf228
FI
549}
550
551void nfs_init_cinfo(struct nfs_commit_info *cinfo,
552 struct inode *inode,
553 struct nfs_direct_req *dreq)
554{
1763da12
FI
555 if (dreq)
556 nfs_init_cinfo_from_dreq(cinfo, dreq);
557 else
558 nfs_init_cinfo_from_inode(cinfo, inode);
ea2cf228
FI
559}
560EXPORT_SYMBOL_GPL(nfs_init_cinfo);
8dd37758
TM
561
562/*
563 * Add a request to the inode's commit list.
564 */
1763da12 565void
ea2cf228
FI
566nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
567 struct nfs_commit_info *cinfo)
8dd37758 568{
ea2cf228 569 if (pnfs_mark_request_commit(req, lseg, cinfo))
8dd37758 570 return;
ea2cf228 571 nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo);
8dd37758 572}
8e821cad 573
d6d6dc7c
FI
574static void
575nfs_clear_page_commit(struct page *page)
576{
577 dec_zone_page_state(page, NR_UNSTABLE_NFS);
d56b4ddf 578 dec_bdi_stat(page_file_mapping(page)->backing_dev_info, BDI_RECLAIMABLE);
d6d6dc7c
FI
579}
580
8dd37758 581static void
e468bae9
TM
582nfs_clear_request_commit(struct nfs_page *req)
583{
8dd37758
TM
584 if (test_bit(PG_CLEAN, &req->wb_flags)) {
585 struct inode *inode = req->wb_context->dentry->d_inode;
ea2cf228 586 struct nfs_commit_info cinfo;
e468bae9 587
ea2cf228
FI
588 nfs_init_cinfo_from_inode(&cinfo, inode);
589 if (!pnfs_clear_request_commit(req, &cinfo)) {
590 spin_lock(cinfo.lock);
591 nfs_request_remove_commit_list(req, &cinfo);
592 spin_unlock(cinfo.lock);
8dd37758 593 }
d6d6dc7c 594 nfs_clear_page_commit(req->wb_page);
e468bae9 595 }
e468bae9
TM
596}
597
8e821cad 598static inline
d45f60c6 599int nfs_write_need_commit(struct nfs_pgio_header *hdr)
8e821cad 600{
d45f60c6
WAA
601 if (hdr->writeverf.committed == NFS_DATA_SYNC)
602 return hdr->lseg == NULL;
603 return hdr->writeverf.committed != NFS_FILE_SYNC;
8e821cad
TM
604}
605
8e821cad 606#else
68cd6fa4
BS
607static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
608 struct inode *inode)
609{
610}
611
612void nfs_init_cinfo(struct nfs_commit_info *cinfo,
613 struct inode *inode,
614 struct nfs_direct_req *dreq)
615{
616}
617
1763da12 618void
ea2cf228
FI
619nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
620 struct nfs_commit_info *cinfo)
8e821cad
TM
621{
622}
623
8dd37758 624static void
e468bae9
TM
625nfs_clear_request_commit(struct nfs_page *req)
626{
e468bae9
TM
627}
628
8e821cad 629static inline
d45f60c6 630int nfs_write_need_commit(struct nfs_pgio_header *hdr)
8e821cad
TM
631{
632 return 0;
633}
634
6c75dc0d
FI
635#endif
636
061ae2ed 637static void nfs_write_completion(struct nfs_pgio_header *hdr)
8e821cad 638{
ea2cf228 639 struct nfs_commit_info cinfo;
6c75dc0d 640 unsigned long bytes = 0;
2bfc6e56 641 bool do_destroy;
6c75dc0d
FI
642
643 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
644 goto out;
ea2cf228 645 nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
6c75dc0d
FI
646 while (!list_empty(&hdr->pages)) {
647 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
6c75dc0d
FI
648
649 bytes += req->wb_bytes;
650 nfs_list_remove_request(req);
651 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
652 (hdr->good_bytes < bytes)) {
d1182b33 653 nfs_set_pageerror(req->wb_page);
6c75dc0d
FI
654 nfs_context_set_write_error(req->wb_context, hdr->error);
655 goto remove_req;
656 }
657 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) {
658 nfs_mark_request_dirty(req);
659 goto next;
660 }
661 if (test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) {
f79d06f5 662 memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
ea2cf228 663 nfs_mark_request_commit(req, hdr->lseg, &cinfo);
6c75dc0d
FI
664 goto next;
665 }
666remove_req:
667 nfs_inode_remove_request(req);
668next:
1d1afcbc 669 nfs_unlock_request(req);
20633f04 670 nfs_end_page_writeback(req);
2bfc6e56 671 do_destroy = !test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags);
3aff4ebb 672 nfs_release_request(req);
6c75dc0d
FI
673 }
674out:
675 hdr->release(hdr);
8e821cad 676}
1da177e4 677
89d77c8f 678#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
ce59515c 679unsigned long
ea2cf228 680nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
fb8a1f11 681{
ea2cf228 682 return cinfo->mds->ncommit;
d6d6dc7c
FI
683}
684
ea2cf228 685/* cinfo->lock held by caller */
1763da12 686int
ea2cf228
FI
687nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
688 struct nfs_commit_info *cinfo, int max)
d6d6dc7c
FI
689{
690 struct nfs_page *req, *tmp;
691 int ret = 0;
692
693 list_for_each_entry_safe(req, tmp, src, wb_list) {
8dd37758
TM
694 if (!nfs_lock_request(req))
695 continue;
7ad84aa9 696 kref_get(&req->wb_kref);
ea2cf228 697 if (cond_resched_lock(cinfo->lock))
3b3be88d 698 list_safe_reset_next(req, tmp, wb_list);
ea2cf228 699 nfs_request_remove_commit_list(req, cinfo);
8dd37758
TM
700 nfs_list_add_request(req, dst);
701 ret++;
1763da12 702 if ((ret == max) && !cinfo->dreq)
8dd37758 703 break;
d6d6dc7c
FI
704 }
705 return ret;
fb8a1f11
TM
706}
707
1da177e4
LT
708/*
709 * nfs_scan_commit - Scan an inode for commit requests
710 * @inode: NFS inode to scan
ea2cf228
FI
711 * @dst: mds destination list
712 * @cinfo: mds and ds lists of reqs ready to commit
1da177e4
LT
713 *
714 * Moves requests from the inode's 'commit' request list.
715 * The requests are *not* checked to ensure that they form a contiguous set.
716 */
1763da12 717int
ea2cf228
FI
718nfs_scan_commit(struct inode *inode, struct list_head *dst,
719 struct nfs_commit_info *cinfo)
1da177e4 720{
d6d6dc7c 721 int ret = 0;
fb8a1f11 722
ea2cf228
FI
723 spin_lock(cinfo->lock);
724 if (cinfo->mds->ncommit > 0) {
8dd37758 725 const int max = INT_MAX;
d6d6dc7c 726
ea2cf228
FI
727 ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
728 cinfo, max);
729 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
d6d6dc7c 730 }
ea2cf228 731 spin_unlock(cinfo->lock);
ff778d02 732 return ret;
1da177e4 733}
d6d6dc7c 734
c42de9dd 735#else
ce59515c 736unsigned long nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
fb8a1f11
TM
737{
738 return 0;
739}
740
1763da12
FI
741int nfs_scan_commit(struct inode *inode, struct list_head *dst,
742 struct nfs_commit_info *cinfo)
c42de9dd
TM
743{
744 return 0;
745}
1da177e4
LT
746#endif
747
1da177e4 748/*
e7d39069
TM
749 * Search for an existing write request, and attempt to update
750 * it to reflect a new dirty region on a given page.
1da177e4 751 *
e7d39069
TM
752 * If the attempt fails, then the existing request is flushed out
753 * to disk.
1da177e4 754 */
e7d39069
TM
755static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
756 struct page *page,
757 unsigned int offset,
758 unsigned int bytes)
1da177e4 759{
e7d39069
TM
760 struct nfs_page *req;
761 unsigned int rqend;
762 unsigned int end;
763 int error;
764
765 if (!PagePrivate(page))
766 return NULL;
1da177e4
LT
767
768 end = offset + bytes;
e7d39069 769 spin_lock(&inode->i_lock);
1da177e4 770
1da177e4 771 for (;;) {
29418aa4 772 req = nfs_page_find_request_locked(NFS_I(inode), page);
e7d39069
TM
773 if (req == NULL)
774 goto out_unlock;
775
2bfc6e56
WAA
776 /* should be handled by nfs_flush_incompatible */
777 WARN_ON_ONCE(req->wb_head != req);
778 WARN_ON_ONCE(req->wb_this_page != req);
779
e7d39069
TM
780 rqend = req->wb_offset + req->wb_bytes;
781 /*
782 * Tell the caller to flush out the request if
783 * the offsets are non-contiguous.
784 * Note: nfs_flush_incompatible() will already
785 * have flushed out requests having wrong owners.
786 */
e468bae9 787 if (offset > rqend
e7d39069
TM
788 || end < req->wb_offset)
789 goto out_flushme;
790
7ad84aa9 791 if (nfs_lock_request(req))
1da177e4 792 break;
1da177e4 793
e7d39069 794 /* The request is locked, so wait and then retry */
587142f8 795 spin_unlock(&inode->i_lock);
e7d39069
TM
796 error = nfs_wait_on_request(req);
797 nfs_release_request(req);
798 if (error != 0)
799 goto out_err;
800 spin_lock(&inode->i_lock);
1da177e4
LT
801 }
802
803 /* Okay, the request matches. Update the region */
804 if (offset < req->wb_offset) {
805 req->wb_offset = offset;
806 req->wb_pgbase = offset;
1da177e4 807 }
1da177e4
LT
808 if (end > rqend)
809 req->wb_bytes = end - req->wb_offset;
e7d39069
TM
810 else
811 req->wb_bytes = rqend - req->wb_offset;
812out_unlock:
813 spin_unlock(&inode->i_lock);
ca138f36
FI
814 if (req)
815 nfs_clear_request_commit(req);
e7d39069
TM
816 return req;
817out_flushme:
818 spin_unlock(&inode->i_lock);
819 nfs_release_request(req);
820 error = nfs_wb_page(inode, page);
821out_err:
822 return ERR_PTR(error);
823}
824
825/*
826 * Try to update an existing write request, or create one if there is none.
827 *
828 * Note: Should always be called with the Page Lock held to prevent races
829 * if we have to add a new request. Also assumes that the caller has
830 * already called nfs_flush_incompatible() if necessary.
831 */
832static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
833 struct page *page, unsigned int offset, unsigned int bytes)
834{
d56b4ddf 835 struct inode *inode = page_file_mapping(page)->host;
e7d39069 836 struct nfs_page *req;
1da177e4 837
e7d39069
TM
838 req = nfs_try_to_update_request(inode, page, offset, bytes);
839 if (req != NULL)
840 goto out;
2bfc6e56 841 req = nfs_create_request(ctx, page, NULL, offset, bytes);
e7d39069
TM
842 if (IS_ERR(req))
843 goto out;
d6d6dc7c 844 nfs_inode_add_request(inode, req);
efc91ed0 845out:
61e930a9 846 return req;
1da177e4
LT
847}
848
e7d39069
TM
849static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
850 unsigned int offset, unsigned int count)
851{
852 struct nfs_page *req;
853
854 req = nfs_setup_write_request(ctx, page, offset, count);
855 if (IS_ERR(req))
856 return PTR_ERR(req);
857 /* Update file length */
858 nfs_grow_file(page, offset, count);
d72ddcba 859 nfs_mark_uptodate(req);
a6305ddb 860 nfs_mark_request_dirty(req);
1d1afcbc 861 nfs_unlock_and_release_request(req);
e7d39069
TM
862 return 0;
863}
864
1da177e4
LT
865int nfs_flush_incompatible(struct file *file, struct page *page)
866{
cd3758e3 867 struct nfs_open_context *ctx = nfs_file_open_context(file);
2a369153 868 struct nfs_lock_context *l_ctx;
1da177e4 869 struct nfs_page *req;
1a54533e 870 int do_flush, status;
1da177e4
LT
871 /*
872 * Look for a request corresponding to this page. If there
873 * is one, and it belongs to another file, we flush it out
874 * before we try to copy anything into the page. Do this
875 * due to the lack of an ACCESS-type call in NFSv2.
876 * Also do the same if we find a request from an existing
877 * dropped page.
878 */
1a54533e
TM
879 do {
880 req = nfs_page_find_request(page);
881 if (req == NULL)
882 return 0;
2a369153
TM
883 l_ctx = req->wb_lock_context;
884 do_flush = req->wb_page != page || req->wb_context != ctx;
2bfc6e56
WAA
885 /* for now, flush if more than 1 request in page_group */
886 do_flush |= req->wb_this_page != req;
0f1d2605 887 if (l_ctx && ctx->dentry->d_inode->i_flock != NULL) {
2a369153
TM
888 do_flush |= l_ctx->lockowner.l_owner != current->files
889 || l_ctx->lockowner.l_pid != current->tgid;
890 }
1da177e4 891 nfs_release_request(req);
1a54533e
TM
892 if (!do_flush)
893 return 0;
d56b4ddf 894 status = nfs_wb_page(page_file_mapping(page)->host, page);
1a54533e
TM
895 } while (status == 0);
896 return status;
1da177e4
LT
897}
898
dc24826b
AA
899/*
900 * Avoid buffered writes when a open context credential's key would
901 * expire soon.
902 *
903 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
904 *
905 * Return 0 and set a credential flag which triggers the inode to flush
906 * and performs NFS_FILE_SYNC writes if the key will expired within
907 * RPC_KEY_EXPIRE_TIMEO.
908 */
909int
910nfs_key_timeout_notify(struct file *filp, struct inode *inode)
911{
912 struct nfs_open_context *ctx = nfs_file_open_context(filp);
913 struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
914
915 return rpcauth_key_timeout_notify(auth, ctx->cred);
916}
917
918/*
919 * Test if the open context credential key is marked to expire soon.
920 */
921bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx)
922{
923 return rpcauth_cred_key_to_expire(ctx->cred);
924}
925
5d47a356
TM
926/*
927 * If the page cache is marked as unsafe or invalid, then we can't rely on
928 * the PageUptodate() flag. In this case, we will need to turn off
929 * write optimisations that depend on the page contents being correct.
930 */
8d197a56 931static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
5d47a356 932{
d529ef83
JL
933 struct nfs_inode *nfsi = NFS_I(inode);
934
8d197a56
TM
935 if (nfs_have_delegated_attributes(inode))
936 goto out;
18dd78c4 937 if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
d529ef83 938 return false;
4db72b40 939 smp_rmb();
d529ef83 940 if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
8d197a56
TM
941 return false;
942out:
18dd78c4
SM
943 if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
944 return false;
8d197a56 945 return PageUptodate(page) != 0;
5d47a356
TM
946}
947
c7559663
SM
948/* If we know the page is up to date, and we're not using byte range locks (or
949 * if we have the whole file locked for writing), it may be more efficient to
950 * extend the write to cover the entire page in order to avoid fragmentation
951 * inefficiencies.
952 *
263b4509
SM
953 * If the file is opened for synchronous writes then we can just skip the rest
954 * of the checks.
c7559663
SM
955 */
956static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
957{
958 if (file->f_flags & O_DSYNC)
959 return 0;
263b4509
SM
960 if (!nfs_write_pageuptodate(page, inode))
961 return 0;
c7559663
SM
962 if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
963 return 1;
263b4509 964 if (inode->i_flock == NULL || (inode->i_flock->fl_start == 0 &&
c7559663 965 inode->i_flock->fl_end == OFFSET_MAX &&
263b4509 966 inode->i_flock->fl_type != F_RDLCK))
c7559663
SM
967 return 1;
968 return 0;
969}
970
1da177e4
LT
971/*
972 * Update and possibly write a cached page of an NFS file.
973 *
974 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
975 * things with a page scheduled for an RPC call (e.g. invalidate it).
976 */
977int nfs_updatepage(struct file *file, struct page *page,
978 unsigned int offset, unsigned int count)
979{
cd3758e3 980 struct nfs_open_context *ctx = nfs_file_open_context(file);
d56b4ddf 981 struct inode *inode = page_file_mapping(page)->host;
1da177e4
LT
982 int status = 0;
983
91d5b470
CL
984 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
985
6de1472f
AV
986 dprintk("NFS: nfs_updatepage(%pD2 %d@%lld)\n",
987 file, count, (long long)(page_file_offset(page) + offset));
1da177e4 988
c7559663 989 if (nfs_can_extend_write(file, page, inode)) {
49a70f27 990 count = max(count + offset, nfs_page_length(page));
1da177e4 991 offset = 0;
1da177e4
LT
992 }
993
e21195a7 994 status = nfs_writepage_setup(ctx, page, offset, count);
03fa9e84
TM
995 if (status < 0)
996 nfs_set_pageerror(page);
59b7c05f
TM
997 else
998 __set_page_dirty_nobuffers(page);
1da177e4 999
48186c7d 1000 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n",
1da177e4 1001 status, (long long)i_size_read(inode));
1da177e4
LT
1002 return status;
1003}
1004
3ff7576d 1005static int flush_task_priority(int how)
1da177e4
LT
1006{
1007 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
1008 case FLUSH_HIGHPRI:
1009 return RPC_PRIORITY_HIGH;
1010 case FLUSH_LOWPRI:
1011 return RPC_PRIORITY_LOW;
1012 }
1013 return RPC_PRIORITY_NORMAL;
1014}
1015
d45f60c6
WAA
1016static void nfs_initiate_write(struct nfs_pgio_header *hdr,
1017 struct rpc_message *msg,
1ed26f33 1018 struct rpc_task_setup *task_setup_data, int how)
1da177e4 1019{
d45f60c6 1020 struct inode *inode = hdr->inode;
3ff7576d 1021 int priority = flush_task_priority(how);
d138d5d1 1022
1ed26f33 1023 task_setup_data->priority = priority;
d45f60c6 1024 NFS_PROTO(inode)->write_setup(hdr, msg);
d138d5d1 1025
8c21c62c 1026 nfs4_state_protect_write(NFS_SERVER(inode)->nfs_client,
d45f60c6 1027 &task_setup_data->rpc_client, msg, hdr);
275acaaf
TM
1028}
1029
6d884e8f
F
1030/* If a nfs_flush_* function fails, it should remove reqs from @head and
1031 * call this on each, which will prepare them to be retried on next
1032 * writeback using standard nfs.
1033 */
1034static void nfs_redirty_request(struct nfs_page *req)
1035{
1036 nfs_mark_request_dirty(req);
1d1afcbc 1037 nfs_unlock_request(req);
20633f04 1038 nfs_end_page_writeback(req);
3aff4ebb 1039 nfs_release_request(req);
6d884e8f
F
1040}
1041
061ae2ed 1042static void nfs_async_write_error(struct list_head *head)
6c75dc0d
FI
1043{
1044 struct nfs_page *req;
1045
1046 while (!list_empty(head)) {
1047 req = nfs_list_entry(head->next);
1048 nfs_list_remove_request(req);
1049 nfs_redirty_request(req);
1050 }
1051}
1052
061ae2ed
FI
1053static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1054 .error_cleanup = nfs_async_write_error,
1055 .completion = nfs_write_completion,
1056};
1057
57208fa7 1058void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
a20c93e3 1059 struct inode *inode, int ioflags, bool force_mds,
061ae2ed 1060 const struct nfs_pgio_completion_ops *compl_ops)
1da177e4 1061{
a20c93e3 1062 struct nfs_server *server = NFS_SERVER(inode);
41d8d5b7 1063 const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
a20c93e3
CH
1064
1065#ifdef CONFIG_NFS_V4_1
1066 if (server->pnfs_curr_ld && !force_mds)
1067 pg_ops = server->pnfs_curr_ld->pg_write_ops;
1068#endif
4a0de55c
AS
1069 nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1070 server->wsize, ioflags);
1751c363 1071}
ddda8e0a 1072EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
1da177e4 1073
dce81290
TM
1074void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
1075{
41d8d5b7 1076 pgio->pg_ops = &nfs_pgio_rw_ops;
dce81290
TM
1077 pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1078}
1f945357 1079EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
dce81290 1080
1da177e4 1081
0b7c0153
FI
1082void nfs_commit_prepare(struct rpc_task *task, void *calldata)
1083{
1084 struct nfs_commit_data *data = calldata;
1085
1086 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
1087}
1088
d45f60c6 1089static void nfs_writeback_release_common(struct nfs_pgio_header *hdr)
c9d8f89d 1090{
d45f60c6 1091 int status = hdr->task.tk_status;
788e7a89 1092
d45f60c6 1093 if ((status >= 0) && nfs_write_need_commit(hdr)) {
6c75dc0d
FI
1094 spin_lock(&hdr->lock);
1095 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags))
1096 ; /* Do nothing */
1097 else if (!test_and_set_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags))
d45f60c6
WAA
1098 memcpy(&hdr->verf, &hdr->writeverf, sizeof(hdr->verf));
1099 else if (memcmp(&hdr->verf, &hdr->writeverf, sizeof(hdr->verf)))
6c75dc0d
FI
1100 set_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags);
1101 spin_unlock(&hdr->lock);
1da177e4
LT
1102 }
1103}
1104
1f2edbe3
TM
1105/*
1106 * Special version of should_remove_suid() that ignores capabilities.
1107 */
1108static int nfs_should_remove_suid(const struct inode *inode)
1109{
1110 umode_t mode = inode->i_mode;
1111 int kill = 0;
1112
1113 /* suid always must be killed */
1114 if (unlikely(mode & S_ISUID))
1115 kill = ATTR_KILL_SUID;
788e7a89 1116
1f2edbe3
TM
1117 /*
1118 * sgid without any exec bits is just a mandatory locking mark; leave
1119 * it alone. If some exec bits are set, it's a real sgid; kill it.
1120 */
1121 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1122 kill |= ATTR_KILL_SGID;
1123
1124 if (unlikely(kill && S_ISREG(mode)))
1125 return kill;
1126
1127 return 0;
1128}
788e7a89 1129
1da177e4
LT
1130/*
1131 * This function is called when the WRITE call is complete.
1132 */
d45f60c6
WAA
1133static int nfs_writeback_done(struct rpc_task *task,
1134 struct nfs_pgio_header *hdr,
0eecb214 1135 struct inode *inode)
1da177e4 1136{
788e7a89 1137 int status;
1da177e4 1138
f551e44f
CL
1139 /*
1140 * ->write_done will attempt to use post-op attributes to detect
1141 * conflicting writes by other clients. A strict interpretation
1142 * of close-to-open would allow us to continue caching even if
1143 * another writer had changed the file, but some applications
1144 * depend on tighter cache coherency when writing.
1145 */
d45f60c6 1146 status = NFS_PROTO(inode)->write_done(task, hdr);
788e7a89 1147 if (status != 0)
0eecb214 1148 return status;
d45f60c6 1149 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
91d5b470 1150
89d77c8f 1151#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
d45f60c6
WAA
1152 if (hdr->res.verf->committed < hdr->args.stable &&
1153 task->tk_status >= 0) {
1da177e4
LT
1154 /* We tried a write call, but the server did not
1155 * commit data to stable storage even though we
1156 * requested it.
1157 * Note: There is a known bug in Tru64 < 5.0 in which
1158 * the server reports NFS_DATA_SYNC, but performs
1159 * NFS_FILE_SYNC. We therefore implement this checking
1160 * as a dprintk() in order to avoid filling syslog.
1161 */
1162 static unsigned long complain;
1163
a69aef14 1164 /* Note this will print the MDS for a DS write */
1da177e4 1165 if (time_before(complain, jiffies)) {
48186c7d 1166 dprintk("NFS: faulty NFS server %s:"
1da177e4 1167 " (committed = %d) != (stable = %d)\n",
cd841605 1168 NFS_SERVER(inode)->nfs_client->cl_hostname,
d45f60c6 1169 hdr->res.verf->committed, hdr->args.stable);
1da177e4
LT
1170 complain = jiffies + 300 * HZ;
1171 }
1172 }
1173#endif
1f2edbe3
TM
1174
1175 /* Deal with the suid/sgid bit corner case */
1176 if (nfs_should_remove_suid(inode))
1177 nfs_mark_for_revalidate(inode);
0eecb214
AS
1178 return 0;
1179}
1180
1181/*
1182 * This function is called when the WRITE call is complete.
1183 */
d45f60c6
WAA
1184static void nfs_writeback_result(struct rpc_task *task,
1185 struct nfs_pgio_header *hdr)
0eecb214 1186{
d45f60c6
WAA
1187 struct nfs_pgio_args *argp = &hdr->args;
1188 struct nfs_pgio_res *resp = &hdr->res;
1f2edbe3
TM
1189
1190 if (resp->count < argp->count) {
1da177e4
LT
1191 static unsigned long complain;
1192
6c75dc0d 1193 /* This a short write! */
d45f60c6 1194 nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
91d5b470 1195
1da177e4 1196 /* Has the server at least made some progress? */
6c75dc0d
FI
1197 if (resp->count == 0) {
1198 if (time_before(complain, jiffies)) {
1199 printk(KERN_WARNING
1200 "NFS: Server wrote zero bytes, expected %u.\n",
1201 argp->count);
1202 complain = jiffies + 300 * HZ;
1da177e4 1203 }
d45f60c6 1204 nfs_set_pgio_error(hdr, -EIO, argp->offset);
6c75dc0d 1205 task->tk_status = -EIO;
13602896 1206 return;
1da177e4 1207 }
6c75dc0d
FI
1208 /* Was this an NFSv2 write or an NFSv3 stable write? */
1209 if (resp->verf->committed != NFS_UNSTABLE) {
1210 /* Resend from where the server left off */
d45f60c6 1211 hdr->mds_offset += resp->count;
6c75dc0d
FI
1212 argp->offset += resp->count;
1213 argp->pgbase += resp->count;
1214 argp->count -= resp->count;
1215 } else {
1216 /* Resend as a stable write in order to avoid
1217 * headaches in the case of a server crash.
1218 */
1219 argp->stable = NFS_FILE_SYNC;
1da177e4 1220 }
6c75dc0d 1221 rpc_restart_call_prepare(task);
1da177e4 1222 }
1da177e4
LT
1223}
1224
1225
89d77c8f 1226#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
71d0a611
TM
1227static int nfs_commit_set_lock(struct nfs_inode *nfsi, int may_wait)
1228{
b8413f98
TM
1229 int ret;
1230
71d0a611
TM
1231 if (!test_and_set_bit(NFS_INO_COMMIT, &nfsi->flags))
1232 return 1;
b8413f98
TM
1233 if (!may_wait)
1234 return 0;
1235 ret = out_of_line_wait_on_bit_lock(&nfsi->flags,
1236 NFS_INO_COMMIT,
1237 nfs_wait_bit_killable,
1238 TASK_KILLABLE);
1239 return (ret < 0) ? ret : 1;
71d0a611
TM
1240}
1241
f453a54a 1242static void nfs_commit_clear_lock(struct nfs_inode *nfsi)
71d0a611
TM
1243{
1244 clear_bit(NFS_INO_COMMIT, &nfsi->flags);
4e857c58 1245 smp_mb__after_atomic();
71d0a611
TM
1246 wake_up_bit(&nfsi->flags, NFS_INO_COMMIT);
1247}
1248
0b7c0153 1249void nfs_commitdata_release(struct nfs_commit_data *data)
1da177e4 1250{
0b7c0153
FI
1251 put_nfs_open_context(data->context);
1252 nfs_commit_free(data);
1da177e4 1253}
e0c2b380 1254EXPORT_SYMBOL_GPL(nfs_commitdata_release);
1da177e4 1255
0b7c0153 1256int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
9ace33cd 1257 const struct rpc_call_ops *call_ops,
9f0ec176 1258 int how, int flags)
1da177e4 1259{
07737691 1260 struct rpc_task *task;
9ace33cd 1261 int priority = flush_task_priority(how);
bdc7f021
TM
1262 struct rpc_message msg = {
1263 .rpc_argp = &data->args,
1264 .rpc_resp = &data->res,
9ace33cd 1265 .rpc_cred = data->cred,
bdc7f021 1266 };
84115e1c 1267 struct rpc_task_setup task_setup_data = {
07737691 1268 .task = &data->task,
9ace33cd 1269 .rpc_client = clnt,
bdc7f021 1270 .rpc_message = &msg,
9ace33cd 1271 .callback_ops = call_ops,
84115e1c 1272 .callback_data = data,
101070ca 1273 .workqueue = nfsiod_workqueue,
9f0ec176 1274 .flags = RPC_TASK_ASYNC | flags,
3ff7576d 1275 .priority = priority,
84115e1c 1276 };
9ace33cd
FI
1277 /* Set up the initial task struct. */
1278 NFS_PROTO(data->inode)->commit_setup(data, &msg);
1279
1280 dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
1281
8c21c62c
WAA
1282 nfs4_state_protect(NFS_SERVER(data->inode)->nfs_client,
1283 NFS_SP4_MACH_CRED_COMMIT, &task_setup_data.rpc_client, &msg);
1284
9ace33cd
FI
1285 task = rpc_run_task(&task_setup_data);
1286 if (IS_ERR(task))
1287 return PTR_ERR(task);
1288 if (how & FLUSH_SYNC)
1289 rpc_wait_for_completion_task(task);
1290 rpc_put_task(task);
1291 return 0;
1292}
e0c2b380 1293EXPORT_SYMBOL_GPL(nfs_initiate_commit);
9ace33cd
FI
1294
1295/*
1296 * Set up the argument/result storage required for the RPC call.
1297 */
0b7c0153 1298void nfs_init_commit(struct nfs_commit_data *data,
f453a54a
FI
1299 struct list_head *head,
1300 struct pnfs_layout_segment *lseg,
1301 struct nfs_commit_info *cinfo)
9ace33cd
FI
1302{
1303 struct nfs_page *first = nfs_list_entry(head->next);
3d4ff43d 1304 struct inode *inode = first->wb_context->dentry->d_inode;
1da177e4
LT
1305
1306 /* Set up the RPC argument and reply structs
1307 * NB: take care not to mess about with data->commit et al. */
1308
1309 list_splice_init(head, &data->pages);
1da177e4 1310
1da177e4 1311 data->inode = inode;
9ace33cd 1312 data->cred = first->wb_context->cred;
988b6dce 1313 data->lseg = lseg; /* reference transferred */
9ace33cd 1314 data->mds_ops = &nfs_commit_ops;
f453a54a 1315 data->completion_ops = cinfo->completion_ops;
b359f9d0 1316 data->dreq = cinfo->dreq;
1da177e4
LT
1317
1318 data->args.fh = NFS_FH(data->inode);
3da28eb1
TM
1319 /* Note: we always request a commit of the entire inode */
1320 data->args.offset = 0;
1321 data->args.count = 0;
0b7c0153 1322 data->context = get_nfs_open_context(first->wb_context);
1da177e4
LT
1323 data->res.fattr = &data->fattr;
1324 data->res.verf = &data->verf;
0e574af1 1325 nfs_fattr_init(&data->fattr);
1da177e4 1326}
e0c2b380 1327EXPORT_SYMBOL_GPL(nfs_init_commit);
1da177e4 1328
e0c2b380 1329void nfs_retry_commit(struct list_head *page_list,
ea2cf228
FI
1330 struct pnfs_layout_segment *lseg,
1331 struct nfs_commit_info *cinfo)
64bfeb49
FI
1332{
1333 struct nfs_page *req;
1334
1335 while (!list_empty(page_list)) {
1336 req = nfs_list_entry(page_list->next);
1337 nfs_list_remove_request(req);
ea2cf228 1338 nfs_mark_request_commit(req, lseg, cinfo);
56f9cd68
FI
1339 if (!cinfo->dreq) {
1340 dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
d56b4ddf 1341 dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
56f9cd68
FI
1342 BDI_RECLAIMABLE);
1343 }
1d1afcbc 1344 nfs_unlock_and_release_request(req);
64bfeb49
FI
1345 }
1346}
e0c2b380 1347EXPORT_SYMBOL_GPL(nfs_retry_commit);
64bfeb49 1348
1da177e4
LT
1349/*
1350 * Commit dirty pages
1351 */
1352static int
ea2cf228
FI
1353nfs_commit_list(struct inode *inode, struct list_head *head, int how,
1354 struct nfs_commit_info *cinfo)
1da177e4 1355{
0b7c0153 1356 struct nfs_commit_data *data;
1da177e4 1357
c9d8f89d 1358 data = nfs_commitdata_alloc();
1da177e4
LT
1359
1360 if (!data)
1361 goto out_bad;
1362
1363 /* Set up the argument struct */
f453a54a
FI
1364 nfs_init_commit(data, head, NULL, cinfo);
1365 atomic_inc(&cinfo->mds->rpcs_out);
9f0ec176
AA
1366 return nfs_initiate_commit(NFS_CLIENT(inode), data, data->mds_ops,
1367 how, 0);
1da177e4 1368 out_bad:
ea2cf228 1369 nfs_retry_commit(head, NULL, cinfo);
f453a54a 1370 cinfo->completion_ops->error_cleanup(NFS_I(inode));
1da177e4
LT
1371 return -ENOMEM;
1372}
1373
1374/*
1375 * COMMIT call returned
1376 */
788e7a89 1377static void nfs_commit_done(struct rpc_task *task, void *calldata)
1da177e4 1378{
0b7c0153 1379 struct nfs_commit_data *data = calldata;
1da177e4 1380
a3f565b1 1381 dprintk("NFS: %5u nfs_commit_done (status %d)\n",
1da177e4
LT
1382 task->tk_pid, task->tk_status);
1383
788e7a89 1384 /* Call the NFS version-specific code */
c0d0e96b 1385 NFS_PROTO(data->inode)->commit_done(task, data);
c9d8f89d
TM
1386}
1387
f453a54a 1388static void nfs_commit_release_pages(struct nfs_commit_data *data)
c9d8f89d 1389{
5917ce84 1390 struct nfs_page *req;
c9d8f89d 1391 int status = data->task.tk_status;
f453a54a 1392 struct nfs_commit_info cinfo;
788e7a89 1393
1da177e4
LT
1394 while (!list_empty(&data->pages)) {
1395 req = nfs_list_entry(data->pages.next);
1396 nfs_list_remove_request(req);
d6d6dc7c 1397 nfs_clear_page_commit(req->wb_page);
1da177e4 1398
1e8968c5 1399 dprintk("NFS: commit (%s/%llu %d@%lld)",
3d4ff43d 1400 req->wb_context->dentry->d_sb->s_id,
1e8968c5 1401 (unsigned long long)NFS_FILEID(req->wb_context->dentry->d_inode),
1da177e4
LT
1402 req->wb_bytes,
1403 (long long)req_offset(req));
c9d8f89d
TM
1404 if (status < 0) {
1405 nfs_context_set_write_error(req->wb_context, status);
1da177e4 1406 nfs_inode_remove_request(req);
c9d8f89d 1407 dprintk(", error = %d\n", status);
1da177e4
LT
1408 goto next;
1409 }
1410
1411 /* Okay, COMMIT succeeded, apparently. Check the verifier
1412 * returned by the server against all stored verfs. */
2f2c63bc 1413 if (!memcmp(&req->wb_verf, &data->verf.verifier, sizeof(req->wb_verf))) {
1da177e4
LT
1414 /* We have a match */
1415 nfs_inode_remove_request(req);
1416 dprintk(" OK\n");
1417 goto next;
1418 }
1419 /* We have a mismatch. Write the page again */
1420 dprintk(" mismatch\n");
6d884e8f 1421 nfs_mark_request_dirty(req);
05990d1b 1422 set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1da177e4 1423 next:
1d1afcbc 1424 nfs_unlock_and_release_request(req);
1da177e4 1425 }
f453a54a
FI
1426 nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1427 if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
1428 nfs_commit_clear_lock(NFS_I(data->inode));
5917ce84
FI
1429}
1430
1431static void nfs_commit_release(void *calldata)
1432{
0b7c0153 1433 struct nfs_commit_data *data = calldata;
5917ce84 1434
f453a54a 1435 data->completion_ops->completion(data);
c9d8f89d 1436 nfs_commitdata_release(calldata);
1da177e4 1437}
788e7a89
TM
1438
1439static const struct rpc_call_ops nfs_commit_ops = {
0b7c0153 1440 .rpc_call_prepare = nfs_commit_prepare,
788e7a89
TM
1441 .rpc_call_done = nfs_commit_done,
1442 .rpc_release = nfs_commit_release,
1443};
1da177e4 1444
f453a54a
FI
1445static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
1446 .completion = nfs_commit_release_pages,
1447 .error_cleanup = nfs_commit_clear_lock,
1448};
1449
1763da12
FI
1450int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
1451 int how, struct nfs_commit_info *cinfo)
84c53ab5
FI
1452{
1453 int status;
1454
ea2cf228 1455 status = pnfs_commit_list(inode, head, how, cinfo);
84c53ab5 1456 if (status == PNFS_NOT_ATTEMPTED)
ea2cf228 1457 status = nfs_commit_list(inode, head, how, cinfo);
84c53ab5
FI
1458 return status;
1459}
1460
b608b283 1461int nfs_commit_inode(struct inode *inode, int how)
1da177e4 1462{
1da177e4 1463 LIST_HEAD(head);
ea2cf228 1464 struct nfs_commit_info cinfo;
71d0a611 1465 int may_wait = how & FLUSH_SYNC;
b8413f98 1466 int res;
1da177e4 1467
b8413f98
TM
1468 res = nfs_commit_set_lock(NFS_I(inode), may_wait);
1469 if (res <= 0)
c5efa5fc 1470 goto out_mark_dirty;
ea2cf228
FI
1471 nfs_init_cinfo_from_inode(&cinfo, inode);
1472 res = nfs_scan_commit(inode, &head, &cinfo);
1da177e4 1473 if (res) {
a861a1e1
FI
1474 int error;
1475
ea2cf228 1476 error = nfs_generic_commit_list(inode, &head, how, &cinfo);
3da28eb1
TM
1477 if (error < 0)
1478 return error;
b8413f98 1479 if (!may_wait)
c5efa5fc 1480 goto out_mark_dirty;
b8413f98
TM
1481 error = wait_on_bit(&NFS_I(inode)->flags,
1482 NFS_INO_COMMIT,
1483 nfs_wait_bit_killable,
1484 TASK_KILLABLE);
1485 if (error < 0)
1486 return error;
71d0a611
TM
1487 } else
1488 nfs_commit_clear_lock(NFS_I(inode));
c5efa5fc
TM
1489 return res;
1490 /* Note: If we exit without ensuring that the commit is complete,
1491 * we must mark the inode as dirty. Otherwise, future calls to
1492 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure
1493 * that the data is on the disk.
1494 */
1495out_mark_dirty:
1496 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1da177e4
LT
1497 return res;
1498}
8fc795f7
TM
1499
1500static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc)
1501{
420e3646
TM
1502 struct nfs_inode *nfsi = NFS_I(inode);
1503 int flags = FLUSH_SYNC;
1504 int ret = 0;
8fc795f7 1505
3236c3e1 1506 /* no commits means nothing needs to be done */
ea2cf228 1507 if (!nfsi->commit_info.ncommit)
3236c3e1
JL
1508 return ret;
1509
a00dd6c0
JL
1510 if (wbc->sync_mode == WB_SYNC_NONE) {
1511 /* Don't commit yet if this is a non-blocking flush and there
1512 * are a lot of outstanding writes for this mapping.
1513 */
ea2cf228 1514 if (nfsi->commit_info.ncommit <= (nfsi->npages >> 1))
a00dd6c0 1515 goto out_mark_dirty;
420e3646 1516
a00dd6c0 1517 /* don't wait for the COMMIT response */
420e3646 1518 flags = 0;
a00dd6c0
JL
1519 }
1520
420e3646
TM
1521 ret = nfs_commit_inode(inode, flags);
1522 if (ret >= 0) {
1523 if (wbc->sync_mode == WB_SYNC_NONE) {
1524 if (ret < wbc->nr_to_write)
1525 wbc->nr_to_write -= ret;
1526 else
1527 wbc->nr_to_write = 0;
1528 }
8fc795f7 1529 return 0;
420e3646
TM
1530 }
1531out_mark_dirty:
8fc795f7
TM
1532 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1533 return ret;
1534}
c63c7b05 1535#else
8fc795f7
TM
1536static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc)
1537{
1538 return 0;
1539}
1da177e4
LT
1540#endif
1541
8fc795f7
TM
1542int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1543{
a8d8f02c
BS
1544 return nfs_commit_unstable_pages(inode, wbc);
1545}
89d77c8f 1546EXPORT_SYMBOL_GPL(nfs_write_inode);
a8d8f02c 1547
acdc53b2
TM
1548/*
1549 * flush the inode to disk.
1550 */
1551int nfs_wb_all(struct inode *inode)
34901f70
TM
1552{
1553 struct writeback_control wbc = {
72cb77f4 1554 .sync_mode = WB_SYNC_ALL,
34901f70 1555 .nr_to_write = LONG_MAX,
d7fb1207
TM
1556 .range_start = 0,
1557 .range_end = LLONG_MAX,
34901f70 1558 };
f4ce1299
TM
1559 int ret;
1560
1561 trace_nfs_writeback_inode_enter(inode);
1562
1563 ret = sync_inode(inode, &wbc);
34901f70 1564
f4ce1299
TM
1565 trace_nfs_writeback_inode_exit(inode, ret);
1566 return ret;
1c75950b 1567}
ddda8e0a 1568EXPORT_SYMBOL_GPL(nfs_wb_all);
1c75950b 1569
1b3b4a1a
TM
1570int nfs_wb_page_cancel(struct inode *inode, struct page *page)
1571{
1572 struct nfs_page *req;
1b3b4a1a
TM
1573 int ret = 0;
1574
1b3b4a1a 1575 for (;;) {
ba8b06e6 1576 wait_on_page_writeback(page);
1b3b4a1a
TM
1577 req = nfs_page_find_request(page);
1578 if (req == NULL)
1b3b4a1a 1579 break;
7ad84aa9 1580 if (nfs_lock_request(req)) {
8dd37758 1581 nfs_clear_request_commit(req);
1b3b4a1a
TM
1582 nfs_inode_remove_request(req);
1583 /*
1584 * In case nfs_inode_remove_request has marked the
1585 * page as being dirty
1586 */
1587 cancel_dirty_page(page, PAGE_CACHE_SIZE);
1d1afcbc 1588 nfs_unlock_and_release_request(req);
1b3b4a1a
TM
1589 break;
1590 }
1591 ret = nfs_wait_on_request(req);
c9edda71 1592 nfs_release_request(req);
1b3b4a1a 1593 if (ret < 0)
c988950e 1594 break;
1b3b4a1a 1595 }
1b3b4a1a
TM
1596 return ret;
1597}
1598
7f2f12d9
TM
1599/*
1600 * Write back all requests on one page - we do this before reading it.
1601 */
1602int nfs_wb_page(struct inode *inode, struct page *page)
1c75950b 1603{
29418aa4 1604 loff_t range_start = page_file_offset(page);
1c75950b 1605 loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
4d770ccf 1606 struct writeback_control wbc = {
4d770ccf 1607 .sync_mode = WB_SYNC_ALL,
7f2f12d9 1608 .nr_to_write = 0,
4d770ccf
TM
1609 .range_start = range_start,
1610 .range_end = range_end,
1611 };
1612 int ret;
1c75950b 1613
f4ce1299
TM
1614 trace_nfs_writeback_page_enter(inode);
1615
0522f6ad 1616 for (;;) {
ba8b06e6 1617 wait_on_page_writeback(page);
73e3302f
TM
1618 if (clear_page_dirty_for_io(page)) {
1619 ret = nfs_writepage_locked(page, &wbc);
1620 if (ret < 0)
1621 goto out_error;
0522f6ad 1622 continue;
7f2f12d9 1623 }
f4ce1299 1624 ret = 0;
0522f6ad
TM
1625 if (!PagePrivate(page))
1626 break;
1627 ret = nfs_commit_inode(inode, FLUSH_SYNC);
ba8b06e6 1628 if (ret < 0)
73e3302f 1629 goto out_error;
7f2f12d9 1630 }
73e3302f 1631out_error:
f4ce1299 1632 trace_nfs_writeback_page_exit(inode, ret);
4d770ccf 1633 return ret;
1c75950b
TM
1634}
1635
074cc1de
TM
1636#ifdef CONFIG_MIGRATION
1637int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
a6bc32b8 1638 struct page *page, enum migrate_mode mode)
074cc1de 1639{
2da95652
JL
1640 /*
1641 * If PagePrivate is set, then the page is currently associated with
1642 * an in-progress read or write request. Don't try to migrate it.
1643 *
1644 * FIXME: we could do this in principle, but we'll need a way to ensure
1645 * that we can safely release the inode reference while holding
1646 * the page lock.
1647 */
1648 if (PagePrivate(page))
1649 return -EBUSY;
074cc1de 1650
8c209ce7
DH
1651 if (!nfs_fscache_release_page(page, GFP_KERNEL))
1652 return -EBUSY;
074cc1de 1653
a6bc32b8 1654 return migrate_page(mapping, newpage, page, mode);
074cc1de
TM
1655}
1656#endif
1657
f7b422b1 1658int __init nfs_init_writepagecache(void)
1da177e4
LT
1659{
1660 nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
1e7f3a48 1661 sizeof(struct nfs_pgio_header),
1da177e4 1662 0, SLAB_HWCACHE_ALIGN,
20c2df83 1663 NULL);
1da177e4
LT
1664 if (nfs_wdata_cachep == NULL)
1665 return -ENOMEM;
1666
93d2341c
MD
1667 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
1668 nfs_wdata_cachep);
1da177e4 1669 if (nfs_wdata_mempool == NULL)
3dd4765f 1670 goto out_destroy_write_cache;
1da177e4 1671
0b7c0153
FI
1672 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
1673 sizeof(struct nfs_commit_data),
1674 0, SLAB_HWCACHE_ALIGN,
1675 NULL);
1676 if (nfs_cdata_cachep == NULL)
3dd4765f 1677 goto out_destroy_write_mempool;
0b7c0153 1678
93d2341c 1679 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
4c100210 1680 nfs_cdata_cachep);
1da177e4 1681 if (nfs_commit_mempool == NULL)
3dd4765f 1682 goto out_destroy_commit_cache;
1da177e4 1683
89a09141
PZ
1684 /*
1685 * NFS congestion size, scale with available memory.
1686 *
1687 * 64MB: 8192k
1688 * 128MB: 11585k
1689 * 256MB: 16384k
1690 * 512MB: 23170k
1691 * 1GB: 32768k
1692 * 2GB: 46340k
1693 * 4GB: 65536k
1694 * 8GB: 92681k
1695 * 16GB: 131072k
1696 *
1697 * This allows larger machines to have larger/more transfers.
1698 * Limit the default to 256M
1699 */
1700 nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
1701 if (nfs_congestion_kb > 256*1024)
1702 nfs_congestion_kb = 256*1024;
1703
1da177e4 1704 return 0;
3dd4765f
JL
1705
1706out_destroy_commit_cache:
1707 kmem_cache_destroy(nfs_cdata_cachep);
1708out_destroy_write_mempool:
1709 mempool_destroy(nfs_wdata_mempool);
1710out_destroy_write_cache:
1711 kmem_cache_destroy(nfs_wdata_cachep);
1712 return -ENOMEM;
1da177e4
LT
1713}
1714
266bee88 1715void nfs_destroy_writepagecache(void)
1da177e4
LT
1716{
1717 mempool_destroy(nfs_commit_mempool);
3dd4765f 1718 kmem_cache_destroy(nfs_cdata_cachep);
1da177e4 1719 mempool_destroy(nfs_wdata_mempool);
1a1d92c1 1720 kmem_cache_destroy(nfs_wdata_cachep);
1da177e4
LT
1721}
1722
4a0de55c 1723static const struct nfs_rw_ops nfs_rw_write_ops = {
a4cdda59 1724 .rw_mode = FMODE_WRITE,
4a0de55c
AS
1725 .rw_alloc_header = nfs_writehdr_alloc,
1726 .rw_free_header = nfs_writehdr_free,
a4cdda59 1727 .rw_release = nfs_writeback_release_common,
0eecb214
AS
1728 .rw_done = nfs_writeback_done,
1729 .rw_result = nfs_writeback_result,
1ed26f33 1730 .rw_initiate = nfs_initiate_write,
4a0de55c 1731};