]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - fs/nfs/write.c
nfs: nfs_page should take a ref on the head req
[mirror_ubuntu-bionic-kernel.git] / fs / nfs / write.c
CommitLineData
1da177e4
LT
1/*
2 * linux/fs/nfs/write.c
3 *
7c85d900 4 * Write file data over NFS.
1da177e4
LT
5 *
6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
7 */
8
1da177e4
LT
9#include <linux/types.h>
10#include <linux/slab.h>
11#include <linux/mm.h>
12#include <linux/pagemap.h>
13#include <linux/file.h>
1da177e4 14#include <linux/writeback.h>
89a09141 15#include <linux/swap.h>
074cc1de 16#include <linux/migrate.h>
1da177e4
LT
17
18#include <linux/sunrpc/clnt.h>
19#include <linux/nfs_fs.h>
20#include <linux/nfs_mount.h>
21#include <linux/nfs_page.h>
3fcfab16 22#include <linux/backing-dev.h>
afeacc8c 23#include <linux/export.h>
3fcfab16 24
1da177e4 25#include <asm/uaccess.h>
1da177e4
LT
26
27#include "delegation.h"
49a70f27 28#include "internal.h"
91d5b470 29#include "iostat.h"
def6ed7e 30#include "nfs4_fs.h"
074cc1de 31#include "fscache.h"
94ad1c80 32#include "pnfs.h"
1da177e4 33
f4ce1299
TM
34#include "nfstrace.h"
35
1da177e4
LT
36#define NFSDBG_FACILITY NFSDBG_PAGECACHE
37
38#define MIN_POOL_WRITE (32)
39#define MIN_POOL_COMMIT (4)
40
41/*
42 * Local function declarations
43 */
f8512ad0 44static void nfs_redirty_request(struct nfs_page *req);
788e7a89 45static const struct rpc_call_ops nfs_commit_ops;
061ae2ed 46static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
f453a54a 47static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
4a0de55c 48static const struct nfs_rw_ops nfs_rw_write_ops;
1da177e4 49
e18b890b 50static struct kmem_cache *nfs_wdata_cachep;
3feb2d49 51static mempool_t *nfs_wdata_mempool;
0b7c0153 52static struct kmem_cache *nfs_cdata_cachep;
1da177e4
LT
53static mempool_t *nfs_commit_mempool;
54
0b7c0153 55struct nfs_commit_data *nfs_commitdata_alloc(void)
1da177e4 56{
192e501b 57 struct nfs_commit_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
40859d7e 58
1da177e4
LT
59 if (p) {
60 memset(p, 0, sizeof(*p));
61 INIT_LIST_HEAD(&p->pages);
62 }
63 return p;
64}
e0c2b380 65EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
1da177e4 66
0b7c0153 67void nfs_commit_free(struct nfs_commit_data *p)
1da177e4
LT
68{
69 mempool_free(p, nfs_commit_mempool);
70}
e0c2b380 71EXPORT_SYMBOL_GPL(nfs_commit_free);
1da177e4 72
4a0de55c 73static struct nfs_rw_header *nfs_writehdr_alloc(void)
3feb2d49 74{
c0752cdf 75 struct nfs_rw_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
cd841605 76
4a0de55c 77 if (p)
3feb2d49 78 memset(p, 0, sizeof(*p));
3feb2d49
TM
79 return p;
80}
6c75dc0d 81
4a0de55c 82static void nfs_writehdr_free(struct nfs_rw_header *whdr)
3feb2d49 83{
cd841605 84 mempool_free(whdr, nfs_wdata_mempool);
3feb2d49 85}
1da177e4 86
7b159fc1
TM
87static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
88{
89 ctx->error = error;
90 smp_wmb();
91 set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
92}
93
29418aa4
MG
94static struct nfs_page *
95nfs_page_find_request_locked(struct nfs_inode *nfsi, struct page *page)
277459d2
TM
96{
97 struct nfs_page *req = NULL;
98
29418aa4 99 if (PagePrivate(page))
277459d2 100 req = (struct nfs_page *)page_private(page);
29418aa4
MG
101 else if (unlikely(PageSwapCache(page))) {
102 struct nfs_page *freq, *t;
103
104 /* Linearly search the commit list for the correct req */
105 list_for_each_entry_safe(freq, t, &nfsi->commit_info.list, wb_list) {
106 if (freq->wb_page == page) {
107 req = freq;
108 break;
109 }
110 }
277459d2 111 }
29418aa4
MG
112
113 if (req)
114 kref_get(&req->wb_kref);
115
277459d2
TM
116 return req;
117}
118
119static struct nfs_page *nfs_page_find_request(struct page *page)
120{
d56b4ddf 121 struct inode *inode = page_file_mapping(page)->host;
277459d2 122 struct nfs_page *req = NULL;
277459d2 123
587142f8 124 spin_lock(&inode->i_lock);
29418aa4 125 req = nfs_page_find_request_locked(NFS_I(inode), page);
587142f8 126 spin_unlock(&inode->i_lock);
277459d2
TM
127 return req;
128}
129
1da177e4
LT
130/* Adjust the file length if we're writing beyond the end */
131static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
132{
d56b4ddf 133 struct inode *inode = page_file_mapping(page)->host;
a3d01454
TM
134 loff_t end, i_size;
135 pgoff_t end_index;
1da177e4 136
a3d01454
TM
137 spin_lock(&inode->i_lock);
138 i_size = i_size_read(inode);
139 end_index = (i_size - 1) >> PAGE_CACHE_SHIFT;
d56b4ddf 140 if (i_size > 0 && page_file_index(page) < end_index)
a3d01454 141 goto out;
d56b4ddf 142 end = page_file_offset(page) + ((loff_t)offset+count);
1da177e4 143 if (i_size >= end)
a3d01454 144 goto out;
1da177e4 145 i_size_write(inode, end);
a3d01454
TM
146 nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
147out:
148 spin_unlock(&inode->i_lock);
1da177e4
LT
149}
150
a301b777
TM
151/* A writeback failed: mark the page as bad, and invalidate the page cache */
152static void nfs_set_pageerror(struct page *page)
153{
d56b4ddf 154 nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page));
a301b777
TM
155}
156
d72ddcba
WAA
157/*
158 * nfs_page_group_search_locked
159 * @head - head request of page group
160 * @page_offset - offset into page
161 *
162 * Search page group with head @head to find a request that contains the
163 * page offset @page_offset.
164 *
165 * Returns a pointer to the first matching nfs request, or NULL if no
166 * match is found.
167 *
168 * Must be called with the page group lock held
169 */
170static struct nfs_page *
171nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
172{
173 struct nfs_page *req;
174
175 WARN_ON_ONCE(head != head->wb_head);
176 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_head->wb_flags));
177
178 req = head;
179 do {
180 if (page_offset >= req->wb_pgbase &&
181 page_offset < (req->wb_pgbase + req->wb_bytes))
182 return req;
183
184 req = req->wb_this_page;
185 } while (req != head);
186
187 return NULL;
188}
189
190/*
191 * nfs_page_group_covers_page
192 * @head - head request of page group
193 *
194 * Return true if the page group with head @head covers the whole page,
195 * returns false otherwise
196 */
197static bool nfs_page_group_covers_page(struct nfs_page *req)
198{
199 struct nfs_page *tmp;
200 unsigned int pos = 0;
201 unsigned int len = nfs_page_length(req->wb_page);
202
203 nfs_page_group_lock(req);
204
205 do {
206 tmp = nfs_page_group_search_locked(req->wb_head, pos);
207 if (tmp) {
208 /* no way this should happen */
209 WARN_ON_ONCE(tmp->wb_pgbase != pos);
210 pos += tmp->wb_bytes - (pos - tmp->wb_pgbase);
211 }
212 } while (tmp && pos < len);
213
214 nfs_page_group_unlock(req);
215 WARN_ON_ONCE(pos > len);
216 return pos == len;
217}
218
1da177e4
LT
219/* We can set the PG_uptodate flag if we see that a write request
220 * covers the full page.
221 */
d72ddcba 222static void nfs_mark_uptodate(struct nfs_page *req)
1da177e4 223{
d72ddcba 224 if (PageUptodate(req->wb_page))
1da177e4 225 return;
d72ddcba 226 if (!nfs_page_group_covers_page(req))
1da177e4 227 return;
d72ddcba 228 SetPageUptodate(req->wb_page);
1da177e4
LT
229}
230
1da177e4
LT
231static int wb_priority(struct writeback_control *wbc)
232{
233 if (wbc->for_reclaim)
c63c7b05 234 return FLUSH_HIGHPRI | FLUSH_STABLE;
b17621fe 235 if (wbc->for_kupdate || wbc->for_background)
b31268ac
TM
236 return FLUSH_LOWPRI | FLUSH_COND_STABLE;
237 return FLUSH_COND_STABLE;
1da177e4
LT
238}
239
89a09141
PZ
240/*
241 * NFS congestion control
242 */
243
244int nfs_congestion_kb;
245
246#define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))
247#define NFS_CONGESTION_OFF_THRESH \
248 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
249
deed85e7 250static void nfs_set_page_writeback(struct page *page)
89a09141 251{
deed85e7 252 struct nfs_server *nfss = NFS_SERVER(page_file_mapping(page)->host);
5a6d41b3
TM
253 int ret = test_set_page_writeback(page);
254
deed85e7 255 WARN_ON_ONCE(ret != 0);
89a09141 256
deed85e7
TM
257 if (atomic_long_inc_return(&nfss->writeback) >
258 NFS_CONGESTION_ON_THRESH) {
259 set_bdi_congested(&nfss->backing_dev_info,
260 BLK_RW_ASYNC);
89a09141
PZ
261 }
262}
263
20633f04 264static void nfs_end_page_writeback(struct nfs_page *req)
89a09141 265{
20633f04 266 struct inode *inode = page_file_mapping(req->wb_page)->host;
89a09141
PZ
267 struct nfs_server *nfss = NFS_SERVER(inode);
268
20633f04
WAA
269 if (!nfs_page_group_sync_on_bit(req, PG_WB_END))
270 return;
271
272 end_page_writeback(req->wb_page);
c4dc4bee 273 if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
8aa7e847 274 clear_bdi_congested(&nfss->backing_dev_info, BLK_RW_ASYNC);
89a09141
PZ
275}
276
cfb506e1 277static struct nfs_page *nfs_find_and_lock_request(struct page *page, bool nonblock)
e261f51f 278{
d56b4ddf 279 struct inode *inode = page_file_mapping(page)->host;
e261f51f 280 struct nfs_page *req;
e261f51f
TM
281 int ret;
282
587142f8 283 spin_lock(&inode->i_lock);
074cc1de 284 for (;;) {
29418aa4 285 req = nfs_page_find_request_locked(NFS_I(inode), page);
074cc1de
TM
286 if (req == NULL)
287 break;
7ad84aa9 288 if (nfs_lock_request(req))
e261f51f
TM
289 break;
290 /* Note: If we hold the page lock, as is the case in nfs_writepage,
7ad84aa9 291 * then the call to nfs_lock_request() will always
e261f51f
TM
292 * succeed provided that someone hasn't already marked the
293 * request as dirty (in which case we don't care).
294 */
587142f8 295 spin_unlock(&inode->i_lock);
cfb506e1
TM
296 if (!nonblock)
297 ret = nfs_wait_on_request(req);
298 else
299 ret = -EAGAIN;
e261f51f
TM
300 nfs_release_request(req);
301 if (ret != 0)
074cc1de 302 return ERR_PTR(ret);
587142f8 303 spin_lock(&inode->i_lock);
e261f51f 304 }
587142f8 305 spin_unlock(&inode->i_lock);
074cc1de
TM
306 return req;
307}
308
309/*
310 * Find an associated nfs write request, and prepare to flush it out
311 * May return an error if the user signalled nfs_wait_on_request().
312 */
313static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
cfb506e1 314 struct page *page, bool nonblock)
074cc1de
TM
315{
316 struct nfs_page *req;
317 int ret = 0;
318
cfb506e1 319 req = nfs_find_and_lock_request(page, nonblock);
074cc1de
TM
320 if (!req)
321 goto out;
322 ret = PTR_ERR(req);
323 if (IS_ERR(req))
324 goto out;
325
deed85e7
TM
326 nfs_set_page_writeback(page);
327 WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
074cc1de 328
deed85e7 329 ret = 0;
f8512ad0
FI
330 if (!nfs_pageio_add_request(pgio, req)) {
331 nfs_redirty_request(req);
074cc1de 332 ret = pgio->pg_error;
f8512ad0 333 }
074cc1de
TM
334out:
335 return ret;
e261f51f
TM
336}
337
f758c885 338static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
1da177e4 339{
d56b4ddf 340 struct inode *inode = page_file_mapping(page)->host;
cfb506e1 341 int ret;
1da177e4 342
91d5b470
CL
343 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
344 nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
345
d56b4ddf 346 nfs_pageio_cond_complete(pgio, page_file_index(page));
1b430bee 347 ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE);
cfb506e1
TM
348 if (ret == -EAGAIN) {
349 redirty_page_for_writepage(wbc, page);
350 ret = 0;
351 }
352 return ret;
f758c885 353}
7fe7f848 354
f758c885
TM
355/*
356 * Write an mmapped page to the server.
357 */
358static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc)
359{
360 struct nfs_pageio_descriptor pgio;
361 int err;
49a70f27 362
a20c93e3
CH
363 nfs_pageio_init_write(&pgio, page->mapping->host, wb_priority(wbc),
364 false, &nfs_async_write_completion_ops);
f758c885
TM
365 err = nfs_do_writepage(page, wbc, &pgio);
366 nfs_pageio_complete(&pgio);
367 if (err < 0)
368 return err;
369 if (pgio.pg_error < 0)
370 return pgio.pg_error;
371 return 0;
4d770ccf
TM
372}
373
374int nfs_writepage(struct page *page, struct writeback_control *wbc)
375{
f758c885 376 int ret;
4d770ccf 377
f758c885 378 ret = nfs_writepage_locked(page, wbc);
1da177e4 379 unlock_page(page);
f758c885
TM
380 return ret;
381}
382
383static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
384{
385 int ret;
386
387 ret = nfs_do_writepage(page, wbc, data);
388 unlock_page(page);
389 return ret;
1da177e4
LT
390}
391
1da177e4
LT
392int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
393{
1da177e4 394 struct inode *inode = mapping->host;
72cb77f4 395 unsigned long *bitlock = &NFS_I(inode)->flags;
c63c7b05 396 struct nfs_pageio_descriptor pgio;
1da177e4
LT
397 int err;
398
72cb77f4
TM
399 /* Stop dirtying of new pages while we sync */
400 err = wait_on_bit_lock(bitlock, NFS_INO_FLUSHING,
401 nfs_wait_bit_killable, TASK_KILLABLE);
402 if (err)
403 goto out_err;
404
91d5b470
CL
405 nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
406
a20c93e3
CH
407 nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
408 &nfs_async_write_completion_ops);
f758c885 409 err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
c63c7b05 410 nfs_pageio_complete(&pgio);
72cb77f4
TM
411
412 clear_bit_unlock(NFS_INO_FLUSHING, bitlock);
4e857c58 413 smp_mb__after_atomic();
72cb77f4
TM
414 wake_up_bit(bitlock, NFS_INO_FLUSHING);
415
f758c885 416 if (err < 0)
72cb77f4
TM
417 goto out_err;
418 err = pgio.pg_error;
419 if (err < 0)
420 goto out_err;
c63c7b05 421 return 0;
72cb77f4
TM
422out_err:
423 return err;
1da177e4
LT
424}
425
426/*
427 * Insert a write request into an inode
428 */
d6d6dc7c 429static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
1da177e4
LT
430{
431 struct nfs_inode *nfsi = NFS_I(inode);
e7d39069 432
2bfc6e56
WAA
433 WARN_ON_ONCE(req->wb_this_page != req);
434
e7d39069 435 /* Lock the request! */
7ad84aa9 436 nfs_lock_request(req);
e7d39069
TM
437
438 spin_lock(&inode->i_lock);
011e2a7f 439 if (!nfsi->npages && NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
a9a4a87a 440 inode->i_version++;
29418aa4
MG
441 /*
442 * Swap-space should not get truncated. Hence no need to plug the race
443 * with invalidate/truncate.
444 */
445 if (likely(!PageSwapCache(req->wb_page))) {
446 set_bit(PG_MAPPED, &req->wb_flags);
447 SetPagePrivate(req->wb_page);
448 set_page_private(req->wb_page, (unsigned long)req);
449 }
1da177e4 450 nfsi->npages++;
17089a29
WAA
451 /* this a head request for a page group - mark it as having an
452 * extra reference so sub groups can follow suit */
453 WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
c03b4024 454 kref_get(&req->wb_kref);
e7d39069 455 spin_unlock(&inode->i_lock);
1da177e4
LT
456}
457
458/*
89a09141 459 * Remove a write request from an inode
1da177e4
LT
460 */
461static void nfs_inode_remove_request(struct nfs_page *req)
462{
3d4ff43d 463 struct inode *inode = req->wb_context->dentry->d_inode;
1da177e4 464 struct nfs_inode *nfsi = NFS_I(inode);
20633f04 465 struct nfs_page *head;
1da177e4 466
20633f04
WAA
467 if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
468 head = req->wb_head;
469
470 spin_lock(&inode->i_lock);
471 if (likely(!PageSwapCache(head->wb_page))) {
472 set_page_private(head->wb_page, 0);
473 ClearPagePrivate(head->wb_page);
474 clear_bit(PG_MAPPED, &head->wb_flags);
475 }
476 nfsi->npages--;
477 spin_unlock(&inode->i_lock);
29418aa4 478 }
17089a29
WAA
479
480 if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags))
481 nfs_release_request(req);
1da177e4
LT
482}
483
61822ab5 484static void
6d884e8f 485nfs_mark_request_dirty(struct nfs_page *req)
61822ab5 486{
61822ab5
TM
487 __set_page_dirty_nobuffers(req->wb_page);
488}
489
89d77c8f 490#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
8dd37758
TM
491/**
492 * nfs_request_add_commit_list - add request to a commit list
493 * @req: pointer to a struct nfs_page
ea2cf228
FI
494 * @dst: commit list head
495 * @cinfo: holds list lock and accounting info
8dd37758 496 *
ea2cf228 497 * This sets the PG_CLEAN bit, updates the cinfo count of
8dd37758
TM
498 * number of outstanding requests requiring a commit as well as
499 * the MM page stats.
500 *
ea2cf228 501 * The caller must _not_ hold the cinfo->lock, but must be
8dd37758 502 * holding the nfs_page lock.
1da177e4 503 */
8dd37758 504void
ea2cf228
FI
505nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst,
506 struct nfs_commit_info *cinfo)
1da177e4 507{
e468bae9 508 set_bit(PG_CLEAN, &(req)->wb_flags);
ea2cf228
FI
509 spin_lock(cinfo->lock);
510 nfs_list_add_request(req, dst);
511 cinfo->mds->ncommit++;
512 spin_unlock(cinfo->lock);
56f9cd68
FI
513 if (!cinfo->dreq) {
514 inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
d56b4ddf 515 inc_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
56f9cd68
FI
516 BDI_RECLAIMABLE);
517 __mark_inode_dirty(req->wb_context->dentry->d_inode,
518 I_DIRTY_DATASYNC);
519 }
1da177e4 520}
8dd37758
TM
521EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);
522
523/**
524 * nfs_request_remove_commit_list - Remove request from a commit list
525 * @req: pointer to a nfs_page
ea2cf228 526 * @cinfo: holds list lock and accounting info
8dd37758 527 *
ea2cf228 528 * This clears the PG_CLEAN bit, and updates the cinfo's count of
8dd37758
TM
529 * number of outstanding requests requiring a commit
530 * It does not update the MM page stats.
531 *
ea2cf228 532 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
8dd37758
TM
533 */
534void
ea2cf228
FI
535nfs_request_remove_commit_list(struct nfs_page *req,
536 struct nfs_commit_info *cinfo)
8dd37758 537{
8dd37758
TM
538 if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
539 return;
540 nfs_list_remove_request(req);
ea2cf228 541 cinfo->mds->ncommit--;
8dd37758
TM
542}
543EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);
544
ea2cf228
FI
545static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
546 struct inode *inode)
547{
548 cinfo->lock = &inode->i_lock;
549 cinfo->mds = &NFS_I(inode)->commit_info;
550 cinfo->ds = pnfs_get_ds_info(inode);
b359f9d0 551 cinfo->dreq = NULL;
f453a54a 552 cinfo->completion_ops = &nfs_commit_completion_ops;
ea2cf228
FI
553}
554
555void nfs_init_cinfo(struct nfs_commit_info *cinfo,
556 struct inode *inode,
557 struct nfs_direct_req *dreq)
558{
1763da12
FI
559 if (dreq)
560 nfs_init_cinfo_from_dreq(cinfo, dreq);
561 else
562 nfs_init_cinfo_from_inode(cinfo, inode);
ea2cf228
FI
563}
564EXPORT_SYMBOL_GPL(nfs_init_cinfo);
8dd37758
TM
565
566/*
567 * Add a request to the inode's commit list.
568 */
1763da12 569void
ea2cf228
FI
570nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
571 struct nfs_commit_info *cinfo)
8dd37758 572{
ea2cf228 573 if (pnfs_mark_request_commit(req, lseg, cinfo))
8dd37758 574 return;
ea2cf228 575 nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo);
8dd37758 576}
8e821cad 577
d6d6dc7c
FI
578static void
579nfs_clear_page_commit(struct page *page)
580{
581 dec_zone_page_state(page, NR_UNSTABLE_NFS);
d56b4ddf 582 dec_bdi_stat(page_file_mapping(page)->backing_dev_info, BDI_RECLAIMABLE);
d6d6dc7c
FI
583}
584
8dd37758 585static void
e468bae9
TM
586nfs_clear_request_commit(struct nfs_page *req)
587{
8dd37758
TM
588 if (test_bit(PG_CLEAN, &req->wb_flags)) {
589 struct inode *inode = req->wb_context->dentry->d_inode;
ea2cf228 590 struct nfs_commit_info cinfo;
e468bae9 591
ea2cf228
FI
592 nfs_init_cinfo_from_inode(&cinfo, inode);
593 if (!pnfs_clear_request_commit(req, &cinfo)) {
594 spin_lock(cinfo.lock);
595 nfs_request_remove_commit_list(req, &cinfo);
596 spin_unlock(cinfo.lock);
8dd37758 597 }
d6d6dc7c 598 nfs_clear_page_commit(req->wb_page);
e468bae9 599 }
e468bae9
TM
600}
601
8e821cad 602static inline
9c7e1b3d 603int nfs_write_need_commit(struct nfs_pgio_data *data)
8e821cad 604{
465d5243 605 if (data->verf.committed == NFS_DATA_SYNC)
cd841605
FI
606 return data->header->lseg == NULL;
607 return data->verf.committed != NFS_FILE_SYNC;
8e821cad
TM
608}
609
8e821cad 610#else
68cd6fa4
BS
611static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
612 struct inode *inode)
613{
614}
615
616void nfs_init_cinfo(struct nfs_commit_info *cinfo,
617 struct inode *inode,
618 struct nfs_direct_req *dreq)
619{
620}
621
1763da12 622void
ea2cf228
FI
623nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
624 struct nfs_commit_info *cinfo)
8e821cad
TM
625{
626}
627
8dd37758 628static void
e468bae9
TM
629nfs_clear_request_commit(struct nfs_page *req)
630{
e468bae9
TM
631}
632
8e821cad 633static inline
9c7e1b3d 634int nfs_write_need_commit(struct nfs_pgio_data *data)
8e821cad
TM
635{
636 return 0;
637}
638
6c75dc0d
FI
639#endif
640
061ae2ed 641static void nfs_write_completion(struct nfs_pgio_header *hdr)
8e821cad 642{
ea2cf228 643 struct nfs_commit_info cinfo;
6c75dc0d 644 unsigned long bytes = 0;
2bfc6e56 645 bool do_destroy;
6c75dc0d
FI
646
647 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
648 goto out;
ea2cf228 649 nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
6c75dc0d
FI
650 while (!list_empty(&hdr->pages)) {
651 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
6c75dc0d
FI
652
653 bytes += req->wb_bytes;
654 nfs_list_remove_request(req);
655 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
656 (hdr->good_bytes < bytes)) {
d1182b33 657 nfs_set_pageerror(req->wb_page);
6c75dc0d
FI
658 nfs_context_set_write_error(req->wb_context, hdr->error);
659 goto remove_req;
660 }
661 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) {
662 nfs_mark_request_dirty(req);
663 goto next;
664 }
665 if (test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) {
f79d06f5 666 memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
ea2cf228 667 nfs_mark_request_commit(req, hdr->lseg, &cinfo);
6c75dc0d
FI
668 goto next;
669 }
670remove_req:
671 nfs_inode_remove_request(req);
672next:
1d1afcbc 673 nfs_unlock_request(req);
20633f04 674 nfs_end_page_writeback(req);
2bfc6e56 675 do_destroy = !test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags);
3aff4ebb 676 nfs_release_request(req);
6c75dc0d
FI
677 }
678out:
679 hdr->release(hdr);
8e821cad 680}
1da177e4 681
89d77c8f 682#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
ce59515c 683unsigned long
ea2cf228 684nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
fb8a1f11 685{
ea2cf228 686 return cinfo->mds->ncommit;
d6d6dc7c
FI
687}
688
ea2cf228 689/* cinfo->lock held by caller */
1763da12 690int
ea2cf228
FI
691nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
692 struct nfs_commit_info *cinfo, int max)
d6d6dc7c
FI
693{
694 struct nfs_page *req, *tmp;
695 int ret = 0;
696
697 list_for_each_entry_safe(req, tmp, src, wb_list) {
8dd37758
TM
698 if (!nfs_lock_request(req))
699 continue;
7ad84aa9 700 kref_get(&req->wb_kref);
ea2cf228 701 if (cond_resched_lock(cinfo->lock))
3b3be88d 702 list_safe_reset_next(req, tmp, wb_list);
ea2cf228 703 nfs_request_remove_commit_list(req, cinfo);
8dd37758
TM
704 nfs_list_add_request(req, dst);
705 ret++;
1763da12 706 if ((ret == max) && !cinfo->dreq)
8dd37758 707 break;
d6d6dc7c
FI
708 }
709 return ret;
fb8a1f11
TM
710}
711
1da177e4
LT
712/*
713 * nfs_scan_commit - Scan an inode for commit requests
714 * @inode: NFS inode to scan
ea2cf228
FI
715 * @dst: mds destination list
716 * @cinfo: mds and ds lists of reqs ready to commit
1da177e4
LT
717 *
718 * Moves requests from the inode's 'commit' request list.
719 * The requests are *not* checked to ensure that they form a contiguous set.
720 */
1763da12 721int
ea2cf228
FI
722nfs_scan_commit(struct inode *inode, struct list_head *dst,
723 struct nfs_commit_info *cinfo)
1da177e4 724{
d6d6dc7c 725 int ret = 0;
fb8a1f11 726
ea2cf228
FI
727 spin_lock(cinfo->lock);
728 if (cinfo->mds->ncommit > 0) {
8dd37758 729 const int max = INT_MAX;
d6d6dc7c 730
ea2cf228
FI
731 ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
732 cinfo, max);
733 ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
d6d6dc7c 734 }
ea2cf228 735 spin_unlock(cinfo->lock);
ff778d02 736 return ret;
1da177e4 737}
d6d6dc7c 738
c42de9dd 739#else
ce59515c 740unsigned long nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
fb8a1f11
TM
741{
742 return 0;
743}
744
1763da12
FI
745int nfs_scan_commit(struct inode *inode, struct list_head *dst,
746 struct nfs_commit_info *cinfo)
c42de9dd
TM
747{
748 return 0;
749}
1da177e4
LT
750#endif
751
1da177e4 752/*
e7d39069
TM
753 * Search for an existing write request, and attempt to update
754 * it to reflect a new dirty region on a given page.
1da177e4 755 *
e7d39069
TM
756 * If the attempt fails, then the existing request is flushed out
757 * to disk.
1da177e4 758 */
e7d39069
TM
759static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
760 struct page *page,
761 unsigned int offset,
762 unsigned int bytes)
1da177e4 763{
e7d39069
TM
764 struct nfs_page *req;
765 unsigned int rqend;
766 unsigned int end;
767 int error;
768
769 if (!PagePrivate(page))
770 return NULL;
1da177e4
LT
771
772 end = offset + bytes;
e7d39069 773 spin_lock(&inode->i_lock);
1da177e4 774
1da177e4 775 for (;;) {
29418aa4 776 req = nfs_page_find_request_locked(NFS_I(inode), page);
e7d39069
TM
777 if (req == NULL)
778 goto out_unlock;
779
2bfc6e56
WAA
780 /* should be handled by nfs_flush_incompatible */
781 WARN_ON_ONCE(req->wb_head != req);
782 WARN_ON_ONCE(req->wb_this_page != req);
783
e7d39069
TM
784 rqend = req->wb_offset + req->wb_bytes;
785 /*
786 * Tell the caller to flush out the request if
787 * the offsets are non-contiguous.
788 * Note: nfs_flush_incompatible() will already
789 * have flushed out requests having wrong owners.
790 */
e468bae9 791 if (offset > rqend
e7d39069
TM
792 || end < req->wb_offset)
793 goto out_flushme;
794
7ad84aa9 795 if (nfs_lock_request(req))
1da177e4 796 break;
1da177e4 797
e7d39069 798 /* The request is locked, so wait and then retry */
587142f8 799 spin_unlock(&inode->i_lock);
e7d39069
TM
800 error = nfs_wait_on_request(req);
801 nfs_release_request(req);
802 if (error != 0)
803 goto out_err;
804 spin_lock(&inode->i_lock);
1da177e4
LT
805 }
806
807 /* Okay, the request matches. Update the region */
808 if (offset < req->wb_offset) {
809 req->wb_offset = offset;
810 req->wb_pgbase = offset;
1da177e4 811 }
1da177e4
LT
812 if (end > rqend)
813 req->wb_bytes = end - req->wb_offset;
e7d39069
TM
814 else
815 req->wb_bytes = rqend - req->wb_offset;
816out_unlock:
817 spin_unlock(&inode->i_lock);
ca138f36
FI
818 if (req)
819 nfs_clear_request_commit(req);
e7d39069
TM
820 return req;
821out_flushme:
822 spin_unlock(&inode->i_lock);
823 nfs_release_request(req);
824 error = nfs_wb_page(inode, page);
825out_err:
826 return ERR_PTR(error);
827}
828
829/*
830 * Try to update an existing write request, or create one if there is none.
831 *
832 * Note: Should always be called with the Page Lock held to prevent races
833 * if we have to add a new request. Also assumes that the caller has
834 * already called nfs_flush_incompatible() if necessary.
835 */
836static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
837 struct page *page, unsigned int offset, unsigned int bytes)
838{
d56b4ddf 839 struct inode *inode = page_file_mapping(page)->host;
e7d39069 840 struct nfs_page *req;
1da177e4 841
e7d39069
TM
842 req = nfs_try_to_update_request(inode, page, offset, bytes);
843 if (req != NULL)
844 goto out;
2bfc6e56 845 req = nfs_create_request(ctx, page, NULL, offset, bytes);
e7d39069
TM
846 if (IS_ERR(req))
847 goto out;
d6d6dc7c 848 nfs_inode_add_request(inode, req);
efc91ed0 849out:
61e930a9 850 return req;
1da177e4
LT
851}
852
e7d39069
TM
853static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
854 unsigned int offset, unsigned int count)
855{
856 struct nfs_page *req;
857
858 req = nfs_setup_write_request(ctx, page, offset, count);
859 if (IS_ERR(req))
860 return PTR_ERR(req);
861 /* Update file length */
862 nfs_grow_file(page, offset, count);
d72ddcba 863 nfs_mark_uptodate(req);
a6305ddb 864 nfs_mark_request_dirty(req);
1d1afcbc 865 nfs_unlock_and_release_request(req);
e7d39069
TM
866 return 0;
867}
868
1da177e4
LT
869int nfs_flush_incompatible(struct file *file, struct page *page)
870{
cd3758e3 871 struct nfs_open_context *ctx = nfs_file_open_context(file);
2a369153 872 struct nfs_lock_context *l_ctx;
1da177e4 873 struct nfs_page *req;
1a54533e 874 int do_flush, status;
1da177e4
LT
875 /*
876 * Look for a request corresponding to this page. If there
877 * is one, and it belongs to another file, we flush it out
878 * before we try to copy anything into the page. Do this
879 * due to the lack of an ACCESS-type call in NFSv2.
880 * Also do the same if we find a request from an existing
881 * dropped page.
882 */
1a54533e
TM
883 do {
884 req = nfs_page_find_request(page);
885 if (req == NULL)
886 return 0;
2a369153
TM
887 l_ctx = req->wb_lock_context;
888 do_flush = req->wb_page != page || req->wb_context != ctx;
2bfc6e56
WAA
889 /* for now, flush if more than 1 request in page_group */
890 do_flush |= req->wb_this_page != req;
0f1d2605 891 if (l_ctx && ctx->dentry->d_inode->i_flock != NULL) {
2a369153
TM
892 do_flush |= l_ctx->lockowner.l_owner != current->files
893 || l_ctx->lockowner.l_pid != current->tgid;
894 }
1da177e4 895 nfs_release_request(req);
1a54533e
TM
896 if (!do_flush)
897 return 0;
d56b4ddf 898 status = nfs_wb_page(page_file_mapping(page)->host, page);
1a54533e
TM
899 } while (status == 0);
900 return status;
1da177e4
LT
901}
902
dc24826b
AA
903/*
904 * Avoid buffered writes when a open context credential's key would
905 * expire soon.
906 *
907 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
908 *
909 * Return 0 and set a credential flag which triggers the inode to flush
910 * and performs NFS_FILE_SYNC writes if the key will expired within
911 * RPC_KEY_EXPIRE_TIMEO.
912 */
913int
914nfs_key_timeout_notify(struct file *filp, struct inode *inode)
915{
916 struct nfs_open_context *ctx = nfs_file_open_context(filp);
917 struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
918
919 return rpcauth_key_timeout_notify(auth, ctx->cred);
920}
921
922/*
923 * Test if the open context credential key is marked to expire soon.
924 */
925bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx)
926{
927 return rpcauth_cred_key_to_expire(ctx->cred);
928}
929
5d47a356
TM
930/*
931 * If the page cache is marked as unsafe or invalid, then we can't rely on
932 * the PageUptodate() flag. In this case, we will need to turn off
933 * write optimisations that depend on the page contents being correct.
934 */
8d197a56 935static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
5d47a356 936{
d529ef83
JL
937 struct nfs_inode *nfsi = NFS_I(inode);
938
8d197a56
TM
939 if (nfs_have_delegated_attributes(inode))
940 goto out;
18dd78c4 941 if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
d529ef83 942 return false;
4db72b40 943 smp_rmb();
d529ef83 944 if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
8d197a56
TM
945 return false;
946out:
18dd78c4
SM
947 if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
948 return false;
8d197a56 949 return PageUptodate(page) != 0;
5d47a356
TM
950}
951
c7559663
SM
952/* If we know the page is up to date, and we're not using byte range locks (or
953 * if we have the whole file locked for writing), it may be more efficient to
954 * extend the write to cover the entire page in order to avoid fragmentation
955 * inefficiencies.
956 *
263b4509
SM
957 * If the file is opened for synchronous writes then we can just skip the rest
958 * of the checks.
c7559663
SM
959 */
960static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
961{
962 if (file->f_flags & O_DSYNC)
963 return 0;
263b4509
SM
964 if (!nfs_write_pageuptodate(page, inode))
965 return 0;
c7559663
SM
966 if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
967 return 1;
263b4509 968 if (inode->i_flock == NULL || (inode->i_flock->fl_start == 0 &&
c7559663 969 inode->i_flock->fl_end == OFFSET_MAX &&
263b4509 970 inode->i_flock->fl_type != F_RDLCK))
c7559663
SM
971 return 1;
972 return 0;
973}
974
1da177e4
LT
975/*
976 * Update and possibly write a cached page of an NFS file.
977 *
978 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
979 * things with a page scheduled for an RPC call (e.g. invalidate it).
980 */
981int nfs_updatepage(struct file *file, struct page *page,
982 unsigned int offset, unsigned int count)
983{
cd3758e3 984 struct nfs_open_context *ctx = nfs_file_open_context(file);
d56b4ddf 985 struct inode *inode = page_file_mapping(page)->host;
1da177e4
LT
986 int status = 0;
987
91d5b470
CL
988 nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
989
6de1472f
AV
990 dprintk("NFS: nfs_updatepage(%pD2 %d@%lld)\n",
991 file, count, (long long)(page_file_offset(page) + offset));
1da177e4 992
c7559663 993 if (nfs_can_extend_write(file, page, inode)) {
49a70f27 994 count = max(count + offset, nfs_page_length(page));
1da177e4 995 offset = 0;
1da177e4
LT
996 }
997
e21195a7 998 status = nfs_writepage_setup(ctx, page, offset, count);
03fa9e84
TM
999 if (status < 0)
1000 nfs_set_pageerror(page);
59b7c05f
TM
1001 else
1002 __set_page_dirty_nobuffers(page);
1da177e4 1003
48186c7d 1004 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n",
1da177e4 1005 status, (long long)i_size_read(inode));
1da177e4
LT
1006 return status;
1007}
1008
3ff7576d 1009static int flush_task_priority(int how)
1da177e4
LT
1010{
1011 switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
1012 case FLUSH_HIGHPRI:
1013 return RPC_PRIORITY_HIGH;
1014 case FLUSH_LOWPRI:
1015 return RPC_PRIORITY_LOW;
1016 }
1017 return RPC_PRIORITY_NORMAL;
1018}
1019
1ed26f33
AS
1020static void nfs_initiate_write(struct nfs_pgio_data *data, struct rpc_message *msg,
1021 struct rpc_task_setup *task_setup_data, int how)
1da177e4 1022{
cd841605 1023 struct inode *inode = data->header->inode;
3ff7576d 1024 int priority = flush_task_priority(how);
d138d5d1 1025
1ed26f33
AS
1026 task_setup_data->priority = priority;
1027 NFS_PROTO(inode)->write_setup(data, msg);
d138d5d1 1028
8c21c62c 1029 nfs4_state_protect_write(NFS_SERVER(inode)->nfs_client,
1ed26f33 1030 &task_setup_data->rpc_client, msg, data);
275acaaf
TM
1031}
1032
6d884e8f
F
1033/* If a nfs_flush_* function fails, it should remove reqs from @head and
1034 * call this on each, which will prepare them to be retried on next
1035 * writeback using standard nfs.
1036 */
1037static void nfs_redirty_request(struct nfs_page *req)
1038{
1039 nfs_mark_request_dirty(req);
1d1afcbc 1040 nfs_unlock_request(req);
20633f04 1041 nfs_end_page_writeback(req);
3aff4ebb 1042 nfs_release_request(req);
6d884e8f
F
1043}
1044
061ae2ed 1045static void nfs_async_write_error(struct list_head *head)
6c75dc0d
FI
1046{
1047 struct nfs_page *req;
1048
1049 while (!list_empty(head)) {
1050 req = nfs_list_entry(head->next);
1051 nfs_list_remove_request(req);
1052 nfs_redirty_request(req);
1053 }
1054}
1055
061ae2ed
FI
1056static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1057 .error_cleanup = nfs_async_write_error,
1058 .completion = nfs_write_completion,
1059};
1060
57208fa7 1061void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
a20c93e3 1062 struct inode *inode, int ioflags, bool force_mds,
061ae2ed 1063 const struct nfs_pgio_completion_ops *compl_ops)
1da177e4 1064{
a20c93e3 1065 struct nfs_server *server = NFS_SERVER(inode);
41d8d5b7 1066 const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
a20c93e3
CH
1067
1068#ifdef CONFIG_NFS_V4_1
1069 if (server->pnfs_curr_ld && !force_mds)
1070 pg_ops = server->pnfs_curr_ld->pg_write_ops;
1071#endif
4a0de55c
AS
1072 nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1073 server->wsize, ioflags);
1751c363 1074}
ddda8e0a 1075EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
1da177e4 1076
dce81290
TM
1077void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
1078{
41d8d5b7 1079 pgio->pg_ops = &nfs_pgio_rw_ops;
dce81290
TM
1080 pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1081}
1f945357 1082EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
dce81290 1083
1da177e4 1084
0b7c0153
FI
1085void nfs_commit_prepare(struct rpc_task *task, void *calldata)
1086{
1087 struct nfs_commit_data *data = calldata;
1088
1089 NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
1090}
1091
a4cdda59 1092static void nfs_writeback_release_common(struct nfs_pgio_data *data)
c9d8f89d 1093{
cd841605 1094 struct nfs_pgio_header *hdr = data->header;
e2fecb21 1095 int status = data->task.tk_status;
788e7a89 1096
6c75dc0d
FI
1097 if ((status >= 0) && nfs_write_need_commit(data)) {
1098 spin_lock(&hdr->lock);
1099 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags))
1100 ; /* Do nothing */
1101 else if (!test_and_set_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags))
f79d06f5
AS
1102 memcpy(&hdr->verf, &data->verf, sizeof(hdr->verf));
1103 else if (memcmp(&hdr->verf, &data->verf, sizeof(hdr->verf)))
6c75dc0d
FI
1104 set_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags);
1105 spin_unlock(&hdr->lock);
1da177e4
LT
1106 }
1107}
1108
1f2edbe3
TM
1109/*
1110 * Special version of should_remove_suid() that ignores capabilities.
1111 */
1112static int nfs_should_remove_suid(const struct inode *inode)
1113{
1114 umode_t mode = inode->i_mode;
1115 int kill = 0;
1116
1117 /* suid always must be killed */
1118 if (unlikely(mode & S_ISUID))
1119 kill = ATTR_KILL_SUID;
788e7a89 1120
1f2edbe3
TM
1121 /*
1122 * sgid without any exec bits is just a mandatory locking mark; leave
1123 * it alone. If some exec bits are set, it's a real sgid; kill it.
1124 */
1125 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1126 kill |= ATTR_KILL_SGID;
1127
1128 if (unlikely(kill && S_ISREG(mode)))
1129 return kill;
1130
1131 return 0;
1132}
788e7a89 1133
1da177e4
LT
1134/*
1135 * This function is called when the WRITE call is complete.
1136 */
0eecb214
AS
1137static int nfs_writeback_done(struct rpc_task *task, struct nfs_pgio_data *data,
1138 struct inode *inode)
1da177e4 1139{
788e7a89 1140 int status;
1da177e4 1141
f551e44f
CL
1142 /*
1143 * ->write_done will attempt to use post-op attributes to detect
1144 * conflicting writes by other clients. A strict interpretation
1145 * of close-to-open would allow us to continue caching even if
1146 * another writer had changed the file, but some applications
1147 * depend on tighter cache coherency when writing.
1148 */
cd841605 1149 status = NFS_PROTO(inode)->write_done(task, data);
788e7a89 1150 if (status != 0)
0eecb214
AS
1151 return status;
1152 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, data->res.count);
91d5b470 1153
89d77c8f 1154#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
0eecb214 1155 if (data->res.verf->committed < data->args.stable && task->tk_status >= 0) {
1da177e4
LT
1156 /* We tried a write call, but the server did not
1157 * commit data to stable storage even though we
1158 * requested it.
1159 * Note: There is a known bug in Tru64 < 5.0 in which
1160 * the server reports NFS_DATA_SYNC, but performs
1161 * NFS_FILE_SYNC. We therefore implement this checking
1162 * as a dprintk() in order to avoid filling syslog.
1163 */
1164 static unsigned long complain;
1165
a69aef14 1166 /* Note this will print the MDS for a DS write */
1da177e4 1167 if (time_before(complain, jiffies)) {
48186c7d 1168 dprintk("NFS: faulty NFS server %s:"
1da177e4 1169 " (committed = %d) != (stable = %d)\n",
cd841605 1170 NFS_SERVER(inode)->nfs_client->cl_hostname,
0eecb214 1171 data->res.verf->committed, data->args.stable);
1da177e4
LT
1172 complain = jiffies + 300 * HZ;
1173 }
1174 }
1175#endif
1f2edbe3
TM
1176
1177 /* Deal with the suid/sgid bit corner case */
1178 if (nfs_should_remove_suid(inode))
1179 nfs_mark_for_revalidate(inode);
0eecb214
AS
1180 return 0;
1181}
1182
1183/*
1184 * This function is called when the WRITE call is complete.
1185 */
1186static void nfs_writeback_result(struct rpc_task *task, struct nfs_pgio_data *data)
1187{
1188 struct nfs_pgio_args *argp = &data->args;
1189 struct nfs_pgio_res *resp = &data->res;
1f2edbe3
TM
1190
1191 if (resp->count < argp->count) {
1da177e4
LT
1192 static unsigned long complain;
1193
6c75dc0d 1194 /* This a short write! */
0eecb214 1195 nfs_inc_stats(data->header->inode, NFSIOS_SHORTWRITE);
91d5b470 1196
1da177e4 1197 /* Has the server at least made some progress? */
6c75dc0d
FI
1198 if (resp->count == 0) {
1199 if (time_before(complain, jiffies)) {
1200 printk(KERN_WARNING
1201 "NFS: Server wrote zero bytes, expected %u.\n",
1202 argp->count);
1203 complain = jiffies + 300 * HZ;
1da177e4 1204 }
6c75dc0d
FI
1205 nfs_set_pgio_error(data->header, -EIO, argp->offset);
1206 task->tk_status = -EIO;
13602896 1207 return;
1da177e4 1208 }
6c75dc0d
FI
1209 /* Was this an NFSv2 write or an NFSv3 stable write? */
1210 if (resp->verf->committed != NFS_UNSTABLE) {
1211 /* Resend from where the server left off */
1212 data->mds_offset += resp->count;
1213 argp->offset += resp->count;
1214 argp->pgbase += resp->count;
1215 argp->count -= resp->count;
1216 } else {
1217 /* Resend as a stable write in order to avoid
1218 * headaches in the case of a server crash.
1219 */
1220 argp->stable = NFS_FILE_SYNC;
1da177e4 1221 }
6c75dc0d 1222 rpc_restart_call_prepare(task);
1da177e4 1223 }
1da177e4
LT
1224}
1225
1226
89d77c8f 1227#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
71d0a611
TM
1228static int nfs_commit_set_lock(struct nfs_inode *nfsi, int may_wait)
1229{
b8413f98
TM
1230 int ret;
1231
71d0a611
TM
1232 if (!test_and_set_bit(NFS_INO_COMMIT, &nfsi->flags))
1233 return 1;
b8413f98
TM
1234 if (!may_wait)
1235 return 0;
1236 ret = out_of_line_wait_on_bit_lock(&nfsi->flags,
1237 NFS_INO_COMMIT,
1238 nfs_wait_bit_killable,
1239 TASK_KILLABLE);
1240 return (ret < 0) ? ret : 1;
71d0a611
TM
1241}
1242
f453a54a 1243static void nfs_commit_clear_lock(struct nfs_inode *nfsi)
71d0a611
TM
1244{
1245 clear_bit(NFS_INO_COMMIT, &nfsi->flags);
4e857c58 1246 smp_mb__after_atomic();
71d0a611
TM
1247 wake_up_bit(&nfsi->flags, NFS_INO_COMMIT);
1248}
1249
0b7c0153 1250void nfs_commitdata_release(struct nfs_commit_data *data)
1da177e4 1251{
0b7c0153
FI
1252 put_nfs_open_context(data->context);
1253 nfs_commit_free(data);
1da177e4 1254}
e0c2b380 1255EXPORT_SYMBOL_GPL(nfs_commitdata_release);
1da177e4 1256
0b7c0153 1257int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
9ace33cd 1258 const struct rpc_call_ops *call_ops,
9f0ec176 1259 int how, int flags)
1da177e4 1260{
07737691 1261 struct rpc_task *task;
9ace33cd 1262 int priority = flush_task_priority(how);
bdc7f021
TM
1263 struct rpc_message msg = {
1264 .rpc_argp = &data->args,
1265 .rpc_resp = &data->res,
9ace33cd 1266 .rpc_cred = data->cred,
bdc7f021 1267 };
84115e1c 1268 struct rpc_task_setup task_setup_data = {
07737691 1269 .task = &data->task,
9ace33cd 1270 .rpc_client = clnt,
bdc7f021 1271 .rpc_message = &msg,
9ace33cd 1272 .callback_ops = call_ops,
84115e1c 1273 .callback_data = data,
101070ca 1274 .workqueue = nfsiod_workqueue,
9f0ec176 1275 .flags = RPC_TASK_ASYNC | flags,
3ff7576d 1276 .priority = priority,
84115e1c 1277 };
9ace33cd
FI
1278 /* Set up the initial task struct. */
1279 NFS_PROTO(data->inode)->commit_setup(data, &msg);
1280
1281 dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
1282
8c21c62c
WAA
1283 nfs4_state_protect(NFS_SERVER(data->inode)->nfs_client,
1284 NFS_SP4_MACH_CRED_COMMIT, &task_setup_data.rpc_client, &msg);
1285
9ace33cd
FI
1286 task = rpc_run_task(&task_setup_data);
1287 if (IS_ERR(task))
1288 return PTR_ERR(task);
1289 if (how & FLUSH_SYNC)
1290 rpc_wait_for_completion_task(task);
1291 rpc_put_task(task);
1292 return 0;
1293}
e0c2b380 1294EXPORT_SYMBOL_GPL(nfs_initiate_commit);
9ace33cd
FI
1295
1296/*
1297 * Set up the argument/result storage required for the RPC call.
1298 */
0b7c0153 1299void nfs_init_commit(struct nfs_commit_data *data,
f453a54a
FI
1300 struct list_head *head,
1301 struct pnfs_layout_segment *lseg,
1302 struct nfs_commit_info *cinfo)
9ace33cd
FI
1303{
1304 struct nfs_page *first = nfs_list_entry(head->next);
3d4ff43d 1305 struct inode *inode = first->wb_context->dentry->d_inode;
1da177e4
LT
1306
1307 /* Set up the RPC argument and reply structs
1308 * NB: take care not to mess about with data->commit et al. */
1309
1310 list_splice_init(head, &data->pages);
1da177e4 1311
1da177e4 1312 data->inode = inode;
9ace33cd 1313 data->cred = first->wb_context->cred;
988b6dce 1314 data->lseg = lseg; /* reference transferred */
9ace33cd 1315 data->mds_ops = &nfs_commit_ops;
f453a54a 1316 data->completion_ops = cinfo->completion_ops;
b359f9d0 1317 data->dreq = cinfo->dreq;
1da177e4
LT
1318
1319 data->args.fh = NFS_FH(data->inode);
3da28eb1
TM
1320 /* Note: we always request a commit of the entire inode */
1321 data->args.offset = 0;
1322 data->args.count = 0;
0b7c0153 1323 data->context = get_nfs_open_context(first->wb_context);
1da177e4
LT
1324 data->res.fattr = &data->fattr;
1325 data->res.verf = &data->verf;
0e574af1 1326 nfs_fattr_init(&data->fattr);
1da177e4 1327}
e0c2b380 1328EXPORT_SYMBOL_GPL(nfs_init_commit);
1da177e4 1329
e0c2b380 1330void nfs_retry_commit(struct list_head *page_list,
ea2cf228
FI
1331 struct pnfs_layout_segment *lseg,
1332 struct nfs_commit_info *cinfo)
64bfeb49
FI
1333{
1334 struct nfs_page *req;
1335
1336 while (!list_empty(page_list)) {
1337 req = nfs_list_entry(page_list->next);
1338 nfs_list_remove_request(req);
ea2cf228 1339 nfs_mark_request_commit(req, lseg, cinfo);
56f9cd68
FI
1340 if (!cinfo->dreq) {
1341 dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
d56b4ddf 1342 dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
56f9cd68
FI
1343 BDI_RECLAIMABLE);
1344 }
1d1afcbc 1345 nfs_unlock_and_release_request(req);
64bfeb49
FI
1346 }
1347}
e0c2b380 1348EXPORT_SYMBOL_GPL(nfs_retry_commit);
64bfeb49 1349
1da177e4
LT
1350/*
1351 * Commit dirty pages
1352 */
1353static int
ea2cf228
FI
1354nfs_commit_list(struct inode *inode, struct list_head *head, int how,
1355 struct nfs_commit_info *cinfo)
1da177e4 1356{
0b7c0153 1357 struct nfs_commit_data *data;
1da177e4 1358
c9d8f89d 1359 data = nfs_commitdata_alloc();
1da177e4
LT
1360
1361 if (!data)
1362 goto out_bad;
1363
1364 /* Set up the argument struct */
f453a54a
FI
1365 nfs_init_commit(data, head, NULL, cinfo);
1366 atomic_inc(&cinfo->mds->rpcs_out);
9f0ec176
AA
1367 return nfs_initiate_commit(NFS_CLIENT(inode), data, data->mds_ops,
1368 how, 0);
1da177e4 1369 out_bad:
ea2cf228 1370 nfs_retry_commit(head, NULL, cinfo);
f453a54a 1371 cinfo->completion_ops->error_cleanup(NFS_I(inode));
1da177e4
LT
1372 return -ENOMEM;
1373}
1374
1375/*
1376 * COMMIT call returned
1377 */
788e7a89 1378static void nfs_commit_done(struct rpc_task *task, void *calldata)
1da177e4 1379{
0b7c0153 1380 struct nfs_commit_data *data = calldata;
1da177e4 1381
a3f565b1 1382 dprintk("NFS: %5u nfs_commit_done (status %d)\n",
1da177e4
LT
1383 task->tk_pid, task->tk_status);
1384
788e7a89 1385 /* Call the NFS version-specific code */
c0d0e96b 1386 NFS_PROTO(data->inode)->commit_done(task, data);
c9d8f89d
TM
1387}
1388
f453a54a 1389static void nfs_commit_release_pages(struct nfs_commit_data *data)
c9d8f89d 1390{
5917ce84 1391 struct nfs_page *req;
c9d8f89d 1392 int status = data->task.tk_status;
f453a54a 1393 struct nfs_commit_info cinfo;
788e7a89 1394
1da177e4
LT
1395 while (!list_empty(&data->pages)) {
1396 req = nfs_list_entry(data->pages.next);
1397 nfs_list_remove_request(req);
d6d6dc7c 1398 nfs_clear_page_commit(req->wb_page);
1da177e4 1399
1e8968c5 1400 dprintk("NFS: commit (%s/%llu %d@%lld)",
3d4ff43d 1401 req->wb_context->dentry->d_sb->s_id,
1e8968c5 1402 (unsigned long long)NFS_FILEID(req->wb_context->dentry->d_inode),
1da177e4
LT
1403 req->wb_bytes,
1404 (long long)req_offset(req));
c9d8f89d
TM
1405 if (status < 0) {
1406 nfs_context_set_write_error(req->wb_context, status);
1da177e4 1407 nfs_inode_remove_request(req);
c9d8f89d 1408 dprintk(", error = %d\n", status);
1da177e4
LT
1409 goto next;
1410 }
1411
1412 /* Okay, COMMIT succeeded, apparently. Check the verifier
1413 * returned by the server against all stored verfs. */
2f2c63bc 1414 if (!memcmp(&req->wb_verf, &data->verf.verifier, sizeof(req->wb_verf))) {
1da177e4
LT
1415 /* We have a match */
1416 nfs_inode_remove_request(req);
1417 dprintk(" OK\n");
1418 goto next;
1419 }
1420 /* We have a mismatch. Write the page again */
1421 dprintk(" mismatch\n");
6d884e8f 1422 nfs_mark_request_dirty(req);
05990d1b 1423 set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1da177e4 1424 next:
1d1afcbc 1425 nfs_unlock_and_release_request(req);
1da177e4 1426 }
f453a54a
FI
1427 nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1428 if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
1429 nfs_commit_clear_lock(NFS_I(data->inode));
5917ce84
FI
1430}
1431
1432static void nfs_commit_release(void *calldata)
1433{
0b7c0153 1434 struct nfs_commit_data *data = calldata;
5917ce84 1435
f453a54a 1436 data->completion_ops->completion(data);
c9d8f89d 1437 nfs_commitdata_release(calldata);
1da177e4 1438}
788e7a89
TM
1439
1440static const struct rpc_call_ops nfs_commit_ops = {
0b7c0153 1441 .rpc_call_prepare = nfs_commit_prepare,
788e7a89
TM
1442 .rpc_call_done = nfs_commit_done,
1443 .rpc_release = nfs_commit_release,
1444};
1da177e4 1445
f453a54a
FI
1446static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
1447 .completion = nfs_commit_release_pages,
1448 .error_cleanup = nfs_commit_clear_lock,
1449};
1450
1763da12
FI
1451int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
1452 int how, struct nfs_commit_info *cinfo)
84c53ab5
FI
1453{
1454 int status;
1455
ea2cf228 1456 status = pnfs_commit_list(inode, head, how, cinfo);
84c53ab5 1457 if (status == PNFS_NOT_ATTEMPTED)
ea2cf228 1458 status = nfs_commit_list(inode, head, how, cinfo);
84c53ab5
FI
1459 return status;
1460}
1461
b608b283 1462int nfs_commit_inode(struct inode *inode, int how)
1da177e4 1463{
1da177e4 1464 LIST_HEAD(head);
ea2cf228 1465 struct nfs_commit_info cinfo;
71d0a611 1466 int may_wait = how & FLUSH_SYNC;
b8413f98 1467 int res;
1da177e4 1468
b8413f98
TM
1469 res = nfs_commit_set_lock(NFS_I(inode), may_wait);
1470 if (res <= 0)
c5efa5fc 1471 goto out_mark_dirty;
ea2cf228
FI
1472 nfs_init_cinfo_from_inode(&cinfo, inode);
1473 res = nfs_scan_commit(inode, &head, &cinfo);
1da177e4 1474 if (res) {
a861a1e1
FI
1475 int error;
1476
ea2cf228 1477 error = nfs_generic_commit_list(inode, &head, how, &cinfo);
3da28eb1
TM
1478 if (error < 0)
1479 return error;
b8413f98 1480 if (!may_wait)
c5efa5fc 1481 goto out_mark_dirty;
b8413f98
TM
1482 error = wait_on_bit(&NFS_I(inode)->flags,
1483 NFS_INO_COMMIT,
1484 nfs_wait_bit_killable,
1485 TASK_KILLABLE);
1486 if (error < 0)
1487 return error;
71d0a611
TM
1488 } else
1489 nfs_commit_clear_lock(NFS_I(inode));
c5efa5fc
TM
1490 return res;
1491 /* Note: If we exit without ensuring that the commit is complete,
1492 * we must mark the inode as dirty. Otherwise, future calls to
1493 * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure
1494 * that the data is on the disk.
1495 */
1496out_mark_dirty:
1497 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1da177e4
LT
1498 return res;
1499}
8fc795f7
TM
1500
1501static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc)
1502{
420e3646
TM
1503 struct nfs_inode *nfsi = NFS_I(inode);
1504 int flags = FLUSH_SYNC;
1505 int ret = 0;
8fc795f7 1506
3236c3e1 1507 /* no commits means nothing needs to be done */
ea2cf228 1508 if (!nfsi->commit_info.ncommit)
3236c3e1
JL
1509 return ret;
1510
a00dd6c0
JL
1511 if (wbc->sync_mode == WB_SYNC_NONE) {
1512 /* Don't commit yet if this is a non-blocking flush and there
1513 * are a lot of outstanding writes for this mapping.
1514 */
ea2cf228 1515 if (nfsi->commit_info.ncommit <= (nfsi->npages >> 1))
a00dd6c0 1516 goto out_mark_dirty;
420e3646 1517
a00dd6c0 1518 /* don't wait for the COMMIT response */
420e3646 1519 flags = 0;
a00dd6c0
JL
1520 }
1521
420e3646
TM
1522 ret = nfs_commit_inode(inode, flags);
1523 if (ret >= 0) {
1524 if (wbc->sync_mode == WB_SYNC_NONE) {
1525 if (ret < wbc->nr_to_write)
1526 wbc->nr_to_write -= ret;
1527 else
1528 wbc->nr_to_write = 0;
1529 }
8fc795f7 1530 return 0;
420e3646
TM
1531 }
1532out_mark_dirty:
8fc795f7
TM
1533 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1534 return ret;
1535}
c63c7b05 1536#else
8fc795f7
TM
1537static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc)
1538{
1539 return 0;
1540}
1da177e4
LT
1541#endif
1542
8fc795f7
TM
1543int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1544{
a8d8f02c
BS
1545 return nfs_commit_unstable_pages(inode, wbc);
1546}
89d77c8f 1547EXPORT_SYMBOL_GPL(nfs_write_inode);
a8d8f02c 1548
acdc53b2
TM
1549/*
1550 * flush the inode to disk.
1551 */
1552int nfs_wb_all(struct inode *inode)
34901f70
TM
1553{
1554 struct writeback_control wbc = {
72cb77f4 1555 .sync_mode = WB_SYNC_ALL,
34901f70 1556 .nr_to_write = LONG_MAX,
d7fb1207
TM
1557 .range_start = 0,
1558 .range_end = LLONG_MAX,
34901f70 1559 };
f4ce1299
TM
1560 int ret;
1561
1562 trace_nfs_writeback_inode_enter(inode);
1563
1564 ret = sync_inode(inode, &wbc);
34901f70 1565
f4ce1299
TM
1566 trace_nfs_writeback_inode_exit(inode, ret);
1567 return ret;
1c75950b 1568}
ddda8e0a 1569EXPORT_SYMBOL_GPL(nfs_wb_all);
1c75950b 1570
1b3b4a1a
TM
1571int nfs_wb_page_cancel(struct inode *inode, struct page *page)
1572{
1573 struct nfs_page *req;
1b3b4a1a
TM
1574 int ret = 0;
1575
1b3b4a1a 1576 for (;;) {
ba8b06e6 1577 wait_on_page_writeback(page);
1b3b4a1a
TM
1578 req = nfs_page_find_request(page);
1579 if (req == NULL)
1b3b4a1a 1580 break;
7ad84aa9 1581 if (nfs_lock_request(req)) {
8dd37758 1582 nfs_clear_request_commit(req);
1b3b4a1a
TM
1583 nfs_inode_remove_request(req);
1584 /*
1585 * In case nfs_inode_remove_request has marked the
1586 * page as being dirty
1587 */
1588 cancel_dirty_page(page, PAGE_CACHE_SIZE);
1d1afcbc 1589 nfs_unlock_and_release_request(req);
1b3b4a1a
TM
1590 break;
1591 }
1592 ret = nfs_wait_on_request(req);
c9edda71 1593 nfs_release_request(req);
1b3b4a1a 1594 if (ret < 0)
c988950e 1595 break;
1b3b4a1a 1596 }
1b3b4a1a
TM
1597 return ret;
1598}
1599
7f2f12d9
TM
1600/*
1601 * Write back all requests on one page - we do this before reading it.
1602 */
1603int nfs_wb_page(struct inode *inode, struct page *page)
1c75950b 1604{
29418aa4 1605 loff_t range_start = page_file_offset(page);
1c75950b 1606 loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
4d770ccf 1607 struct writeback_control wbc = {
4d770ccf 1608 .sync_mode = WB_SYNC_ALL,
7f2f12d9 1609 .nr_to_write = 0,
4d770ccf
TM
1610 .range_start = range_start,
1611 .range_end = range_end,
1612 };
1613 int ret;
1c75950b 1614
f4ce1299
TM
1615 trace_nfs_writeback_page_enter(inode);
1616
0522f6ad 1617 for (;;) {
ba8b06e6 1618 wait_on_page_writeback(page);
73e3302f
TM
1619 if (clear_page_dirty_for_io(page)) {
1620 ret = nfs_writepage_locked(page, &wbc);
1621 if (ret < 0)
1622 goto out_error;
0522f6ad 1623 continue;
7f2f12d9 1624 }
f4ce1299 1625 ret = 0;
0522f6ad
TM
1626 if (!PagePrivate(page))
1627 break;
1628 ret = nfs_commit_inode(inode, FLUSH_SYNC);
ba8b06e6 1629 if (ret < 0)
73e3302f 1630 goto out_error;
7f2f12d9 1631 }
73e3302f 1632out_error:
f4ce1299 1633 trace_nfs_writeback_page_exit(inode, ret);
4d770ccf 1634 return ret;
1c75950b
TM
1635}
1636
074cc1de
TM
1637#ifdef CONFIG_MIGRATION
1638int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
a6bc32b8 1639 struct page *page, enum migrate_mode mode)
074cc1de 1640{
2da95652
JL
1641 /*
1642 * If PagePrivate is set, then the page is currently associated with
1643 * an in-progress read or write request. Don't try to migrate it.
1644 *
1645 * FIXME: we could do this in principle, but we'll need a way to ensure
1646 * that we can safely release the inode reference while holding
1647 * the page lock.
1648 */
1649 if (PagePrivate(page))
1650 return -EBUSY;
074cc1de 1651
8c209ce7
DH
1652 if (!nfs_fscache_release_page(page, GFP_KERNEL))
1653 return -EBUSY;
074cc1de 1654
a6bc32b8 1655 return migrate_page(mapping, newpage, page, mode);
074cc1de
TM
1656}
1657#endif
1658
f7b422b1 1659int __init nfs_init_writepagecache(void)
1da177e4
LT
1660{
1661 nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
c0752cdf 1662 sizeof(struct nfs_rw_header),
1da177e4 1663 0, SLAB_HWCACHE_ALIGN,
20c2df83 1664 NULL);
1da177e4
LT
1665 if (nfs_wdata_cachep == NULL)
1666 return -ENOMEM;
1667
93d2341c
MD
1668 nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
1669 nfs_wdata_cachep);
1da177e4 1670 if (nfs_wdata_mempool == NULL)
3dd4765f 1671 goto out_destroy_write_cache;
1da177e4 1672
0b7c0153
FI
1673 nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
1674 sizeof(struct nfs_commit_data),
1675 0, SLAB_HWCACHE_ALIGN,
1676 NULL);
1677 if (nfs_cdata_cachep == NULL)
3dd4765f 1678 goto out_destroy_write_mempool;
0b7c0153 1679
93d2341c 1680 nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
4c100210 1681 nfs_cdata_cachep);
1da177e4 1682 if (nfs_commit_mempool == NULL)
3dd4765f 1683 goto out_destroy_commit_cache;
1da177e4 1684
89a09141
PZ
1685 /*
1686 * NFS congestion size, scale with available memory.
1687 *
1688 * 64MB: 8192k
1689 * 128MB: 11585k
1690 * 256MB: 16384k
1691 * 512MB: 23170k
1692 * 1GB: 32768k
1693 * 2GB: 46340k
1694 * 4GB: 65536k
1695 * 8GB: 92681k
1696 * 16GB: 131072k
1697 *
1698 * This allows larger machines to have larger/more transfers.
1699 * Limit the default to 256M
1700 */
1701 nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
1702 if (nfs_congestion_kb > 256*1024)
1703 nfs_congestion_kb = 256*1024;
1704
1da177e4 1705 return 0;
3dd4765f
JL
1706
1707out_destroy_commit_cache:
1708 kmem_cache_destroy(nfs_cdata_cachep);
1709out_destroy_write_mempool:
1710 mempool_destroy(nfs_wdata_mempool);
1711out_destroy_write_cache:
1712 kmem_cache_destroy(nfs_wdata_cachep);
1713 return -ENOMEM;
1da177e4
LT
1714}
1715
266bee88 1716void nfs_destroy_writepagecache(void)
1da177e4
LT
1717{
1718 mempool_destroy(nfs_commit_mempool);
3dd4765f 1719 kmem_cache_destroy(nfs_cdata_cachep);
1da177e4 1720 mempool_destroy(nfs_wdata_mempool);
1a1d92c1 1721 kmem_cache_destroy(nfs_wdata_cachep);
1da177e4
LT
1722}
1723
4a0de55c 1724static const struct nfs_rw_ops nfs_rw_write_ops = {
a4cdda59 1725 .rw_mode = FMODE_WRITE,
4a0de55c
AS
1726 .rw_alloc_header = nfs_writehdr_alloc,
1727 .rw_free_header = nfs_writehdr_free,
a4cdda59 1728 .rw_release = nfs_writeback_release_common,
0eecb214
AS
1729 .rw_done = nfs_writeback_done,
1730 .rw_result = nfs_writeback_result,
1ed26f33 1731 .rw_initiate = nfs_initiate_write,
4a0de55c 1732};