]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
31143d5d DH |
2 | /* handling of writes to regular files and writing back to the server |
3 | * | |
4 | * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. | |
5 | * Written by David Howells (dhowells@redhat.com) | |
31143d5d | 6 | */ |
4343d008 | 7 | |
4af3c9cc | 8 | #include <linux/backing-dev.h> |
31143d5d DH |
9 | #include <linux/slab.h> |
10 | #include <linux/fs.h> | |
11 | #include <linux/pagemap.h> | |
12 | #include <linux/writeback.h> | |
13 | #include <linux/pagevec.h> | |
3003bbd0 DH |
14 | #include <linux/netfs.h> |
15 | #include <linux/fscache.h> | |
31143d5d DH |
16 | #include "internal.h" |
17 | ||
31143d5d DH |
18 | /* |
19 | * mark a page as having been made dirty and thus needing writeback | |
20 | */ | |
21 | int afs_set_page_dirty(struct page *page) | |
22 | { | |
23 | _enter(""); | |
24 | return __set_page_dirty_nobuffers(page); | |
25 | } | |
26 | ||
31143d5d DH |
27 | /* |
28 | * prepare to perform part of a write to a page | |
31143d5d | 29 | */ |
15b4650e NP |
30 | int afs_write_begin(struct file *file, struct address_space *mapping, |
31 | loff_t pos, unsigned len, unsigned flags, | |
21db2cdc | 32 | struct page **_page, void **fsdata) |
31143d5d | 33 | { |
496ad9aa | 34 | struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); |
15b4650e | 35 | struct page *page; |
4343d008 | 36 | unsigned long priv; |
e87b03f5 DH |
37 | unsigned f, from; |
38 | unsigned t, to; | |
39 | pgoff_t index; | |
31143d5d DH |
40 | int ret; |
41 | ||
e87b03f5 DH |
42 | _enter("{%llx:%llu},%llx,%x", |
43 | vnode->fid.vid, vnode->fid.vnode, pos, len); | |
31143d5d | 44 | |
3003bbd0 DH |
45 | /* Prefetch area to be written into the cache if we're caching this |
46 | * file. We need to do this before we get a lock on the page in case | |
47 | * there's more than one writer competing for the same cache block. | |
48 | */ | |
49 | ret = netfs_write_begin(file, mapping, pos, len, flags, &page, fsdata, | |
50 | &afs_req_ops, NULL); | |
51 | if (ret < 0) | |
52 | return ret; | |
630f5dda | 53 | |
e87b03f5 DH |
54 | index = page->index; |
55 | from = pos - index * PAGE_SIZE; | |
56 | to = from + len; | |
57 | ||
31143d5d | 58 | try_again: |
4343d008 DH |
59 | /* See if this page is already partially written in a way that we can |
60 | * merge the new write with. | |
61 | */ | |
4343d008 DH |
62 | if (PagePrivate(page)) { |
63 | priv = page_private(page); | |
67d78a6f DH |
64 | f = afs_page_dirty_from(page, priv); |
65 | t = afs_page_dirty_to(page, priv); | |
4343d008 | 66 | ASSERTCMP(f, <=, t); |
31143d5d | 67 | |
5a039c32 | 68 | if (PageWriteback(page)) { |
67d78a6f | 69 | trace_afs_page_dirty(vnode, tracepoint_string("alrdy"), page); |
5a039c32 DH |
70 | goto flush_conflicting_write; |
71 | } | |
5a813276 DH |
72 | /* If the file is being filled locally, allow inter-write |
73 | * spaces to be merged into writes. If it's not, only write | |
74 | * back what the user gives us. | |
75 | */ | |
76 | if (!test_bit(AFS_VNODE_NEW_CONTENT, &vnode->flags) && | |
77 | (to < f || from > t)) | |
4343d008 | 78 | goto flush_conflicting_write; |
31143d5d DH |
79 | } |
80 | ||
21db2cdc | 81 | *_page = page; |
4343d008 | 82 | _leave(" = 0"); |
31143d5d DH |
83 | return 0; |
84 | ||
4343d008 DH |
85 | /* The previous write and this write aren't adjacent or overlapping, so |
86 | * flush the page out. | |
87 | */ | |
88 | flush_conflicting_write: | |
31143d5d | 89 | _debug("flush conflict"); |
4343d008 | 90 | ret = write_one_page(page); |
21db2cdc DH |
91 | if (ret < 0) |
92 | goto error; | |
31143d5d | 93 | |
4343d008 | 94 | ret = lock_page_killable(page); |
21db2cdc DH |
95 | if (ret < 0) |
96 | goto error; | |
31143d5d | 97 | goto try_again; |
21db2cdc DH |
98 | |
99 | error: | |
100 | put_page(page); | |
101 | _leave(" = %d", ret); | |
102 | return ret; | |
31143d5d DH |
103 | } |
104 | ||
105 | /* | |
106 | * finalise part of a write to a page | |
107 | */ | |
15b4650e NP |
108 | int afs_write_end(struct file *file, struct address_space *mapping, |
109 | loff_t pos, unsigned len, unsigned copied, | |
110 | struct page *page, void *fsdata) | |
31143d5d | 111 | { |
496ad9aa | 112 | struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); |
f792e3ac | 113 | unsigned long priv; |
e87b03f5 | 114 | unsigned int f, from = pos & (thp_size(page) - 1); |
f792e3ac | 115 | unsigned int t, to = from + copied; |
31143d5d DH |
116 | loff_t i_size, maybe_i_size; |
117 | ||
3b6492df | 118 | _enter("{%llx:%llu},{%lx}", |
15b4650e | 119 | vnode->fid.vid, vnode->fid.vnode, page->index); |
31143d5d | 120 | |
66e9c6a8 DH |
121 | if (!PageUptodate(page)) { |
122 | if (copied < len) { | |
123 | copied = 0; | |
124 | goto out; | |
125 | } | |
126 | ||
127 | SetPageUptodate(page); | |
128 | } | |
129 | ||
3ad216ee DH |
130 | if (copied == 0) |
131 | goto out; | |
132 | ||
15b4650e | 133 | maybe_i_size = pos + copied; |
31143d5d DH |
134 | |
135 | i_size = i_size_read(&vnode->vfs_inode); | |
136 | if (maybe_i_size > i_size) { | |
1f32ef79 | 137 | write_seqlock(&vnode->cb_lock); |
31143d5d DH |
138 | i_size = i_size_read(&vnode->vfs_inode); |
139 | if (maybe_i_size > i_size) | |
9d37e1ca | 140 | afs_set_i_size(vnode, maybe_i_size); |
1f32ef79 | 141 | write_sequnlock(&vnode->cb_lock); |
31143d5d DH |
142 | } |
143 | ||
f792e3ac DH |
144 | if (PagePrivate(page)) { |
145 | priv = page_private(page); | |
67d78a6f DH |
146 | f = afs_page_dirty_from(page, priv); |
147 | t = afs_page_dirty_to(page, priv); | |
f792e3ac DH |
148 | if (from < f) |
149 | f = from; | |
150 | if (to > t) | |
151 | t = to; | |
67d78a6f | 152 | priv = afs_page_dirty(page, f, t); |
f792e3ac | 153 | set_page_private(page, priv); |
67d78a6f | 154 | trace_afs_page_dirty(vnode, tracepoint_string("dirty+"), page); |
f792e3ac | 155 | } else { |
67d78a6f | 156 | priv = afs_page_dirty(page, from, to); |
f792e3ac | 157 | attach_page_private(page, (void *)priv); |
67d78a6f | 158 | trace_afs_page_dirty(vnode, tracepoint_string("dirty"), page); |
f792e3ac DH |
159 | } |
160 | ||
e87b03f5 DH |
161 | if (set_page_dirty(page)) |
162 | _debug("dirtied %lx", page->index); | |
afae457d DH |
163 | |
164 | out: | |
15b4650e | 165 | unlock_page(page); |
09cbfeaf | 166 | put_page(page); |
3003bbd0 | 167 | return copied; |
31143d5d DH |
168 | } |
169 | ||
170 | /* | |
171 | * kill all the pages in the given range | |
172 | */ | |
4343d008 | 173 | static void afs_kill_pages(struct address_space *mapping, |
e87b03f5 | 174 | loff_t start, loff_t len) |
31143d5d | 175 | { |
4343d008 | 176 | struct afs_vnode *vnode = AFS_FS_I(mapping->host); |
31143d5d | 177 | struct pagevec pv; |
e87b03f5 | 178 | unsigned int loop, psize; |
31143d5d | 179 | |
e87b03f5 DH |
180 | _enter("{%llx:%llu},%llx @%llx", |
181 | vnode->fid.vid, vnode->fid.vnode, len, start); | |
31143d5d | 182 | |
86679820 | 183 | pagevec_init(&pv); |
31143d5d DH |
184 | |
185 | do { | |
e87b03f5 | 186 | _debug("kill %llx @%llx", len, start); |
31143d5d | 187 | |
e87b03f5 DH |
188 | pv.nr = find_get_pages_contig(mapping, start / PAGE_SIZE, |
189 | PAGEVEC_SIZE, pv.pages); | |
190 | if (pv.nr == 0) | |
191 | break; | |
31143d5d | 192 | |
e87b03f5 | 193 | for (loop = 0; loop < pv.nr; loop++) { |
7286a35e | 194 | struct page *page = pv.pages[loop]; |
e87b03f5 DH |
195 | |
196 | if (page->index * PAGE_SIZE >= start + len) | |
197 | break; | |
198 | ||
199 | psize = thp_size(page); | |
200 | start += psize; | |
201 | len -= psize; | |
7286a35e | 202 | ClearPageUptodate(page); |
4343d008 | 203 | end_page_writeback(page); |
4343d008 DH |
204 | lock_page(page); |
205 | generic_error_remove_page(mapping, page); | |
21bd68f1 | 206 | unlock_page(page); |
31143d5d DH |
207 | } |
208 | ||
209 | __pagevec_release(&pv); | |
e87b03f5 | 210 | } while (len > 0); |
31143d5d DH |
211 | |
212 | _leave(""); | |
213 | } | |
214 | ||
215 | /* | |
4343d008 DH |
216 | * Redirty all the pages in a given range. |
217 | */ | |
218 | static void afs_redirty_pages(struct writeback_control *wbc, | |
219 | struct address_space *mapping, | |
e87b03f5 | 220 | loff_t start, loff_t len) |
4343d008 DH |
221 | { |
222 | struct afs_vnode *vnode = AFS_FS_I(mapping->host); | |
223 | struct pagevec pv; | |
e87b03f5 | 224 | unsigned int loop, psize; |
4343d008 | 225 | |
e87b03f5 DH |
226 | _enter("{%llx:%llu},%llx @%llx", |
227 | vnode->fid.vid, vnode->fid.vnode, len, start); | |
4343d008 | 228 | |
487e2c9f | 229 | pagevec_init(&pv); |
4343d008 DH |
230 | |
231 | do { | |
e87b03f5 | 232 | _debug("redirty %llx @%llx", len, start); |
4343d008 | 233 | |
e87b03f5 DH |
234 | pv.nr = find_get_pages_contig(mapping, start / PAGE_SIZE, |
235 | PAGEVEC_SIZE, pv.pages); | |
236 | if (pv.nr == 0) | |
237 | break; | |
4343d008 | 238 | |
e87b03f5 | 239 | for (loop = 0; loop < pv.nr; loop++) { |
4343d008 DH |
240 | struct page *page = pv.pages[loop]; |
241 | ||
e87b03f5 DH |
242 | if (page->index * PAGE_SIZE >= start + len) |
243 | break; | |
244 | ||
245 | psize = thp_size(page); | |
246 | start += psize; | |
247 | len -= psize; | |
4343d008 DH |
248 | redirty_page_for_writepage(wbc, page); |
249 | end_page_writeback(page); | |
4343d008 DH |
250 | } |
251 | ||
252 | __pagevec_release(&pv); | |
e87b03f5 | 253 | } while (len > 0); |
31143d5d DH |
254 | |
255 | _leave(""); | |
256 | } | |
257 | ||
a58823ac DH |
258 | /* |
259 | * completion of write to server | |
260 | */ | |
e87b03f5 | 261 | static void afs_pages_written_back(struct afs_vnode *vnode, loff_t start, unsigned int len) |
a58823ac | 262 | { |
bd80d8a8 DH |
263 | struct address_space *mapping = vnode->vfs_inode.i_mapping; |
264 | struct page *page; | |
e87b03f5 | 265 | pgoff_t end; |
bd80d8a8 | 266 | |
e87b03f5 | 267 | XA_STATE(xas, &mapping->i_pages, start / PAGE_SIZE); |
a58823ac | 268 | |
e87b03f5 DH |
269 | _enter("{%llx:%llu},{%x @%llx}", |
270 | vnode->fid.vid, vnode->fid.vnode, len, start); | |
a58823ac | 271 | |
bd80d8a8 | 272 | rcu_read_lock(); |
a58823ac | 273 | |
e87b03f5 DH |
274 | end = (start + len - 1) / PAGE_SIZE; |
275 | xas_for_each(&xas, page, end) { | |
276 | if (!PageWriteback(page)) { | |
277 | kdebug("bad %x @%llx page %lx %lx", len, start, page->index, end); | |
278 | ASSERT(PageWriteback(page)); | |
279 | } | |
a58823ac | 280 | |
bd80d8a8 | 281 | trace_afs_page_dirty(vnode, tracepoint_string("clear"), page); |
e87b03f5 | 282 | detach_page_private(page); |
bd80d8a8 DH |
283 | page_endio(page, true, 0); |
284 | } | |
a58823ac | 285 | |
bd80d8a8 | 286 | rcu_read_unlock(); |
a58823ac DH |
287 | |
288 | afs_prune_wb_keys(vnode); | |
289 | _leave(""); | |
290 | } | |
291 | ||
d2ddc776 | 292 | /* |
e49c7b2f DH |
293 | * Find a key to use for the writeback. We cached the keys used to author the |
294 | * writes on the vnode. *_wbk will contain the last writeback key used or NULL | |
295 | * and we need to start from there if it's set. | |
d2ddc776 | 296 | */ |
e49c7b2f DH |
297 | static int afs_get_writeback_key(struct afs_vnode *vnode, |
298 | struct afs_wb_key **_wbk) | |
d2ddc776 | 299 | { |
4343d008 DH |
300 | struct afs_wb_key *wbk = NULL; |
301 | struct list_head *p; | |
302 | int ret = -ENOKEY, ret2; | |
d2ddc776 | 303 | |
4343d008 | 304 | spin_lock(&vnode->wb_lock); |
e49c7b2f DH |
305 | if (*_wbk) |
306 | p = (*_wbk)->vnode_link.next; | |
307 | else | |
308 | p = vnode->wb_keys.next; | |
4343d008 | 309 | |
4343d008 DH |
310 | while (p != &vnode->wb_keys) { |
311 | wbk = list_entry(p, struct afs_wb_key, vnode_link); | |
312 | _debug("wbk %u", key_serial(wbk->key)); | |
313 | ret2 = key_validate(wbk->key); | |
e49c7b2f DH |
314 | if (ret2 == 0) { |
315 | refcount_inc(&wbk->usage); | |
316 | _debug("USE WB KEY %u", key_serial(wbk->key)); | |
317 | break; | |
318 | } | |
319 | ||
320 | wbk = NULL; | |
4343d008 DH |
321 | if (ret == -ENOKEY) |
322 | ret = ret2; | |
323 | p = p->next; | |
324 | } | |
325 | ||
326 | spin_unlock(&vnode->wb_lock); | |
e49c7b2f DH |
327 | if (*_wbk) |
328 | afs_put_wb_key(*_wbk); | |
329 | *_wbk = wbk; | |
330 | return 0; | |
331 | } | |
4343d008 | 332 | |
e49c7b2f DH |
333 | static void afs_store_data_success(struct afs_operation *op) |
334 | { | |
335 | struct afs_vnode *vnode = op->file[0].vnode; | |
4343d008 | 336 | |
da8d0755 | 337 | op->ctime = op->file[0].scb.status.mtime_client; |
e49c7b2f DH |
338 | afs_vnode_commit_status(op, &op->file[0]); |
339 | if (op->error == 0) { | |
d383e346 | 340 | if (!op->store.laundering) |
e87b03f5 | 341 | afs_pages_written_back(vnode, op->store.pos, op->store.size); |
e49c7b2f | 342 | afs_stat_v(vnode, n_stores); |
bd80d8a8 | 343 | atomic_long_add(op->store.size, &afs_v2net(vnode)->n_store_bytes); |
e49c7b2f DH |
344 | } |
345 | } | |
4343d008 | 346 | |
e49c7b2f DH |
347 | static const struct afs_operation_ops afs_store_data_operation = { |
348 | .issue_afs_rpc = afs_fs_store_data, | |
349 | .issue_yfs_rpc = yfs_fs_store_data, | |
350 | .success = afs_store_data_success, | |
351 | }; | |
a58823ac | 352 | |
e49c7b2f DH |
353 | /* |
354 | * write to a file | |
355 | */ | |
e87b03f5 | 356 | static int afs_store_data(struct afs_vnode *vnode, struct iov_iter *iter, loff_t pos, |
bd80d8a8 | 357 | bool laundering) |
e49c7b2f | 358 | { |
e49c7b2f DH |
359 | struct afs_operation *op; |
360 | struct afs_wb_key *wbk = NULL; | |
bd80d8a8 DH |
361 | loff_t size = iov_iter_count(iter), i_size; |
362 | int ret = -ENOKEY; | |
e49c7b2f | 363 | |
bd80d8a8 | 364 | _enter("%s{%llx:%llu.%u},%llx,%llx", |
e49c7b2f DH |
365 | vnode->volume->name, |
366 | vnode->fid.vid, | |
367 | vnode->fid.vnode, | |
368 | vnode->fid.unique, | |
bd80d8a8 | 369 | size, pos); |
d2ddc776 | 370 | |
e49c7b2f DH |
371 | ret = afs_get_writeback_key(vnode, &wbk); |
372 | if (ret) { | |
373 | _leave(" = %d [no keys]", ret); | |
374 | return ret; | |
d2ddc776 DH |
375 | } |
376 | ||
e49c7b2f DH |
377 | op = afs_alloc_operation(wbk->key, vnode->volume); |
378 | if (IS_ERR(op)) { | |
379 | afs_put_wb_key(wbk); | |
380 | return -ENOMEM; | |
381 | } | |
382 | ||
bd80d8a8 DH |
383 | i_size = i_size_read(&vnode->vfs_inode); |
384 | ||
e49c7b2f DH |
385 | afs_op_set_vnode(op, 0, vnode); |
386 | op->file[0].dv_delta = 1; | |
22650f14 | 387 | op->file[0].modification = true; |
bd80d8a8 DH |
388 | op->store.write_iter = iter; |
389 | op->store.pos = pos; | |
bd80d8a8 DH |
390 | op->store.size = size; |
391 | op->store.i_size = max(pos + size, i_size); | |
d383e346 | 392 | op->store.laundering = laundering; |
b3597945 | 393 | op->mtime = vnode->vfs_inode.i_mtime; |
811f04ba | 394 | op->flags |= AFS_OPERATION_UNINTR; |
e49c7b2f DH |
395 | op->ops = &afs_store_data_operation; |
396 | ||
397 | try_next_key: | |
398 | afs_begin_vnode_operation(op); | |
399 | afs_wait_for_operation(op); | |
400 | ||
401 | switch (op->error) { | |
4343d008 DH |
402 | case -EACCES: |
403 | case -EPERM: | |
404 | case -ENOKEY: | |
405 | case -EKEYEXPIRED: | |
406 | case -EKEYREJECTED: | |
407 | case -EKEYREVOKED: | |
408 | _debug("next"); | |
e49c7b2f DH |
409 | |
410 | ret = afs_get_writeback_key(vnode, &wbk); | |
411 | if (ret == 0) { | |
412 | key_put(op->key); | |
413 | op->key = key_get(wbk->key); | |
414 | goto try_next_key; | |
415 | } | |
416 | break; | |
4343d008 DH |
417 | } |
418 | ||
419 | afs_put_wb_key(wbk); | |
e49c7b2f DH |
420 | _leave(" = %d", op->error); |
421 | return afs_put_operation(op); | |
d2ddc776 DH |
422 | } |
423 | ||
31143d5d | 424 | /* |
810caa3e DH |
425 | * Extend the region to be written back to include subsequent contiguously |
426 | * dirty pages if possible, but don't sleep while doing so. | |
427 | * | |
428 | * If this page holds new content, then we can include filler zeros in the | |
429 | * writeback. | |
31143d5d | 430 | */ |
810caa3e DH |
431 | static void afs_extend_writeback(struct address_space *mapping, |
432 | struct afs_vnode *vnode, | |
433 | long *_count, | |
e87b03f5 DH |
434 | loff_t start, |
435 | loff_t max_len, | |
436 | bool new_content, | |
437 | unsigned int *_len) | |
31143d5d | 438 | { |
e87b03f5 DH |
439 | struct pagevec pvec; |
440 | struct page *page; | |
441 | unsigned long priv; | |
442 | unsigned int psize, filler = 0; | |
443 | unsigned int f, t; | |
444 | loff_t len = *_len; | |
445 | pgoff_t index = (start + len) / PAGE_SIZE; | |
446 | bool stop = true; | |
447 | unsigned int i; | |
448 | ||
449 | XA_STATE(xas, &mapping->i_pages, index); | |
450 | pagevec_init(&pvec); | |
4343d008 | 451 | |
31143d5d | 452 | do { |
e87b03f5 DH |
453 | /* Firstly, we gather up a batch of contiguous dirty pages |
454 | * under the RCU read lock - but we can't clear the dirty flags | |
455 | * there if any of those pages are mapped. | |
456 | */ | |
457 | rcu_read_lock(); | |
31143d5d | 458 | |
e87b03f5 DH |
459 | xas_for_each(&xas, page, ULONG_MAX) { |
460 | stop = true; | |
461 | if (xas_retry(&xas, page)) | |
462 | continue; | |
463 | if (xa_is_value(page)) | |
464 | break; | |
465 | if (page->index != index) | |
5a813276 | 466 | break; |
e87b03f5 DH |
467 | |
468 | if (!page_cache_get_speculative(page)) { | |
469 | xas_reset(&xas); | |
470 | continue; | |
471 | } | |
472 | ||
473 | /* Has the page moved or been split? */ | |
581b2027 DH |
474 | if (unlikely(page != xas_reload(&xas))) { |
475 | put_page(page); | |
31143d5d | 476 | break; |
581b2027 | 477 | } |
e87b03f5 | 478 | |
581b2027 DH |
479 | if (!trylock_page(page)) { |
480 | put_page(page); | |
31143d5d | 481 | break; |
581b2027 | 482 | } |
4343d008 | 483 | if (!PageDirty(page) || PageWriteback(page)) { |
31143d5d | 484 | unlock_page(page); |
581b2027 | 485 | put_page(page); |
31143d5d DH |
486 | break; |
487 | } | |
4343d008 | 488 | |
e87b03f5 | 489 | psize = thp_size(page); |
4343d008 | 490 | priv = page_private(page); |
67d78a6f DH |
491 | f = afs_page_dirty_from(page, priv); |
492 | t = afs_page_dirty_to(page, priv); | |
810caa3e | 493 | if (f != 0 && !new_content) { |
31143d5d | 494 | unlock_page(page); |
581b2027 | 495 | put_page(page); |
31143d5d DH |
496 | break; |
497 | } | |
4343d008 | 498 | |
e87b03f5 DH |
499 | len += filler + t; |
500 | filler = psize - t; | |
501 | if (len >= max_len || *_count <= 0) | |
502 | stop = true; | |
503 | else if (t == psize || new_content) | |
504 | stop = false; | |
505 | ||
506 | index += thp_nr_pages(page); | |
507 | if (!pagevec_add(&pvec, page)) | |
508 | break; | |
509 | if (stop) | |
510 | break; | |
511 | } | |
512 | ||
513 | if (!stop) | |
514 | xas_pause(&xas); | |
515 | rcu_read_unlock(); | |
516 | ||
517 | /* Now, if we obtained any pages, we can shift them to being | |
518 | * writable and mark them for caching. | |
519 | */ | |
520 | if (!pagevec_count(&pvec)) | |
521 | break; | |
522 | ||
523 | for (i = 0; i < pagevec_count(&pvec); i++) { | |
524 | page = pvec.pages[i]; | |
67d78a6f | 525 | trace_afs_page_dirty(vnode, tracepoint_string("store+"), page); |
13524ab3 | 526 | |
31143d5d DH |
527 | if (!clear_page_dirty_for_io(page)) |
528 | BUG(); | |
529 | if (test_set_page_writeback(page)) | |
530 | BUG(); | |
e87b03f5 DH |
531 | |
532 | *_count -= thp_nr_pages(page); | |
31143d5d | 533 | unlock_page(page); |
31143d5d DH |
534 | } |
535 | ||
e87b03f5 DH |
536 | pagevec_release(&pvec); |
537 | cond_resched(); | |
538 | } while (!stop); | |
31143d5d | 539 | |
e87b03f5 | 540 | *_len = len; |
810caa3e DH |
541 | } |
542 | ||
543 | /* | |
544 | * Synchronously write back the locked page and any subsequent non-locked dirty | |
545 | * pages. | |
546 | */ | |
e87b03f5 DH |
547 | static ssize_t afs_write_back_from_locked_page(struct address_space *mapping, |
548 | struct writeback_control *wbc, | |
549 | struct page *page, | |
550 | loff_t start, loff_t end) | |
810caa3e DH |
551 | { |
552 | struct afs_vnode *vnode = AFS_FS_I(mapping->host); | |
553 | struct iov_iter iter; | |
e87b03f5 DH |
554 | unsigned long priv; |
555 | unsigned int offset, to, len, max_len; | |
556 | loff_t i_size = i_size_read(&vnode->vfs_inode); | |
810caa3e | 557 | bool new_content = test_bit(AFS_VNODE_NEW_CONTENT, &vnode->flags); |
e87b03f5 | 558 | long count = wbc->nr_to_write; |
810caa3e DH |
559 | int ret; |
560 | ||
e87b03f5 | 561 | _enter(",%lx,%llx-%llx", page->index, start, end); |
810caa3e | 562 | |
e87b03f5 | 563 | if (test_set_page_writeback(page)) |
810caa3e DH |
564 | BUG(); |
565 | ||
e87b03f5 DH |
566 | count -= thp_nr_pages(page); |
567 | ||
810caa3e DH |
568 | /* Find all consecutive lockable dirty pages that have contiguous |
569 | * written regions, stopping when we find a page that is not | |
570 | * immediately lockable, is not dirty or is missing, or we reach the | |
571 | * end of the range. | |
572 | */ | |
e87b03f5 DH |
573 | priv = page_private(page); |
574 | offset = afs_page_dirty_from(page, priv); | |
575 | to = afs_page_dirty_to(page, priv); | |
576 | trace_afs_page_dirty(vnode, tracepoint_string("store"), page); | |
577 | ||
578 | len = to - offset; | |
579 | start += offset; | |
580 | if (start < i_size) { | |
581 | /* Trim the write to the EOF; the extra data is ignored. Also | |
582 | * put an upper limit on the size of a single storedata op. | |
583 | */ | |
584 | max_len = 65536 * 4096; | |
585 | max_len = min_t(unsigned long long, max_len, end - start + 1); | |
586 | max_len = min_t(unsigned long long, max_len, i_size - start); | |
587 | ||
588 | if (len < max_len && | |
589 | (to == thp_size(page) || new_content)) | |
590 | afs_extend_writeback(mapping, vnode, &count, | |
591 | start, max_len, new_content, &len); | |
592 | len = min_t(loff_t, len, max_len); | |
593 | } | |
810caa3e | 594 | |
4343d008 DH |
595 | /* We now have a contiguous set of dirty pages, each with writeback |
596 | * set; the first page is still locked at this point, but all the rest | |
597 | * have been unlocked. | |
598 | */ | |
e87b03f5 | 599 | unlock_page(page); |
793fe82e | 600 | |
e87b03f5 DH |
601 | if (start < i_size) { |
602 | _debug("write back %x @%llx [%llx]", len, start, i_size); | |
bd80d8a8 | 603 | |
e87b03f5 DH |
604 | iov_iter_xarray(&iter, WRITE, &mapping->i_pages, start, len); |
605 | ret = afs_store_data(vnode, &iter, start, false); | |
bd80d8a8 | 606 | } else { |
e87b03f5 DH |
607 | _debug("write discard %x @%llx [%llx]", len, start, i_size); |
608 | ||
bd80d8a8 | 609 | /* The dirty region was entirely beyond the EOF. */ |
e87b03f5 | 610 | afs_pages_written_back(vnode, start, len); |
bd80d8a8 DH |
611 | ret = 0; |
612 | } | |
31143d5d | 613 | |
4343d008 DH |
614 | switch (ret) { |
615 | case 0: | |
e87b03f5 DH |
616 | wbc->nr_to_write = count; |
617 | ret = len; | |
4343d008 DH |
618 | break; |
619 | ||
620 | default: | |
621 | pr_notice("kAFS: Unexpected error from FS.StoreData %d\n", ret); | |
df561f66 | 622 | fallthrough; |
4343d008 DH |
623 | case -EACCES: |
624 | case -EPERM: | |
625 | case -ENOKEY: | |
626 | case -EKEYEXPIRED: | |
627 | case -EKEYREJECTED: | |
628 | case -EKEYREVOKED: | |
e87b03f5 | 629 | afs_redirty_pages(wbc, mapping, start, len); |
4343d008 DH |
630 | mapping_set_error(mapping, ret); |
631 | break; | |
632 | ||
633 | case -EDQUOT: | |
634 | case -ENOSPC: | |
e87b03f5 | 635 | afs_redirty_pages(wbc, mapping, start, len); |
4343d008 DH |
636 | mapping_set_error(mapping, -ENOSPC); |
637 | break; | |
638 | ||
639 | case -EROFS: | |
640 | case -EIO: | |
641 | case -EREMOTEIO: | |
642 | case -EFBIG: | |
643 | case -ENOENT: | |
644 | case -ENOMEDIUM: | |
645 | case -ENXIO: | |
f51375cd | 646 | trace_afs_file_error(vnode, ret, afs_file_error_writeback_fail); |
e87b03f5 | 647 | afs_kill_pages(mapping, start, len); |
4343d008 DH |
648 | mapping_set_error(mapping, ret); |
649 | break; | |
31143d5d DH |
650 | } |
651 | ||
652 | _leave(" = %d", ret); | |
653 | return ret; | |
654 | } | |
655 | ||
656 | /* | |
657 | * write a page back to the server | |
658 | * - the caller locked the page for us | |
659 | */ | |
660 | int afs_writepage(struct page *page, struct writeback_control *wbc) | |
661 | { | |
e87b03f5 DH |
662 | ssize_t ret; |
663 | loff_t start; | |
31143d5d DH |
664 | |
665 | _enter("{%lx},", page->index); | |
666 | ||
e87b03f5 | 667 | start = page->index * PAGE_SIZE; |
4343d008 | 668 | ret = afs_write_back_from_locked_page(page->mapping, wbc, page, |
e87b03f5 | 669 | start, LLONG_MAX - start); |
31143d5d | 670 | if (ret < 0) { |
e87b03f5 DH |
671 | _leave(" = %zd", ret); |
672 | return ret; | |
31143d5d DH |
673 | } |
674 | ||
31143d5d DH |
675 | _leave(" = 0"); |
676 | return 0; | |
677 | } | |
678 | ||
679 | /* | |
680 | * write a region of pages back to the server | |
681 | */ | |
c1206a2c AB |
682 | static int afs_writepages_region(struct address_space *mapping, |
683 | struct writeback_control *wbc, | |
e87b03f5 | 684 | loff_t start, loff_t end, loff_t *_next) |
31143d5d | 685 | { |
31143d5d | 686 | struct page *page; |
e87b03f5 DH |
687 | ssize_t ret; |
688 | int n; | |
31143d5d | 689 | |
e87b03f5 | 690 | _enter("%llx,%llx,", start, end); |
31143d5d DH |
691 | |
692 | do { | |
e87b03f5 DH |
693 | pgoff_t index = start / PAGE_SIZE; |
694 | ||
695 | n = find_get_pages_range_tag(mapping, &index, end / PAGE_SIZE, | |
696 | PAGECACHE_TAG_DIRTY, 1, &page); | |
31143d5d DH |
697 | if (!n) |
698 | break; | |
699 | ||
e87b03f5 DH |
700 | start = (loff_t)page->index * PAGE_SIZE; /* May regress with THPs */ |
701 | ||
31143d5d DH |
702 | _debug("wback %lx", page->index); |
703 | ||
e87b03f5 | 704 | /* At this point we hold neither the i_pages lock nor the |
b93b0163 MW |
705 | * page lock: the page may be truncated or invalidated |
706 | * (changing page->mapping to NULL), or even swizzled | |
707 | * back from swapper_space to tmpfs file mapping | |
31143d5d | 708 | */ |
e87b03f5 DH |
709 | if (wbc->sync_mode != WB_SYNC_NONE) { |
710 | ret = lock_page_killable(page); | |
711 | if (ret < 0) { | |
712 | put_page(page); | |
713 | return ret; | |
714 | } | |
715 | } else { | |
716 | if (!trylock_page(page)) { | |
717 | put_page(page); | |
718 | return 0; | |
719 | } | |
4343d008 | 720 | } |
31143d5d | 721 | |
c5051c7b | 722 | if (page->mapping != mapping || !PageDirty(page)) { |
e87b03f5 | 723 | start += thp_size(page); |
31143d5d | 724 | unlock_page(page); |
09cbfeaf | 725 | put_page(page); |
31143d5d DH |
726 | continue; |
727 | } | |
728 | ||
c5051c7b | 729 | if (PageWriteback(page)) { |
31143d5d | 730 | unlock_page(page); |
c5051c7b DH |
731 | if (wbc->sync_mode != WB_SYNC_NONE) |
732 | wait_on_page_writeback(page); | |
29c8bbbd | 733 | put_page(page); |
31143d5d DH |
734 | continue; |
735 | } | |
736 | ||
65a15109 DH |
737 | if (!clear_page_dirty_for_io(page)) |
738 | BUG(); | |
e87b03f5 | 739 | ret = afs_write_back_from_locked_page(mapping, wbc, page, start, end); |
09cbfeaf | 740 | put_page(page); |
31143d5d | 741 | if (ret < 0) { |
e87b03f5 | 742 | _leave(" = %zd", ret); |
31143d5d DH |
743 | return ret; |
744 | } | |
745 | ||
dc255730 | 746 | start += ret; |
31143d5d | 747 | |
31143d5d | 748 | cond_resched(); |
e87b03f5 | 749 | } while (wbc->nr_to_write > 0); |
31143d5d | 750 | |
e87b03f5 DH |
751 | *_next = start; |
752 | _leave(" = 0 [%llx]", *_next); | |
31143d5d DH |
753 | return 0; |
754 | } | |
755 | ||
756 | /* | |
757 | * write some of the pending data back to the server | |
758 | */ | |
759 | int afs_writepages(struct address_space *mapping, | |
760 | struct writeback_control *wbc) | |
761 | { | |
ec0fa0b6 | 762 | struct afs_vnode *vnode = AFS_FS_I(mapping->host); |
e87b03f5 | 763 | loff_t start, next; |
31143d5d DH |
764 | int ret; |
765 | ||
766 | _enter(""); | |
767 | ||
ec0fa0b6 DH |
768 | /* We have to be careful as we can end up racing with setattr() |
769 | * truncating the pagecache since the caller doesn't take a lock here | |
770 | * to prevent it. | |
771 | */ | |
772 | if (wbc->sync_mode == WB_SYNC_ALL) | |
773 | down_read(&vnode->validate_lock); | |
774 | else if (!down_read_trylock(&vnode->validate_lock)) | |
775 | return 0; | |
776 | ||
31143d5d | 777 | if (wbc->range_cyclic) { |
e87b03f5 DH |
778 | start = mapping->writeback_index * PAGE_SIZE; |
779 | ret = afs_writepages_region(mapping, wbc, start, LLONG_MAX, &next); | |
afe69498 TR |
780 | if (ret == 0) { |
781 | mapping->writeback_index = next / PAGE_SIZE; | |
782 | if (start > 0 && wbc->nr_to_write > 0) { | |
783 | ret = afs_writepages_region(mapping, wbc, 0, | |
784 | start, &next); | |
785 | if (ret == 0) | |
786 | mapping->writeback_index = | |
787 | next / PAGE_SIZE; | |
788 | } | |
789 | } | |
31143d5d | 790 | } else if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) { |
e87b03f5 | 791 | ret = afs_writepages_region(mapping, wbc, 0, LLONG_MAX, &next); |
afe69498 | 792 | if (wbc->nr_to_write > 0 && ret == 0) |
5a972474 | 793 | mapping->writeback_index = next / PAGE_SIZE; |
31143d5d | 794 | } else { |
e87b03f5 DH |
795 | ret = afs_writepages_region(mapping, wbc, |
796 | wbc->range_start, wbc->range_end, &next); | |
31143d5d DH |
797 | } |
798 | ||
ec0fa0b6 | 799 | up_read(&vnode->validate_lock); |
31143d5d DH |
800 | _leave(" = %d", ret); |
801 | return ret; | |
802 | } | |
803 | ||
31143d5d DH |
804 | /* |
805 | * write to an AFS file | |
806 | */ | |
50b5551d | 807 | ssize_t afs_file_write(struct kiocb *iocb, struct iov_iter *from) |
31143d5d | 808 | { |
496ad9aa | 809 | struct afs_vnode *vnode = AFS_FS_I(file_inode(iocb->ki_filp)); |
3978d816 | 810 | struct afs_file *af = iocb->ki_filp->private_data; |
31143d5d | 811 | ssize_t result; |
50b5551d | 812 | size_t count = iov_iter_count(from); |
31143d5d | 813 | |
3b6492df | 814 | _enter("{%llx:%llu},{%zu},", |
50b5551d | 815 | vnode->fid.vid, vnode->fid.vnode, count); |
31143d5d DH |
816 | |
817 | if (IS_SWAPFILE(&vnode->vfs_inode)) { | |
818 | printk(KERN_INFO | |
819 | "AFS: Attempt to write to active swap file!\n"); | |
820 | return -EBUSY; | |
821 | } | |
822 | ||
823 | if (!count) | |
824 | return 0; | |
825 | ||
3978d816 DH |
826 | result = afs_validate(vnode, af->key); |
827 | if (result < 0) | |
828 | return result; | |
829 | ||
50b5551d | 830 | result = generic_file_write_iter(iocb, from); |
31143d5d | 831 | |
31143d5d DH |
832 | _leave(" = %zd", result); |
833 | return result; | |
834 | } | |
835 | ||
31143d5d DH |
836 | /* |
837 | * flush any dirty pages for this process, and check for write errors. | |
838 | * - the return status from this call provides a reliable indication of | |
839 | * whether any write errors occurred for this process. | |
840 | */ | |
02c24a82 | 841 | int afs_fsync(struct file *file, loff_t start, loff_t end, int datasync) |
31143d5d | 842 | { |
3978d816 DH |
843 | struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); |
844 | struct afs_file *af = file->private_data; | |
845 | int ret; | |
31143d5d | 846 | |
3b6492df | 847 | _enter("{%llx:%llu},{n=%pD},%d", |
3c981bfc | 848 | vnode->fid.vid, vnode->fid.vnode, file, |
31143d5d DH |
849 | datasync); |
850 | ||
3978d816 DH |
851 | ret = afs_validate(vnode, af->key); |
852 | if (ret < 0) | |
853 | return ret; | |
854 | ||
4343d008 | 855 | return file_write_and_wait_range(file, start, end); |
31143d5d | 856 | } |
9b3f26c9 DH |
857 | |
858 | /* | |
859 | * notification that a previously read-only page is about to become writable | |
860 | * - if it returns an error, the caller will deliver a bus error signal | |
861 | */ | |
0722f186 | 862 | vm_fault_t afs_page_mkwrite(struct vm_fault *vmf) |
9b3f26c9 | 863 | { |
490e016f MWO |
864 | struct folio *folio = page_folio(vmf->page); |
865 | struct page *page = &folio->page; | |
1cf7a151 DH |
866 | struct file *file = vmf->vma->vm_file; |
867 | struct inode *inode = file_inode(file); | |
868 | struct afs_vnode *vnode = AFS_FS_I(inode); | |
3978d816 | 869 | struct afs_file *af = file->private_data; |
1cf7a151 | 870 | unsigned long priv; |
9620ad86 | 871 | vm_fault_t ret = VM_FAULT_RETRY; |
9b3f26c9 | 872 | |
e87b03f5 | 873 | _enter("{{%llx:%llu}},{%lx}", vnode->fid.vid, vnode->fid.vnode, page->index); |
9b3f26c9 | 874 | |
3978d816 DH |
875 | afs_validate(vnode, af->key); |
876 | ||
1cf7a151 | 877 | sb_start_pagefault(inode->i_sb); |
9b3f26c9 | 878 | |
1cf7a151 DH |
879 | /* Wait for the page to be written to the cache before we allow it to |
880 | * be modified. We then assume the entire page will need writing back. | |
881 | */ | |
630f5dda | 882 | #ifdef CONFIG_AFS_FSCACHE |
e87b03f5 | 883 | if (PageFsCache(page) && |
5cbf0398 | 884 | wait_on_page_fscache_killable(page) < 0) |
9620ad86 | 885 | goto out; |
630f5dda | 886 | #endif |
9b3f26c9 | 887 | |
490e016f | 888 | if (folio_wait_writeback_killable(folio)) |
9620ad86 | 889 | goto out; |
1cf7a151 | 890 | |
e87b03f5 | 891 | if (lock_page_killable(page) < 0) |
9620ad86 | 892 | goto out; |
1cf7a151 DH |
893 | |
894 | /* We mustn't change page->private until writeback is complete as that | |
895 | * details the portion of the page we need to write back and we might | |
896 | * need to redirty the page if there's a problem. | |
897 | */ | |
490e016f MWO |
898 | if (folio_wait_writeback_killable(folio) < 0) { |
899 | folio_unlock(folio); | |
9620ad86 | 900 | goto out; |
5cbf0398 | 901 | } |
1cf7a151 | 902 | |
e87b03f5 | 903 | priv = afs_page_dirty(page, 0, thp_size(page)); |
f86726a6 | 904 | priv = afs_page_dirty_mmapped(priv); |
e87b03f5 DH |
905 | if (PagePrivate(page)) { |
906 | set_page_private(page, priv); | |
907 | trace_afs_page_dirty(vnode, tracepoint_string("mkwrite+"), page); | |
908 | } else { | |
909 | attach_page_private(page, (void *)priv); | |
910 | trace_afs_page_dirty(vnode, tracepoint_string("mkwrite"), page); | |
911 | } | |
bb413489 | 912 | file_update_time(file); |
1cf7a151 | 913 | |
9620ad86 MWO |
914 | ret = VM_FAULT_LOCKED; |
915 | out: | |
1cf7a151 | 916 | sb_end_pagefault(inode->i_sb); |
9620ad86 | 917 | return ret; |
9b3f26c9 | 918 | } |
4343d008 DH |
919 | |
920 | /* | |
921 | * Prune the keys cached for writeback. The caller must hold vnode->wb_lock. | |
922 | */ | |
923 | void afs_prune_wb_keys(struct afs_vnode *vnode) | |
924 | { | |
925 | LIST_HEAD(graveyard); | |
926 | struct afs_wb_key *wbk, *tmp; | |
927 | ||
928 | /* Discard unused keys */ | |
929 | spin_lock(&vnode->wb_lock); | |
930 | ||
931 | if (!mapping_tagged(&vnode->vfs_inode.i_data, PAGECACHE_TAG_WRITEBACK) && | |
932 | !mapping_tagged(&vnode->vfs_inode.i_data, PAGECACHE_TAG_DIRTY)) { | |
933 | list_for_each_entry_safe(wbk, tmp, &vnode->wb_keys, vnode_link) { | |
934 | if (refcount_read(&wbk->usage) == 1) | |
935 | list_move(&wbk->vnode_link, &graveyard); | |
936 | } | |
937 | } | |
938 | ||
939 | spin_unlock(&vnode->wb_lock); | |
940 | ||
941 | while (!list_empty(&graveyard)) { | |
942 | wbk = list_entry(graveyard.next, struct afs_wb_key, vnode_link); | |
943 | list_del(&wbk->vnode_link); | |
944 | afs_put_wb_key(wbk); | |
945 | } | |
946 | } | |
947 | ||
948 | /* | |
949 | * Clean up a page during invalidation. | |
950 | */ | |
951 | int afs_launder_page(struct page *page) | |
952 | { | |
953 | struct address_space *mapping = page->mapping; | |
954 | struct afs_vnode *vnode = AFS_FS_I(mapping->host); | |
bd80d8a8 DH |
955 | struct iov_iter iter; |
956 | struct bio_vec bv[1]; | |
4343d008 DH |
957 | unsigned long priv; |
958 | unsigned int f, t; | |
959 | int ret = 0; | |
960 | ||
961 | _enter("{%lx}", page->index); | |
962 | ||
963 | priv = page_private(page); | |
964 | if (clear_page_dirty_for_io(page)) { | |
965 | f = 0; | |
e87b03f5 | 966 | t = thp_size(page); |
4343d008 | 967 | if (PagePrivate(page)) { |
67d78a6f DH |
968 | f = afs_page_dirty_from(page, priv); |
969 | t = afs_page_dirty_to(page, priv); | |
4343d008 DH |
970 | } |
971 | ||
bd80d8a8 DH |
972 | bv[0].bv_page = page; |
973 | bv[0].bv_offset = f; | |
974 | bv[0].bv_len = t - f; | |
975 | iov_iter_bvec(&iter, WRITE, bv, 1, bv[0].bv_len); | |
976 | ||
67d78a6f | 977 | trace_afs_page_dirty(vnode, tracepoint_string("launder"), page); |
5c052248 | 978 | ret = afs_store_data(vnode, &iter, page_offset(page) + f, true); |
4343d008 DH |
979 | } |
980 | ||
67d78a6f | 981 | trace_afs_page_dirty(vnode, tracepoint_string("laundered"), page); |
e87b03f5 | 982 | detach_page_private(page); |
630f5dda | 983 | wait_on_page_fscache(page); |
4343d008 | 984 | return ret; |
9b3f26c9 | 985 | } |