]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
36c95590 DH |
2 | /* FS-Cache object state machine handler |
3 | * | |
4 | * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. | |
5 | * Written by David Howells (dhowells@redhat.com) | |
6 | * | |
36c95590 DH |
7 | * See Documentation/filesystems/caching/object.txt for a description of the |
8 | * object state machine and the in-kernel representations. | |
9 | */ | |
10 | ||
11 | #define FSCACHE_DEBUG_LEVEL COOKIE | |
12 | #include <linux/module.h> | |
ef778e7a | 13 | #include <linux/slab.h> |
caaef690 | 14 | #include <linux/prefetch.h> |
36c95590 DH |
15 | #include "internal.h" |
16 | ||
caaef690 DH |
17 | static const struct fscache_state *fscache_abort_initialisation(struct fscache_object *, int); |
18 | static const struct fscache_state *fscache_kill_dependents(struct fscache_object *, int); | |
19 | static const struct fscache_state *fscache_drop_object(struct fscache_object *, int); | |
20 | static const struct fscache_state *fscache_initialise_object(struct fscache_object *, int); | |
21 | static const struct fscache_state *fscache_invalidate_object(struct fscache_object *, int); | |
22 | static const struct fscache_state *fscache_jumpstart_dependents(struct fscache_object *, int); | |
23 | static const struct fscache_state *fscache_kill_object(struct fscache_object *, int); | |
24 | static const struct fscache_state *fscache_lookup_failure(struct fscache_object *, int); | |
25 | static const struct fscache_state *fscache_look_up_object(struct fscache_object *, int); | |
26 | static const struct fscache_state *fscache_object_available(struct fscache_object *, int); | |
27 | static const struct fscache_state *fscache_parent_ready(struct fscache_object *, int); | |
28 | static const struct fscache_state *fscache_update_object(struct fscache_object *, int); | |
e26bfebd | 29 | static const struct fscache_state *fscache_object_dead(struct fscache_object *, int); |
caaef690 DH |
30 | |
31 | #define __STATE_NAME(n) fscache_osm_##n | |
32 | #define STATE(n) (&__STATE_NAME(n)) | |
33 | ||
34 | /* | |
35 | * Define a work state. Work states are execution states. No event processing | |
36 | * is performed by them. The function attached to a work state returns a | |
37 | * pointer indicating the next state to which the state machine should | |
38 | * transition. Returning NO_TRANSIT repeats the current state, but goes back | |
39 | * to the scheduler first. | |
40 | */ | |
41 | #define WORK_STATE(n, sn, f) \ | |
42 | const struct fscache_state __STATE_NAME(n) = { \ | |
43 | .name = #n, \ | |
44 | .short_name = sn, \ | |
45 | .work = f \ | |
46 | } | |
47 | ||
48 | /* | |
49 | * Returns from work states. | |
50 | */ | |
51 | #define transit_to(state) ({ prefetch(&STATE(state)->work); STATE(state); }) | |
52 | ||
53 | #define NO_TRANSIT ((struct fscache_state *)NULL) | |
54 | ||
55 | /* | |
56 | * Define a wait state. Wait states are event processing states. No execution | |
57 | * is performed by them. Wait states are just tables of "if event X occurs, | |
58 | * clear it and transition to state Y". The dispatcher returns to the | |
59 | * scheduler if none of the events in which the wait state has an interest are | |
60 | * currently pending. | |
61 | */ | |
62 | #define WAIT_STATE(n, sn, ...) \ | |
63 | const struct fscache_state __STATE_NAME(n) = { \ | |
64 | .name = #n, \ | |
65 | .short_name = sn, \ | |
66 | .work = NULL, \ | |
67 | .transitions = { __VA_ARGS__, { 0, NULL } } \ | |
68 | } | |
69 | ||
70 | #define TRANSIT_TO(state, emask) \ | |
71 | { .events = (emask), .transit_to = STATE(state) } | |
72 | ||
73 | /* | |
74 | * The object state machine. | |
75 | */ | |
76 | static WORK_STATE(INIT_OBJECT, "INIT", fscache_initialise_object); | |
77 | static WORK_STATE(PARENT_READY, "PRDY", fscache_parent_ready); | |
78 | static WORK_STATE(ABORT_INIT, "ABRT", fscache_abort_initialisation); | |
79 | static WORK_STATE(LOOK_UP_OBJECT, "LOOK", fscache_look_up_object); | |
80 | static WORK_STATE(CREATE_OBJECT, "CRTO", fscache_look_up_object); | |
81 | static WORK_STATE(OBJECT_AVAILABLE, "AVBL", fscache_object_available); | |
82 | static WORK_STATE(JUMPSTART_DEPS, "JUMP", fscache_jumpstart_dependents); | |
83 | ||
84 | static WORK_STATE(INVALIDATE_OBJECT, "INVL", fscache_invalidate_object); | |
85 | static WORK_STATE(UPDATE_OBJECT, "UPDT", fscache_update_object); | |
86 | ||
87 | static WORK_STATE(LOOKUP_FAILURE, "LCFL", fscache_lookup_failure); | |
88 | static WORK_STATE(KILL_OBJECT, "KILL", fscache_kill_object); | |
89 | static WORK_STATE(KILL_DEPENDENTS, "KDEP", fscache_kill_dependents); | |
90 | static WORK_STATE(DROP_OBJECT, "DROP", fscache_drop_object); | |
e26bfebd | 91 | static WORK_STATE(OBJECT_DEAD, "DEAD", fscache_object_dead); |
caaef690 DH |
92 | |
93 | static WAIT_STATE(WAIT_FOR_INIT, "?INI", | |
94 | TRANSIT_TO(INIT_OBJECT, 1 << FSCACHE_OBJECT_EV_NEW_CHILD)); | |
95 | ||
96 | static WAIT_STATE(WAIT_FOR_PARENT, "?PRN", | |
97 | TRANSIT_TO(PARENT_READY, 1 << FSCACHE_OBJECT_EV_PARENT_READY)); | |
98 | ||
99 | static WAIT_STATE(WAIT_FOR_CMD, "?CMD", | |
100 | TRANSIT_TO(INVALIDATE_OBJECT, 1 << FSCACHE_OBJECT_EV_INVALIDATE), | |
101 | TRANSIT_TO(UPDATE_OBJECT, 1 << FSCACHE_OBJECT_EV_UPDATE), | |
102 | TRANSIT_TO(JUMPSTART_DEPS, 1 << FSCACHE_OBJECT_EV_NEW_CHILD)); | |
103 | ||
104 | static WAIT_STATE(WAIT_FOR_CLEARANCE, "?CLR", | |
105 | TRANSIT_TO(KILL_OBJECT, 1 << FSCACHE_OBJECT_EV_CLEARED)); | |
106 | ||
107 | /* | |
108 | * Out-of-band event transition tables. These are for handling unexpected | |
109 | * events, such as an I/O error. If an OOB event occurs, the state machine | |
110 | * clears and disables the event and forces a transition to the nominated work | |
111 | * state (acurrently executing work states will complete first). | |
112 | * | |
113 | * In such a situation, object->state remembers the state the machine should | |
114 | * have been in/gone to and returning NO_TRANSIT returns to that. | |
115 | */ | |
116 | static const struct fscache_transition fscache_osm_init_oob[] = { | |
117 | TRANSIT_TO(ABORT_INIT, | |
118 | (1 << FSCACHE_OBJECT_EV_ERROR) | | |
119 | (1 << FSCACHE_OBJECT_EV_KILL)), | |
120 | { 0, NULL } | |
36c95590 | 121 | }; |
caaef690 DH |
122 | |
123 | static const struct fscache_transition fscache_osm_lookup_oob[] = { | |
124 | TRANSIT_TO(LOOKUP_FAILURE, | |
125 | (1 << FSCACHE_OBJECT_EV_ERROR) | | |
126 | (1 << FSCACHE_OBJECT_EV_KILL)), | |
127 | { 0, NULL } | |
128 | }; | |
129 | ||
130 | static const struct fscache_transition fscache_osm_run_oob[] = { | |
131 | TRANSIT_TO(KILL_OBJECT, | |
132 | (1 << FSCACHE_OBJECT_EV_ERROR) | | |
133 | (1 << FSCACHE_OBJECT_EV_KILL)), | |
134 | { 0, NULL } | |
440f0aff DH |
135 | }; |
136 | ||
a18feb55 DH |
137 | static int fscache_get_object(struct fscache_object *, |
138 | enum fscache_obj_ref_trace); | |
139 | static void fscache_put_object(struct fscache_object *, | |
140 | enum fscache_obj_ref_trace); | |
caaef690 | 141 | static bool fscache_enqueue_dependents(struct fscache_object *, int); |
36c95590 | 142 | static void fscache_dequeue_object(struct fscache_object *); |
402cb8dd | 143 | static void fscache_update_aux_data(struct fscache_object *); |
36c95590 | 144 | |
36c95590 DH |
145 | /* |
146 | * we need to notify the parent when an op completes that we had outstanding | |
147 | * upon it | |
148 | */ | |
149 | static inline void fscache_done_parent_op(struct fscache_object *object) | |
150 | { | |
151 | struct fscache_object *parent = object->parent; | |
152 | ||
153 | _enter("OBJ%x {OBJ%x,%x}", | |
154 | object->debug_id, parent->debug_id, parent->n_ops); | |
155 | ||
156 | spin_lock_nested(&parent->lock, 1); | |
36c95590 | 157 | parent->n_obj_ops--; |
1362729b | 158 | parent->n_ops--; |
36c95590 DH |
159 | if (parent->n_ops == 0) |
160 | fscache_raise_event(parent, FSCACHE_OBJECT_EV_CLEARED); | |
161 | spin_unlock(&parent->lock); | |
162 | } | |
163 | ||
ef778e7a | 164 | /* |
caaef690 | 165 | * Object state machine dispatcher. |
ef778e7a | 166 | */ |
caaef690 | 167 | static void fscache_object_sm_dispatcher(struct fscache_object *object) |
ef778e7a | 168 | { |
caaef690 DH |
169 | const struct fscache_transition *t; |
170 | const struct fscache_state *state, *new_state; | |
171 | unsigned long events, event_mask; | |
a18feb55 | 172 | bool oob; |
caaef690 | 173 | int event = -1; |
36c95590 DH |
174 | |
175 | ASSERT(object != NULL); | |
176 | ||
177 | _enter("{OBJ%x,%s,%lx}", | |
caaef690 DH |
178 | object->debug_id, object->state->name, object->events); |
179 | ||
180 | event_mask = object->event_mask; | |
181 | restart: | |
182 | object->event_mask = 0; /* Mask normal event handling */ | |
183 | state = object->state; | |
184 | restart_masked: | |
185 | events = object->events; | |
186 | ||
187 | /* Handle any out-of-band events (typically an error) */ | |
188 | if (events & object->oob_event_mask) { | |
189 | _debug("{OBJ%x} oob %lx", | |
190 | object->debug_id, events & object->oob_event_mask); | |
a18feb55 | 191 | oob = true; |
caaef690 DH |
192 | for (t = object->oob_table; t->events; t++) { |
193 | if (events & t->events) { | |
194 | state = t->transit_to; | |
195 | ASSERT(state->work != NULL); | |
196 | event = fls(events & t->events) - 1; | |
197 | __clear_bit(event, &object->oob_event_mask); | |
198 | clear_bit(event, &object->events); | |
199 | goto execute_work_state; | |
200 | } | |
d461d26d | 201 | } |
caaef690 | 202 | } |
a18feb55 | 203 | oob = false; |
36c95590 | 204 | |
caaef690 DH |
205 | /* Wait states are just transition tables */ |
206 | if (!state->work) { | |
207 | if (events & event_mask) { | |
208 | for (t = state->transitions; t->events; t++) { | |
209 | if (events & t->events) { | |
210 | new_state = t->transit_to; | |
211 | event = fls(events & t->events) - 1; | |
a18feb55 DH |
212 | trace_fscache_osm(object, state, |
213 | true, false, event); | |
caaef690 DH |
214 | clear_bit(event, &object->events); |
215 | _debug("{OBJ%x} ev %d: %s -> %s", | |
216 | object->debug_id, event, | |
217 | state->name, new_state->name); | |
218 | object->state = state = new_state; | |
219 | goto execute_work_state; | |
220 | } | |
221 | } | |
36c95590 | 222 | |
caaef690 DH |
223 | /* The event mask didn't include all the tabled bits */ |
224 | BUG(); | |
36c95590 | 225 | } |
caaef690 DH |
226 | /* Randomly woke up */ |
227 | goto unmask_events; | |
36c95590 DH |
228 | } |
229 | ||
caaef690 DH |
230 | execute_work_state: |
231 | _debug("{OBJ%x} exec %s", object->debug_id, state->name); | |
36c95590 | 232 | |
a18feb55 | 233 | trace_fscache_osm(object, state, false, oob, event); |
caaef690 DH |
234 | new_state = state->work(object, event); |
235 | event = -1; | |
236 | if (new_state == NO_TRANSIT) { | |
237 | _debug("{OBJ%x} %s notrans", object->debug_id, state->name); | |
e26bfebd DH |
238 | if (unlikely(state == STATE(OBJECT_DEAD))) { |
239 | _leave(" [dead]"); | |
240 | return; | |
241 | } | |
caaef690 DH |
242 | fscache_enqueue_object(object); |
243 | event_mask = object->oob_event_mask; | |
244 | goto unmask_events; | |
36c95590 DH |
245 | } |
246 | ||
caaef690 DH |
247 | _debug("{OBJ%x} %s -> %s", |
248 | object->debug_id, state->name, new_state->name); | |
249 | object->state = state = new_state; | |
36c95590 | 250 | |
caaef690 | 251 | if (state->work) { |
e26bfebd | 252 | if (unlikely(state == STATE(OBJECT_DEAD))) { |
caaef690 DH |
253 | _leave(" [dead]"); |
254 | return; | |
255 | } | |
256 | goto restart_masked; | |
257 | } | |
36c95590 | 258 | |
caaef690 DH |
259 | /* Transited to wait state */ |
260 | event_mask = object->oob_event_mask; | |
261 | for (t = state->transitions; t->events; t++) | |
262 | event_mask |= t->events; | |
263 | ||
264 | unmask_events: | |
265 | object->event_mask = event_mask; | |
266 | smp_mb(); | |
267 | events = object->events; | |
268 | if (events & event_mask) | |
269 | goto restart; | |
270 | _leave(" [msk %lx]", event_mask); | |
36c95590 DH |
271 | } |
272 | ||
273 | /* | |
274 | * execute an object | |
275 | */ | |
610be24e | 276 | static void fscache_object_work_func(struct work_struct *work) |
36c95590 DH |
277 | { |
278 | struct fscache_object *object = | |
279 | container_of(work, struct fscache_object, work); | |
280 | unsigned long start; | |
281 | ||
282 | _enter("{OBJ%x}", object->debug_id); | |
283 | ||
36c95590 | 284 | start = jiffies; |
caaef690 | 285 | fscache_object_sm_dispatcher(object); |
36c95590 | 286 | fscache_hist(fscache_objs_histogram, start); |
a18feb55 | 287 | fscache_put_object(object, fscache_obj_put_work); |
36c95590 | 288 | } |
610be24e DH |
289 | |
290 | /** | |
291 | * fscache_object_init - Initialise a cache object description | |
292 | * @object: Object description | |
293 | * @cookie: Cookie object will be attached to | |
294 | * @cache: Cache in which backing object will be found | |
295 | * | |
296 | * Initialise a cache object description to its basic values. | |
297 | * | |
298 | * See Documentation/filesystems/caching/backend-api.txt for a complete | |
299 | * description. | |
300 | */ | |
301 | void fscache_object_init(struct fscache_object *object, | |
302 | struct fscache_cookie *cookie, | |
303 | struct fscache_cache *cache) | |
304 | { | |
caaef690 DH |
305 | const struct fscache_transition *t; |
306 | ||
610be24e DH |
307 | atomic_inc(&cache->object_count); |
308 | ||
caaef690 DH |
309 | object->state = STATE(WAIT_FOR_INIT); |
310 | object->oob_table = fscache_osm_init_oob; | |
311 | object->flags = 1 << FSCACHE_OBJECT_IS_LIVE; | |
610be24e DH |
312 | spin_lock_init(&object->lock); |
313 | INIT_LIST_HEAD(&object->cache_link); | |
314 | INIT_HLIST_NODE(&object->cookie_link); | |
315 | INIT_WORK(&object->work, fscache_object_work_func); | |
316 | INIT_LIST_HEAD(&object->dependents); | |
317 | INIT_LIST_HEAD(&object->dep_link); | |
318 | INIT_LIST_HEAD(&object->pending_ops); | |
319 | object->n_children = 0; | |
320 | object->n_ops = object->n_in_progress = object->n_exclusive = 0; | |
caaef690 | 321 | object->events = 0; |
610be24e DH |
322 | object->store_limit = 0; |
323 | object->store_limit_l = 0; | |
324 | object->cache = cache; | |
325 | object->cookie = cookie; | |
f29507ce | 326 | fscache_cookie_get(cookie, fscache_cookie_get_attach_object); |
610be24e | 327 | object->parent = NULL; |
7026f192 DH |
328 | #ifdef CONFIG_FSCACHE_OBJECT_LIST |
329 | RB_CLEAR_NODE(&object->objlist_link); | |
330 | #endif | |
caaef690 DH |
331 | |
332 | object->oob_event_mask = 0; | |
333 | for (t = object->oob_table; t->events; t++) | |
334 | object->oob_event_mask |= t->events; | |
335 | object->event_mask = object->oob_event_mask; | |
336 | for (t = object->state->transitions; t->events; t++) | |
337 | object->event_mask |= t->events; | |
610be24e DH |
338 | } |
339 | EXPORT_SYMBOL(fscache_object_init); | |
440f0aff | 340 | |
f09b443d DH |
341 | /* |
342 | * Mark the object as no longer being live, making sure that we synchronise | |
343 | * against op submission. | |
344 | */ | |
345 | static inline void fscache_mark_object_dead(struct fscache_object *object) | |
346 | { | |
347 | spin_lock(&object->lock); | |
348 | clear_bit(FSCACHE_OBJECT_IS_LIVE, &object->flags); | |
349 | spin_unlock(&object->lock); | |
350 | } | |
351 | ||
caaef690 DH |
352 | /* |
353 | * Abort object initialisation before we start it. | |
354 | */ | |
355 | static const struct fscache_state *fscache_abort_initialisation(struct fscache_object *object, | |
356 | int event) | |
357 | { | |
caaef690 DH |
358 | _enter("{OBJ%x},%d", object->debug_id, event); |
359 | ||
360 | object->oob_event_mask = 0; | |
caaef690 | 361 | fscache_dequeue_object(object); |
caaef690 DH |
362 | return transit_to(KILL_OBJECT); |
363 | } | |
364 | ||
36c95590 DH |
365 | /* |
366 | * initialise an object | |
367 | * - check the specified object's parent to see if we can make use of it | |
368 | * immediately to do a creation | |
369 | * - we may need to start the process of creating a parent and we need to wait | |
370 | * for the parent's lookup and creation to complete if it's not there yet | |
36c95590 | 371 | */ |
caaef690 DH |
372 | static const struct fscache_state *fscache_initialise_object(struct fscache_object *object, |
373 | int event) | |
36c95590 DH |
374 | { |
375 | struct fscache_object *parent; | |
caaef690 | 376 | bool success; |
36c95590 | 377 | |
caaef690 | 378 | _enter("{OBJ%x},%d", object->debug_id, event); |
36c95590 | 379 | |
caaef690 | 380 | ASSERT(list_empty(&object->dep_link)); |
36c95590 DH |
381 | |
382 | parent = object->parent; | |
383 | if (!parent) { | |
caaef690 | 384 | _leave(" [no parent]"); |
1362729b | 385 | return transit_to(DROP_OBJECT); |
caaef690 | 386 | } |
36c95590 | 387 | |
1362729b | 388 | _debug("parent: %s of:%lx", parent->state->name, parent->flags); |
caaef690 DH |
389 | |
390 | if (fscache_object_is_dying(parent)) { | |
391 | _leave(" [bad parent]"); | |
1362729b | 392 | return transit_to(DROP_OBJECT); |
caaef690 DH |
393 | } |
394 | ||
395 | if (fscache_object_is_available(parent)) { | |
396 | _leave(" [ready]"); | |
397 | return transit_to(PARENT_READY); | |
398 | } | |
399 | ||
400 | _debug("wait"); | |
401 | ||
402 | spin_lock(&parent->lock); | |
403 | fscache_stat(&fscache_n_cop_grab_object); | |
404 | success = false; | |
405 | if (fscache_object_is_live(parent) && | |
a18feb55 | 406 | object->cache->ops->grab_object(object, fscache_obj_get_add_to_deps)) { |
caaef690 DH |
407 | list_add(&object->dep_link, &parent->dependents); |
408 | success = true; | |
409 | } | |
410 | fscache_stat_d(&fscache_n_cop_grab_object); | |
411 | spin_unlock(&parent->lock); | |
412 | if (!success) { | |
413 | _leave(" [grab failed]"); | |
1362729b | 414 | return transit_to(DROP_OBJECT); |
36c95590 DH |
415 | } |
416 | ||
caaef690 DH |
417 | /* fscache_acquire_non_index_cookie() uses this |
418 | * to wake the chain up */ | |
419 | fscache_raise_event(parent, FSCACHE_OBJECT_EV_NEW_CHILD); | |
420 | _leave(" [wait]"); | |
421 | return transit_to(WAIT_FOR_PARENT); | |
422 | } | |
423 | ||
424 | /* | |
425 | * Once the parent object is ready, we should kick off our lookup op. | |
426 | */ | |
427 | static const struct fscache_state *fscache_parent_ready(struct fscache_object *object, | |
428 | int event) | |
429 | { | |
430 | struct fscache_object *parent = object->parent; | |
431 | ||
432 | _enter("{OBJ%x},%d", object->debug_id, event); | |
433 | ||
434 | ASSERT(parent != NULL); | |
435 | ||
436 | spin_lock(&parent->lock); | |
437 | parent->n_ops++; | |
438 | parent->n_obj_ops++; | |
439 | object->lookup_jif = jiffies; | |
440 | spin_unlock(&parent->lock); | |
441 | ||
36c95590 | 442 | _leave(""); |
caaef690 | 443 | return transit_to(LOOK_UP_OBJECT); |
36c95590 DH |
444 | } |
445 | ||
446 | /* | |
447 | * look an object up in the cache from which it was allocated | |
448 | * - we hold an "access lock" on the parent object, so the parent object cannot | |
449 | * be withdrawn by either party till we've finished | |
36c95590 | 450 | */ |
caaef690 DH |
451 | static const struct fscache_state *fscache_look_up_object(struct fscache_object *object, |
452 | int event) | |
36c95590 DH |
453 | { |
454 | struct fscache_cookie *cookie = object->cookie; | |
caaef690 | 455 | struct fscache_object *parent = object->parent; |
fee096de | 456 | int ret; |
36c95590 | 457 | |
caaef690 DH |
458 | _enter("{OBJ%x},%d", object->debug_id, event); |
459 | ||
460 | object->oob_table = fscache_osm_lookup_oob; | |
36c95590 | 461 | |
36c95590 DH |
462 | ASSERT(parent != NULL); |
463 | ASSERTCMP(parent->n_ops, >, 0); | |
464 | ASSERTCMP(parent->n_obj_ops, >, 0); | |
465 | ||
466 | /* make sure the parent is still available */ | |
493f7bc1 | 467 | ASSERT(fscache_object_is_available(parent)); |
36c95590 | 468 | |
493f7bc1 | 469 | if (fscache_object_is_dying(parent) || |
1362729b DH |
470 | test_bit(FSCACHE_IOERROR, &object->cache->flags) || |
471 | !fscache_use_cookie(object)) { | |
caaef690 DH |
472 | _leave(" [unavailable]"); |
473 | return transit_to(LOOKUP_FAILURE); | |
36c95590 DH |
474 | } |
475 | ||
1362729b DH |
476 | _debug("LOOKUP \"%s\" in \"%s\"", |
477 | cookie->def->name, object->cache->tag->name); | |
36c95590 DH |
478 | |
479 | fscache_stat(&fscache_n_object_lookups); | |
52bd75fd | 480 | fscache_stat(&fscache_n_cop_lookup_object); |
fee096de | 481 | ret = object->cache->ops->lookup_object(object); |
52bd75fd | 482 | fscache_stat_d(&fscache_n_cop_lookup_object); |
36c95590 | 483 | |
1362729b | 484 | fscache_unuse_cookie(object); |
36c95590 | 485 | |
fee096de DH |
486 | if (ret == -ETIMEDOUT) { |
487 | /* probably stuck behind another object, so move this one to | |
488 | * the back of the queue */ | |
489 | fscache_stat(&fscache_n_object_lookups_timed_out); | |
caaef690 DH |
490 | _leave(" [timeout]"); |
491 | return NO_TRANSIT; | |
fee096de DH |
492 | } |
493 | ||
caaef690 DH |
494 | if (ret < 0) { |
495 | _leave(" [error]"); | |
496 | return transit_to(LOOKUP_FAILURE); | |
497 | } | |
498 | ||
499 | _leave(" [ok]"); | |
500 | return transit_to(OBJECT_AVAILABLE); | |
36c95590 DH |
501 | } |
502 | ||
503 | /** | |
504 | * fscache_object_lookup_negative - Note negative cookie lookup | |
505 | * @object: Object pointing to cookie to mark | |
506 | * | |
507 | * Note negative lookup, permitting those waiting to read data from an already | |
508 | * existing backing object to continue as there's no data for them to read. | |
509 | */ | |
510 | void fscache_object_lookup_negative(struct fscache_object *object) | |
511 | { | |
512 | struct fscache_cookie *cookie = object->cookie; | |
513 | ||
caaef690 | 514 | _enter("{OBJ%x,%s}", object->debug_id, object->state->name); |
36c95590 | 515 | |
caaef690 | 516 | if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) { |
36c95590 DH |
517 | fscache_stat(&fscache_n_object_lookups_negative); |
518 | ||
caaef690 DH |
519 | /* Allow write requests to begin stacking up and read requests to begin |
520 | * returning ENODATA. | |
521 | */ | |
36c95590 | 522 | set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); |
94d30ae9 | 523 | clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); |
36c95590 DH |
524 | |
525 | _debug("wake up lookup %p", &cookie->flags); | |
caaef690 | 526 | clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); |
36c95590 | 527 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); |
36c95590 | 528 | } |
36c95590 DH |
529 | _leave(""); |
530 | } | |
531 | EXPORT_SYMBOL(fscache_object_lookup_negative); | |
532 | ||
533 | /** | |
534 | * fscache_obtained_object - Note successful object lookup or creation | |
535 | * @object: Object pointing to cookie to mark | |
536 | * | |
537 | * Note successful lookup and/or creation, permitting those waiting to write | |
538 | * data to a backing object to continue. | |
539 | * | |
540 | * Note that after calling this, an object's cookie may be relinquished by the | |
541 | * netfs, and so must be accessed with object lock held. | |
542 | */ | |
543 | void fscache_obtained_object(struct fscache_object *object) | |
544 | { | |
545 | struct fscache_cookie *cookie = object->cookie; | |
546 | ||
caaef690 | 547 | _enter("{OBJ%x,%s}", object->debug_id, object->state->name); |
36c95590 DH |
548 | |
549 | /* if we were still looking up, then we must have a positive lookup | |
550 | * result, in which case there may be data available */ | |
caaef690 | 551 | if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) { |
36c95590 DH |
552 | fscache_stat(&fscache_n_object_lookups_positive); |
553 | ||
caaef690 DH |
554 | /* We do (presumably) have data */ |
555 | clear_bit_unlock(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); | |
94d30ae9 | 556 | clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); |
36c95590 | 557 | |
caaef690 DH |
558 | /* Allow write requests to begin stacking up and read requests |
559 | * to begin shovelling data. | |
560 | */ | |
561 | clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); | |
36c95590 | 562 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); |
36c95590 | 563 | } else { |
36c95590 | 564 | fscache_stat(&fscache_n_object_created); |
36c95590 DH |
565 | } |
566 | ||
caaef690 | 567 | set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags); |
36c95590 DH |
568 | _leave(""); |
569 | } | |
570 | EXPORT_SYMBOL(fscache_obtained_object); | |
571 | ||
572 | /* | |
573 | * handle an object that has just become available | |
574 | */ | |
caaef690 DH |
575 | static const struct fscache_state *fscache_object_available(struct fscache_object *object, |
576 | int event) | |
36c95590 | 577 | { |
caaef690 DH |
578 | _enter("{OBJ%x},%d", object->debug_id, event); |
579 | ||
580 | object->oob_table = fscache_osm_run_oob; | |
36c95590 DH |
581 | |
582 | spin_lock(&object->lock); | |
583 | ||
36c95590 DH |
584 | fscache_done_parent_op(object); |
585 | if (object->n_in_progress == 0) { | |
586 | if (object->n_ops > 0) { | |
587 | ASSERTCMP(object->n_ops, >=, object->n_obj_ops); | |
36c95590 DH |
588 | fscache_start_operations(object); |
589 | } else { | |
590 | ASSERT(list_empty(&object->pending_ops)); | |
591 | } | |
592 | } | |
593 | spin_unlock(&object->lock); | |
594 | ||
52bd75fd | 595 | fscache_stat(&fscache_n_cop_lookup_complete); |
36c95590 | 596 | object->cache->ops->lookup_complete(object); |
52bd75fd | 597 | fscache_stat_d(&fscache_n_cop_lookup_complete); |
36c95590 DH |
598 | |
599 | fscache_hist(fscache_obj_instantiate_histogram, object->lookup_jif); | |
600 | fscache_stat(&fscache_n_object_avail); | |
601 | ||
602 | _leave(""); | |
caaef690 | 603 | return transit_to(JUMPSTART_DEPS); |
36c95590 DH |
604 | } |
605 | ||
606 | /* | |
caaef690 | 607 | * Wake up this object's dependent objects now that we've become available. |
36c95590 | 608 | */ |
caaef690 DH |
609 | static const struct fscache_state *fscache_jumpstart_dependents(struct fscache_object *object, |
610 | int event) | |
36c95590 | 611 | { |
caaef690 | 612 | _enter("{OBJ%x},%d", object->debug_id, event); |
36c95590 | 613 | |
caaef690 DH |
614 | if (!fscache_enqueue_dependents(object, FSCACHE_OBJECT_EV_PARENT_READY)) |
615 | return NO_TRANSIT; /* Not finished; requeue */ | |
616 | return transit_to(WAIT_FOR_CMD); | |
617 | } | |
36c95590 | 618 | |
caaef690 DH |
619 | /* |
620 | * Handle lookup or creation failute. | |
621 | */ | |
622 | static const struct fscache_state *fscache_lookup_failure(struct fscache_object *object, | |
623 | int event) | |
624 | { | |
625 | struct fscache_cookie *cookie; | |
6897e3df | 626 | |
caaef690 | 627 | _enter("{OBJ%x},%d", object->debug_id, event); |
36c95590 | 628 | |
caaef690 | 629 | object->oob_event_mask = 0; |
36c95590 | 630 | |
caaef690 DH |
631 | fscache_stat(&fscache_n_cop_lookup_complete); |
632 | object->cache->ops->lookup_complete(object); | |
633 | fscache_stat_d(&fscache_n_cop_lookup_complete); | |
36c95590 | 634 | |
6515d1db DH |
635 | set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags); |
636 | ||
caaef690 DH |
637 | cookie = object->cookie; |
638 | set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); | |
1362729b | 639 | if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) |
caaef690 | 640 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); |
36c95590 | 641 | |
caaef690 DH |
642 | fscache_done_parent_op(object); |
643 | return transit_to(KILL_OBJECT); | |
644 | } | |
645 | ||
646 | /* | |
647 | * Wait for completion of all active operations on this object and the death of | |
648 | * all child objects of this object. | |
649 | */ | |
650 | static const struct fscache_state *fscache_kill_object(struct fscache_object *object, | |
651 | int event) | |
652 | { | |
653 | _enter("{OBJ%x,%d,%d},%d", | |
654 | object->debug_id, object->n_ops, object->n_children, event); | |
655 | ||
f09b443d | 656 | fscache_mark_object_dead(object); |
1362729b | 657 | object->oob_event_mask = 0; |
caaef690 | 658 | |
6bdded59 DH |
659 | if (test_bit(FSCACHE_OBJECT_RETIRED, &object->flags)) { |
660 | /* Reject any new read/write ops and abort any that are pending. */ | |
661 | clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); | |
662 | fscache_cancel_all_ops(object); | |
663 | } | |
664 | ||
caaef690 DH |
665 | if (list_empty(&object->dependents) && |
666 | object->n_ops == 0 && | |
667 | object->n_children == 0) | |
1362729b | 668 | return transit_to(DROP_OBJECT); |
caaef690 | 669 | |
1362729b DH |
670 | if (object->n_in_progress == 0) { |
671 | spin_lock(&object->lock); | |
672 | if (object->n_ops > 0 && object->n_in_progress == 0) | |
673 | fscache_start_operations(object); | |
674 | spin_unlock(&object->lock); | |
675 | } | |
caaef690 DH |
676 | |
677 | if (!list_empty(&object->dependents)) | |
678 | return transit_to(KILL_DEPENDENTS); | |
679 | ||
680 | return transit_to(WAIT_FOR_CLEARANCE); | |
36c95590 DH |
681 | } |
682 | ||
683 | /* | |
caaef690 | 684 | * Kill dependent objects. |
36c95590 | 685 | */ |
caaef690 DH |
686 | static const struct fscache_state *fscache_kill_dependents(struct fscache_object *object, |
687 | int event) | |
36c95590 | 688 | { |
caaef690 | 689 | _enter("{OBJ%x},%d", object->debug_id, event); |
36c95590 | 690 | |
caaef690 DH |
691 | if (!fscache_enqueue_dependents(object, FSCACHE_OBJECT_EV_KILL)) |
692 | return NO_TRANSIT; /* Not finished */ | |
693 | return transit_to(WAIT_FOR_CLEARANCE); | |
36c95590 DH |
694 | } |
695 | ||
36c95590 | 696 | /* |
caaef690 DH |
697 | * Drop an object's attachments |
698 | */ | |
699 | static const struct fscache_state *fscache_drop_object(struct fscache_object *object, | |
700 | int event) | |
36c95590 | 701 | { |
caaef690 | 702 | struct fscache_object *parent = object->parent; |
1362729b | 703 | struct fscache_cookie *cookie = object->cookie; |
caaef690 | 704 | struct fscache_cache *cache = object->cache; |
1362729b | 705 | bool awaken = false; |
36c95590 | 706 | |
caaef690 | 707 | _enter("{OBJ%x,%d},%d", object->debug_id, object->n_children, event); |
36c95590 | 708 | |
1362729b DH |
709 | ASSERT(cookie != NULL); |
710 | ASSERT(!hlist_unhashed(&object->cookie_link)); | |
711 | ||
402cb8dd DH |
712 | if (test_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags)) { |
713 | _debug("final update"); | |
714 | fscache_update_aux_data(object); | |
715 | } | |
716 | ||
1362729b DH |
717 | /* Make sure the cookie no longer points here and that the netfs isn't |
718 | * waiting for us. | |
719 | */ | |
720 | spin_lock(&cookie->lock); | |
721 | hlist_del_init(&object->cookie_link); | |
94d30ae9 DH |
722 | if (hlist_empty(&cookie->backing_objects) && |
723 | test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) | |
1362729b DH |
724 | awaken = true; |
725 | spin_unlock(&cookie->lock); | |
726 | ||
727 | if (awaken) | |
728 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); | |
c5a94f43 N |
729 | if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) |
730 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP); | |
731 | ||
caaef690 DH |
732 | |
733 | /* Prevent a race with our last child, which has to signal EV_CLEARED | |
734 | * before dropping our spinlock. | |
735 | */ | |
36c95590 | 736 | spin_lock(&object->lock); |
36c95590 DH |
737 | spin_unlock(&object->lock); |
738 | ||
caaef690 DH |
739 | /* Discard from the cache's collection of objects */ |
740 | spin_lock(&cache->object_list_lock); | |
741 | list_del_init(&object->cache_link); | |
742 | spin_unlock(&cache->object_list_lock); | |
743 | ||
744 | fscache_stat(&fscache_n_cop_drop_object); | |
745 | cache->ops->drop_object(object); | |
746 | fscache_stat_d(&fscache_n_cop_drop_object); | |
747 | ||
748 | /* The parent object wants to know when all it dependents have gone */ | |
749 | if (parent) { | |
750 | _debug("release parent OBJ%x {%d}", | |
751 | parent->debug_id, parent->n_children); | |
752 | ||
753 | spin_lock(&parent->lock); | |
754 | parent->n_children--; | |
755 | if (parent->n_children == 0) | |
756 | fscache_raise_event(parent, FSCACHE_OBJECT_EV_CLEARED); | |
757 | spin_unlock(&parent->lock); | |
758 | object->parent = NULL; | |
759 | } | |
760 | ||
761 | /* this just shifts the object release to the work processor */ | |
a18feb55 | 762 | fscache_put_object(object, fscache_obj_put_drop_obj); |
caaef690 | 763 | fscache_stat(&fscache_n_object_dead); |
36c95590 DH |
764 | |
765 | _leave(""); | |
caaef690 | 766 | return transit_to(OBJECT_DEAD); |
36c95590 DH |
767 | } |
768 | ||
769 | /* | |
8b8edefa | 770 | * get a ref on an object |
36c95590 | 771 | */ |
a18feb55 DH |
772 | static int fscache_get_object(struct fscache_object *object, |
773 | enum fscache_obj_ref_trace why) | |
36c95590 | 774 | { |
52bd75fd | 775 | int ret; |
36c95590 | 776 | |
52bd75fd | 777 | fscache_stat(&fscache_n_cop_grab_object); |
a18feb55 | 778 | ret = object->cache->ops->grab_object(object, why) ? 0 : -EAGAIN; |
52bd75fd DH |
779 | fscache_stat_d(&fscache_n_cop_grab_object); |
780 | return ret; | |
36c95590 DH |
781 | } |
782 | ||
783 | /* | |
caaef690 | 784 | * Discard a ref on an object |
36c95590 | 785 | */ |
a18feb55 DH |
786 | static void fscache_put_object(struct fscache_object *object, |
787 | enum fscache_obj_ref_trace why) | |
36c95590 | 788 | { |
52bd75fd | 789 | fscache_stat(&fscache_n_cop_put_object); |
a18feb55 | 790 | object->cache->ops->put_object(object, why); |
52bd75fd | 791 | fscache_stat_d(&fscache_n_cop_put_object); |
36c95590 DH |
792 | } |
793 | ||
1362729b DH |
794 | /** |
795 | * fscache_object_destroy - Note that a cache object is about to be destroyed | |
796 | * @object: The object to be destroyed | |
797 | * | |
798 | * Note the imminent destruction and deallocation of a cache object record. | |
799 | */ | |
800 | void fscache_object_destroy(struct fscache_object *object) | |
801 | { | |
802 | fscache_objlist_remove(object); | |
803 | ||
804 | /* We can get rid of the cookie now */ | |
a18feb55 | 805 | fscache_cookie_put(object->cookie, fscache_cookie_put_object); |
1362729b DH |
806 | object->cookie = NULL; |
807 | } | |
808 | EXPORT_SYMBOL(fscache_object_destroy); | |
809 | ||
36c95590 DH |
810 | /* |
811 | * enqueue an object for metadata-type processing | |
812 | */ | |
813 | void fscache_enqueue_object(struct fscache_object *object) | |
814 | { | |
815 | _enter("{OBJ%x}", object->debug_id); | |
816 | ||
a18feb55 | 817 | if (fscache_get_object(object, fscache_obj_get_queue) >= 0) { |
8b8edefa TH |
818 | wait_queue_head_t *cong_wq = |
819 | &get_cpu_var(fscache_object_cong_wait); | |
820 | ||
821 | if (queue_work(fscache_object_wq, &object->work)) { | |
822 | if (fscache_object_congested()) | |
823 | wake_up(cong_wq); | |
824 | } else | |
a18feb55 | 825 | fscache_put_object(object, fscache_obj_put_queue); |
8b8edefa TH |
826 | |
827 | put_cpu_var(fscache_object_cong_wait); | |
828 | } | |
829 | } | |
830 | ||
831 | /** | |
832 | * fscache_object_sleep_till_congested - Sleep until object wq is congested | |
caaef690 | 833 | * @timeoutp: Scheduler sleep timeout |
8b8edefa TH |
834 | * |
835 | * Allow an object handler to sleep until the object workqueue is congested. | |
836 | * | |
837 | * The caller must set up a wake up event before calling this and must have set | |
838 | * the appropriate sleep mode (such as TASK_UNINTERRUPTIBLE) and tested its own | |
839 | * condition before calling this function as no test is made here. | |
840 | * | |
841 | * %true is returned if the object wq is congested, %false otherwise. | |
842 | */ | |
843 | bool fscache_object_sleep_till_congested(signed long *timeoutp) | |
844 | { | |
170d800a | 845 | wait_queue_head_t *cong_wq = this_cpu_ptr(&fscache_object_cong_wait); |
8b8edefa TH |
846 | DEFINE_WAIT(wait); |
847 | ||
848 | if (fscache_object_congested()) | |
849 | return true; | |
850 | ||
851 | add_wait_queue_exclusive(cong_wq, &wait); | |
852 | if (!fscache_object_congested()) | |
853 | *timeoutp = schedule_timeout(*timeoutp); | |
854 | finish_wait(cong_wq, &wait); | |
855 | ||
856 | return fscache_object_congested(); | |
36c95590 | 857 | } |
8b8edefa | 858 | EXPORT_SYMBOL_GPL(fscache_object_sleep_till_congested); |
36c95590 DH |
859 | |
860 | /* | |
caaef690 DH |
861 | * Enqueue the dependents of an object for metadata-type processing. |
862 | * | |
863 | * If we don't manage to finish the list before the scheduler wants to run | |
864 | * again then return false immediately. We return true if the list was | |
865 | * cleared. | |
36c95590 | 866 | */ |
caaef690 | 867 | static bool fscache_enqueue_dependents(struct fscache_object *object, int event) |
36c95590 DH |
868 | { |
869 | struct fscache_object *dep; | |
caaef690 | 870 | bool ret = true; |
36c95590 DH |
871 | |
872 | _enter("{OBJ%x}", object->debug_id); | |
873 | ||
874 | if (list_empty(&object->dependents)) | |
caaef690 | 875 | return true; |
36c95590 DH |
876 | |
877 | spin_lock(&object->lock); | |
878 | ||
879 | while (!list_empty(&object->dependents)) { | |
880 | dep = list_entry(object->dependents.next, | |
881 | struct fscache_object, dep_link); | |
882 | list_del_init(&dep->dep_link); | |
883 | ||
caaef690 | 884 | fscache_raise_event(dep, event); |
a18feb55 | 885 | fscache_put_object(dep, fscache_obj_put_enq_dep); |
36c95590 | 886 | |
caaef690 DH |
887 | if (!list_empty(&object->dependents) && need_resched()) { |
888 | ret = false; | |
889 | break; | |
890 | } | |
36c95590 DH |
891 | } |
892 | ||
893 | spin_unlock(&object->lock); | |
caaef690 | 894 | return ret; |
36c95590 DH |
895 | } |
896 | ||
897 | /* | |
898 | * remove an object from whatever queue it's waiting on | |
36c95590 | 899 | */ |
caaef690 | 900 | static void fscache_dequeue_object(struct fscache_object *object) |
36c95590 DH |
901 | { |
902 | _enter("{OBJ%x}", object->debug_id); | |
903 | ||
904 | if (!list_empty(&object->dep_link)) { | |
905 | spin_lock(&object->parent->lock); | |
906 | list_del_init(&object->dep_link); | |
907 | spin_unlock(&object->parent->lock); | |
908 | } | |
909 | ||
910 | _leave(""); | |
911 | } | |
912 | ||
913 | /** | |
914 | * fscache_check_aux - Ask the netfs whether an object on disk is still valid | |
915 | * @object: The object to ask about | |
916 | * @data: The auxiliary data for the object | |
917 | * @datalen: The size of the auxiliary data | |
918 | * | |
1362729b DH |
919 | * This function consults the netfs about the coherency state of an object. |
920 | * The caller must be holding a ref on cookie->n_active (held by | |
921 | * fscache_look_up_object() on behalf of the cache backend during object lookup | |
922 | * and creation). | |
36c95590 DH |
923 | */ |
924 | enum fscache_checkaux fscache_check_aux(struct fscache_object *object, | |
ee1235a9 DH |
925 | const void *data, uint16_t datalen, |
926 | loff_t object_size) | |
36c95590 DH |
927 | { |
928 | enum fscache_checkaux result; | |
929 | ||
930 | if (!object->cookie->def->check_aux) { | |
931 | fscache_stat(&fscache_n_checkaux_none); | |
932 | return FSCACHE_CHECKAUX_OKAY; | |
933 | } | |
934 | ||
935 | result = object->cookie->def->check_aux(object->cookie->netfs_data, | |
ee1235a9 | 936 | data, datalen, object_size); |
36c95590 DH |
937 | switch (result) { |
938 | /* entry okay as is */ | |
939 | case FSCACHE_CHECKAUX_OKAY: | |
940 | fscache_stat(&fscache_n_checkaux_okay); | |
941 | break; | |
942 | ||
943 | /* entry requires update */ | |
944 | case FSCACHE_CHECKAUX_NEEDS_UPDATE: | |
945 | fscache_stat(&fscache_n_checkaux_update); | |
946 | break; | |
947 | ||
948 | /* entry requires deletion */ | |
949 | case FSCACHE_CHECKAUX_OBSOLETE: | |
950 | fscache_stat(&fscache_n_checkaux_obsolete); | |
951 | break; | |
952 | ||
953 | default: | |
954 | BUG(); | |
955 | } | |
956 | ||
957 | return result; | |
958 | } | |
959 | EXPORT_SYMBOL(fscache_check_aux); | |
ef778e7a DH |
960 | |
961 | /* | |
962 | * Asynchronously invalidate an object. | |
963 | */ | |
caaef690 DH |
964 | static const struct fscache_state *_fscache_invalidate_object(struct fscache_object *object, |
965 | int event) | |
ef778e7a DH |
966 | { |
967 | struct fscache_operation *op; | |
968 | struct fscache_cookie *cookie = object->cookie; | |
969 | ||
caaef690 DH |
970 | _enter("{OBJ%x},%d", object->debug_id, event); |
971 | ||
1362729b DH |
972 | /* We're going to need the cookie. If the cookie is not available then |
973 | * retire the object instead. | |
974 | */ | |
975 | if (!fscache_use_cookie(object)) { | |
e5a95541 | 976 | ASSERT(radix_tree_empty(&object->cookie->stores)); |
94d30ae9 | 977 | set_bit(FSCACHE_OBJECT_RETIRED, &object->flags); |
1362729b DH |
978 | _leave(" [no cookie]"); |
979 | return transit_to(KILL_OBJECT); | |
980 | } | |
ef778e7a DH |
981 | |
982 | /* Reject any new read/write ops and abort any that are pending. */ | |
983 | fscache_invalidate_writes(cookie); | |
984 | clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); | |
985 | fscache_cancel_all_ops(object); | |
986 | ||
987 | /* Now we have to wait for in-progress reads and writes */ | |
988 | op = kzalloc(sizeof(*op), GFP_KERNEL); | |
1362729b DH |
989 | if (!op) |
990 | goto nomem; | |
ef778e7a | 991 | |
08c2e3d0 | 992 | fscache_operation_init(cookie, op, object->cache->ops->invalidate_object, |
d3b97ca4 | 993 | NULL, NULL); |
1362729b DH |
994 | op->flags = FSCACHE_OP_ASYNC | |
995 | (1 << FSCACHE_OP_EXCLUSIVE) | | |
996 | (1 << FSCACHE_OP_UNUSE_COOKIE); | |
08c2e3d0 | 997 | trace_fscache_page_op(cookie, NULL, op, fscache_page_op_invalidate); |
ef778e7a DH |
998 | |
999 | spin_lock(&cookie->lock); | |
1000 | if (fscache_submit_exclusive_op(object, op) < 0) | |
8d76349d | 1001 | goto submit_op_failed; |
ef778e7a DH |
1002 | spin_unlock(&cookie->lock); |
1003 | fscache_put_operation(op); | |
1004 | ||
1005 | /* Once we've completed the invalidation, we know there will be no data | |
1006 | * stored in the cache and thus we can reinstate the data-check-skip | |
1007 | * optimisation. | |
1008 | */ | |
1009 | set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); | |
1010 | ||
1011 | /* We can allow read and write requests to come in once again. They'll | |
1012 | * queue up behind our exclusive invalidation operation. | |
1013 | */ | |
caaef690 DH |
1014 | if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) |
1015 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); | |
1016 | _leave(" [ok]"); | |
1017 | return transit_to(UPDATE_OBJECT); | |
8d76349d | 1018 | |
1362729b | 1019 | nomem: |
f09b443d | 1020 | fscache_mark_object_dead(object); |
1362729b DH |
1021 | fscache_unuse_cookie(object); |
1022 | _leave(" [ENOMEM]"); | |
1023 | return transit_to(KILL_OBJECT); | |
1024 | ||
8d76349d | 1025 | submit_op_failed: |
f09b443d | 1026 | fscache_mark_object_dead(object); |
8d76349d | 1027 | spin_unlock(&cookie->lock); |
920bce20 | 1028 | fscache_unuse_cookie(object); |
8d76349d | 1029 | kfree(op); |
8d76349d | 1030 | _leave(" [EIO]"); |
caaef690 DH |
1031 | return transit_to(KILL_OBJECT); |
1032 | } | |
1033 | ||
1034 | static const struct fscache_state *fscache_invalidate_object(struct fscache_object *object, | |
1035 | int event) | |
1036 | { | |
1037 | const struct fscache_state *s; | |
1038 | ||
1039 | fscache_stat(&fscache_n_invalidates_run); | |
1040 | fscache_stat(&fscache_n_cop_invalidate_object); | |
1041 | s = _fscache_invalidate_object(object, event); | |
1042 | fscache_stat_d(&fscache_n_cop_invalidate_object); | |
1043 | return s; | |
1044 | } | |
1045 | ||
402cb8dd DH |
1046 | /* |
1047 | * Update auxiliary data. | |
1048 | */ | |
1049 | static void fscache_update_aux_data(struct fscache_object *object) | |
1050 | { | |
1051 | fscache_stat(&fscache_n_updates_run); | |
1052 | fscache_stat(&fscache_n_cop_update_object); | |
1053 | object->cache->ops->update_object(object); | |
1054 | fscache_stat_d(&fscache_n_cop_update_object); | |
1055 | } | |
1056 | ||
caaef690 DH |
1057 | /* |
1058 | * Asynchronously update an object. | |
1059 | */ | |
1060 | static const struct fscache_state *fscache_update_object(struct fscache_object *object, | |
1061 | int event) | |
1062 | { | |
1063 | _enter("{OBJ%x},%d", object->debug_id, event); | |
1064 | ||
402cb8dd | 1065 | fscache_update_aux_data(object); |
caaef690 DH |
1066 | |
1067 | _leave(""); | |
1068 | return transit_to(WAIT_FOR_CMD); | |
ef778e7a | 1069 | } |
182d919b DH |
1070 | |
1071 | /** | |
1072 | * fscache_object_retrying_stale - Note retrying stale object | |
1073 | * @object: The object that will be retried | |
1074 | * | |
1075 | * Note that an object lookup found an on-disk object that was adjudged to be | |
1076 | * stale and has been deleted. The lookup will be retried. | |
1077 | */ | |
1078 | void fscache_object_retrying_stale(struct fscache_object *object) | |
1079 | { | |
1080 | fscache_stat(&fscache_n_cache_no_space_reject); | |
1081 | } | |
1082 | EXPORT_SYMBOL(fscache_object_retrying_stale); | |
1083 | ||
1084 | /** | |
1085 | * fscache_object_mark_killed - Note that an object was killed | |
1086 | * @object: The object that was culled | |
1087 | * @why: The reason the object was killed. | |
1088 | * | |
1089 | * Note that an object was killed. Returns true if the object was | |
1090 | * already marked killed, false if it wasn't. | |
1091 | */ | |
1092 | void fscache_object_mark_killed(struct fscache_object *object, | |
1093 | enum fscache_why_object_killed why) | |
1094 | { | |
1095 | if (test_and_set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags)) { | |
1096 | pr_err("Error: Object already killed by cache [%s]\n", | |
1097 | object->cache->identifier); | |
1098 | return; | |
1099 | } | |
1100 | ||
1101 | switch (why) { | |
1102 | case FSCACHE_OBJECT_NO_SPACE: | |
1103 | fscache_stat(&fscache_n_cache_no_space_reject); | |
1104 | break; | |
1105 | case FSCACHE_OBJECT_IS_STALE: | |
1106 | fscache_stat(&fscache_n_cache_stale_objects); | |
1107 | break; | |
1108 | case FSCACHE_OBJECT_WAS_RETIRED: | |
1109 | fscache_stat(&fscache_n_cache_retired_objects); | |
1110 | break; | |
1111 | case FSCACHE_OBJECT_WAS_CULLED: | |
1112 | fscache_stat(&fscache_n_cache_culled_objects); | |
1113 | break; | |
1114 | } | |
1115 | } | |
1116 | EXPORT_SYMBOL(fscache_object_mark_killed); | |
e26bfebd DH |
1117 | |
1118 | /* | |
1119 | * The object is dead. We can get here if an object gets queued by an event | |
1120 | * that would lead to its death (such as EV_KILL) when the dispatcher is | |
1121 | * already running (and so can be requeued) but hasn't yet cleared the event | |
1122 | * mask. | |
1123 | */ | |
1124 | static const struct fscache_state *fscache_object_dead(struct fscache_object *object, | |
1125 | int event) | |
1126 | { | |
1127 | if (!test_and_set_bit(FSCACHE_OBJECT_RUN_AFTER_DEAD, | |
1128 | &object->flags)) | |
1129 | return NO_TRANSIT; | |
1130 | ||
1131 | WARN(true, "FS-Cache object redispatched after death"); | |
1132 | return NO_TRANSIT; | |
1133 | } |