]>
Commit | Line | Data |
---|---|---|
e7fd4179 DT |
1 | /****************************************************************************** |
2 | ******************************************************************************* | |
3 | ** | |
4 | ** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. | |
5 | ** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. | |
6 | ** | |
7 | ** This copyrighted material is made available to anyone wishing to use, | |
8 | ** modify, copy, or redistribute it subject to the terms and conditions | |
9 | ** of the GNU General Public License v.2. | |
10 | ** | |
11 | ******************************************************************************* | |
12 | ******************************************************************************/ | |
13 | ||
14 | #include "dlm_internal.h" | |
15 | #include "lockspace.h" | |
16 | #include "member.h" | |
17 | #include "dir.h" | |
18 | #include "ast.h" | |
19 | #include "recover.h" | |
20 | #include "lowcomms.h" | |
21 | #include "lock.h" | |
22 | #include "requestqueue.h" | |
23 | #include "recoverd.h" | |
24 | ||
25 | ||
26 | /* If the start for which we're re-enabling locking (seq) has been superseded | |
27 | by a newer stop (ls_recover_seq), we need to leave locking disabled. */ | |
28 | ||
29 | static int enable_locking(struct dlm_ls *ls, uint64_t seq) | |
30 | { | |
31 | int error = -EINTR; | |
32 | ||
33 | spin_lock(&ls->ls_recover_lock); | |
34 | if (ls->ls_recover_seq == seq) { | |
35 | set_bit(LSFL_RUNNING, &ls->ls_flags); | |
36 | up_write(&ls->ls_in_recovery); | |
37 | error = 0; | |
38 | } | |
39 | spin_unlock(&ls->ls_recover_lock); | |
40 | return error; | |
41 | } | |
42 | ||
43 | static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) | |
44 | { | |
45 | unsigned long start; | |
46 | int error, neg = 0; | |
47 | ||
48 | log_debug(ls, "recover %"PRIx64"", rv->seq); | |
49 | ||
50 | down(&ls->ls_recoverd_active); | |
51 | ||
52 | /* | |
53 | * Suspending and resuming dlm_astd ensures that no lkb's from this ls | |
54 | * will be processed by dlm_astd during recovery. | |
55 | */ | |
56 | ||
57 | dlm_astd_suspend(); | |
58 | dlm_astd_resume(); | |
59 | ||
60 | /* | |
61 | * This list of root rsb's will be the basis of most of the recovery | |
62 | * routines. | |
63 | */ | |
64 | ||
65 | dlm_create_root_list(ls); | |
66 | ||
67 | /* | |
68 | * Free all the tossed rsb's so we don't have to recover them. | |
69 | */ | |
70 | ||
71 | dlm_clear_toss_list(ls); | |
72 | ||
73 | /* | |
74 | * Add or remove nodes from the lockspace's ls_nodes list. | |
75 | * Also waits for all nodes to complete dlm_recover_members. | |
76 | */ | |
77 | ||
78 | error = dlm_recover_members(ls, rv, &neg); | |
79 | if (error) { | |
80 | log_error(ls, "recover_members failed %d", error); | |
81 | goto fail; | |
82 | } | |
83 | start = jiffies; | |
84 | ||
85 | /* | |
86 | * Rebuild our own share of the directory by collecting from all other | |
87 | * nodes their master rsb names that hash to us. | |
88 | */ | |
89 | ||
90 | error = dlm_recover_directory(ls); | |
91 | if (error) { | |
92 | log_error(ls, "recover_directory failed %d", error); | |
93 | goto fail; | |
94 | } | |
95 | ||
96 | /* | |
97 | * Purge directory-related requests that are saved in requestqueue. | |
98 | * All dir requests from before recovery are invalid now due to the dir | |
99 | * rebuild and will be resent by the requesting nodes. | |
100 | */ | |
101 | ||
102 | dlm_purge_requestqueue(ls); | |
103 | ||
104 | /* | |
105 | * Wait for all nodes to complete directory rebuild. | |
106 | */ | |
107 | ||
108 | error = dlm_recover_directory_wait(ls); | |
109 | if (error) { | |
110 | log_error(ls, "recover_directory_wait failed %d", error); | |
111 | goto fail; | |
112 | } | |
113 | ||
114 | /* | |
115 | * We may have outstanding operations that are waiting for a reply from | |
116 | * a failed node. Mark these to be resent after recovery. Unlock and | |
117 | * cancel ops can just be completed. | |
118 | */ | |
119 | ||
120 | dlm_recover_waiters_pre(ls); | |
121 | ||
122 | error = dlm_recovery_stopped(ls); | |
123 | if (error) | |
124 | goto fail; | |
125 | ||
126 | if (neg || dlm_no_directory(ls)) { | |
127 | /* | |
128 | * Clear lkb's for departed nodes. | |
129 | */ | |
130 | ||
131 | dlm_purge_locks(ls); | |
132 | ||
133 | /* | |
134 | * Get new master nodeid's for rsb's that were mastered on | |
135 | * departed nodes. | |
136 | */ | |
137 | ||
138 | error = dlm_recover_masters(ls); | |
139 | if (error) { | |
140 | log_error(ls, "recover_masters failed %d", error); | |
141 | goto fail; | |
142 | } | |
143 | ||
144 | /* | |
145 | * Send our locks on remastered rsb's to the new masters. | |
146 | */ | |
147 | ||
148 | error = dlm_recover_locks(ls); | |
149 | if (error) { | |
150 | log_error(ls, "recover_locks failed %d", error); | |
151 | goto fail; | |
152 | } | |
153 | ||
154 | error = dlm_recover_locks_wait(ls); | |
155 | if (error) { | |
156 | log_error(ls, "recover_locks_wait failed %d", error); | |
157 | goto fail; | |
158 | } | |
159 | ||
160 | /* | |
161 | * Finalize state in master rsb's now that all locks can be | |
162 | * checked. This includes conversion resolution and lvb | |
163 | * settings. | |
164 | */ | |
165 | ||
166 | dlm_recover_rsbs(ls); | |
167 | } | |
168 | ||
169 | dlm_release_root_list(ls); | |
170 | ||
171 | dlm_set_recover_status(ls, DLM_RS_DONE); | |
172 | error = dlm_recover_done_wait(ls); | |
173 | if (error) { | |
174 | log_error(ls, "recover_done_wait failed %d", error); | |
175 | goto fail; | |
176 | } | |
177 | ||
178 | dlm_clear_members_gone(ls); | |
179 | ||
180 | error = enable_locking(ls, rv->seq); | |
181 | if (error) { | |
182 | log_error(ls, "enable_locking failed %d", error); | |
183 | goto fail; | |
184 | } | |
185 | ||
186 | error = dlm_process_requestqueue(ls); | |
187 | if (error) { | |
188 | log_error(ls, "process_requestqueue failed %d", error); | |
189 | goto fail; | |
190 | } | |
191 | ||
192 | error = dlm_recover_waiters_post(ls); | |
193 | if (error) { | |
194 | log_error(ls, "recover_waiters_post failed %d", error); | |
195 | goto fail; | |
196 | } | |
197 | ||
198 | dlm_grant_after_purge(ls); | |
199 | ||
200 | dlm_astd_wake(); | |
201 | ||
202 | log_debug(ls, "recover %"PRIx64" done: %u ms", rv->seq, | |
203 | jiffies_to_msecs(jiffies - start)); | |
204 | up(&ls->ls_recoverd_active); | |
205 | ||
206 | return 0; | |
207 | ||
208 | fail: | |
209 | dlm_release_root_list(ls); | |
210 | log_debug(ls, "recover %"PRIx64" error %d", rv->seq, error); | |
211 | up(&ls->ls_recoverd_active); | |
212 | return error; | |
213 | } | |
214 | ||
215 | static void do_ls_recovery(struct dlm_ls *ls) | |
216 | { | |
217 | struct dlm_recover *rv = NULL; | |
218 | ||
219 | spin_lock(&ls->ls_recover_lock); | |
220 | rv = ls->ls_recover_args; | |
221 | ls->ls_recover_args = NULL; | |
222 | clear_bit(LSFL_RECOVERY_STOP, &ls->ls_flags); | |
223 | spin_unlock(&ls->ls_recover_lock); | |
224 | ||
225 | if (rv) { | |
226 | ls_recover(ls, rv); | |
227 | kfree(rv->nodeids); | |
228 | kfree(rv); | |
229 | } | |
230 | } | |
231 | ||
232 | static int dlm_recoverd(void *arg) | |
233 | { | |
234 | struct dlm_ls *ls; | |
235 | ||
236 | ls = dlm_find_lockspace_local(arg); | |
237 | ||
238 | while (!kthread_should_stop()) { | |
239 | set_current_state(TASK_INTERRUPTIBLE); | |
240 | if (!test_bit(LSFL_WORK, &ls->ls_flags)) | |
241 | schedule(); | |
242 | set_current_state(TASK_RUNNING); | |
243 | ||
244 | if (test_and_clear_bit(LSFL_WORK, &ls->ls_flags)) | |
245 | do_ls_recovery(ls); | |
246 | } | |
247 | ||
248 | dlm_put_lockspace(ls); | |
249 | return 0; | |
250 | } | |
251 | ||
252 | void dlm_recoverd_kick(struct dlm_ls *ls) | |
253 | { | |
254 | set_bit(LSFL_WORK, &ls->ls_flags); | |
255 | wake_up_process(ls->ls_recoverd_task); | |
256 | } | |
257 | ||
258 | int dlm_recoverd_start(struct dlm_ls *ls) | |
259 | { | |
260 | struct task_struct *p; | |
261 | int error = 0; | |
262 | ||
263 | p = kthread_run(dlm_recoverd, ls, "dlm_recoverd"); | |
264 | if (IS_ERR(p)) | |
265 | error = PTR_ERR(p); | |
266 | else | |
267 | ls->ls_recoverd_task = p; | |
268 | return error; | |
269 | } | |
270 | ||
271 | void dlm_recoverd_stop(struct dlm_ls *ls) | |
272 | { | |
273 | kthread_stop(ls->ls_recoverd_task); | |
274 | } | |
275 | ||
276 | void dlm_recoverd_suspend(struct dlm_ls *ls) | |
277 | { | |
278 | down(&ls->ls_recoverd_active); | |
279 | } | |
280 | ||
281 | void dlm_recoverd_resume(struct dlm_ls *ls) | |
282 | { | |
283 | up(&ls->ls_recoverd_active); | |
284 | } | |
285 |