]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #include "ImageSync.h" | |
31f18b77 | 5 | #include "InstanceWatcher.h" |
7c673cae | 6 | #include "ProgressContext.h" |
11fdf7f2 TL |
7 | #include "common/debug.h" |
8 | #include "common/Timer.h" | |
7c673cae | 9 | #include "common/errno.h" |
11fdf7f2 | 10 | #include "librbd/DeepCopyRequest.h" |
7c673cae | 11 | #include "librbd/ImageCtx.h" |
11fdf7f2 | 12 | #include "librbd/ImageState.h" |
7c673cae | 13 | #include "librbd/Utils.h" |
11fdf7f2 | 14 | #include "librbd/internal.h" |
1911f103 | 15 | #include "librbd/deep_copy/Handler.h" |
9f95a23c | 16 | #include "tools/rbd_mirror/Threads.h" |
7c673cae FG |
17 | #include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" |
18 | #include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" | |
9f95a23c | 19 | #include "tools/rbd_mirror/image_sync/Types.h" |
7c673cae FG |
20 | |
21 | #define dout_context g_ceph_context | |
22 | #define dout_subsys ceph_subsys_rbd_mirror | |
23 | #undef dout_prefix | |
24 | #define dout_prefix *_dout << "rbd::mirror::ImageSync: " \ | |
25 | << this << " " << __func__ | |
26 | ||
27 | namespace rbd { | |
28 | namespace mirror { | |
29 | ||
30 | using namespace image_sync; | |
28e407b8 | 31 | using librbd::util::create_async_context_callback; |
7c673cae FG |
32 | using librbd::util::create_context_callback; |
33 | using librbd::util::unique_lock_name; | |
34 | ||
11fdf7f2 | 35 | template <typename I> |
1911f103 TL |
36 | class ImageSync<I>::ImageCopyProgressHandler |
37 | : public librbd::deep_copy::NoOpHandler { | |
11fdf7f2 | 38 | public: |
1911f103 | 39 | ImageCopyProgressHandler(ImageSync *image_sync) : image_sync(image_sync) { |
11fdf7f2 TL |
40 | } |
41 | ||
42 | int update_progress(uint64_t object_no, uint64_t object_count) override { | |
43 | image_sync->handle_copy_image_update_progress(object_no, object_count); | |
44 | return 0; | |
45 | } | |
46 | ||
47 | ImageSync *image_sync; | |
48 | }; | |
49 | ||
7c673cae | 50 | template <typename I> |
9f95a23c TL |
51 | ImageSync<I>::ImageSync( |
52 | Threads<I>* threads, | |
53 | I *local_image_ctx, | |
54 | I *remote_image_ctx, | |
55 | const std::string &local_mirror_uuid, | |
56 | image_sync::SyncPointHandler* sync_point_handler, | |
57 | InstanceWatcher<I> *instance_watcher, | |
58 | ProgressContext *progress_ctx, | |
59 | Context *on_finish) | |
60 | : CancelableRequest("rbd::mirror::ImageSync", local_image_ctx->cct, | |
61 | on_finish), | |
62 | m_threads(threads), | |
63 | m_local_image_ctx(local_image_ctx), | |
64 | m_remote_image_ctx(remote_image_ctx), | |
65 | m_local_mirror_uuid(local_mirror_uuid), | |
66 | m_sync_point_handler(sync_point_handler), | |
67 | m_instance_watcher(instance_watcher), | |
31f18b77 | 68 | m_progress_ctx(progress_ctx), |
9f95a23c TL |
69 | m_lock(ceph::make_mutex(unique_lock_name("ImageSync::m_lock", this))), |
70 | m_update_sync_point_interval( | |
71 | m_local_image_ctx->cct->_conf.template get_val<double>( | |
72 | "rbd_mirror_sync_point_update_age")) { | |
7c673cae FG |
73 | } |
74 | ||
75 | template <typename I> | |
76 | ImageSync<I>::~ImageSync() { | |
11fdf7f2 | 77 | ceph_assert(m_image_copy_request == nullptr); |
1911f103 | 78 | ceph_assert(m_image_copy_prog_handler == nullptr); |
11fdf7f2 | 79 | ceph_assert(m_update_sync_ctx == nullptr); |
7c673cae FG |
80 | } |
81 | ||
82 | template <typename I> | |
83 | void ImageSync<I>::send() { | |
31f18b77 | 84 | send_notify_sync_request(); |
7c673cae FG |
85 | } |
86 | ||
87 | template <typename I> | |
88 | void ImageSync<I>::cancel() { | |
9f95a23c | 89 | std::lock_guard locker{m_lock}; |
7c673cae | 90 | |
11fdf7f2 | 91 | dout(10) << dendl; |
7c673cae FG |
92 | |
93 | m_canceled = true; | |
94 | ||
31f18b77 FG |
95 | if (m_instance_watcher->cancel_sync_request(m_local_image_ctx->id)) { |
96 | return; | |
97 | } | |
98 | ||
7c673cae FG |
99 | if (m_image_copy_request != nullptr) { |
100 | m_image_copy_request->cancel(); | |
101 | } | |
102 | } | |
103 | ||
31f18b77 FG |
104 | template <typename I> |
105 | void ImageSync<I>::send_notify_sync_request() { | |
106 | update_progress("NOTIFY_SYNC_REQUEST"); | |
107 | ||
11fdf7f2 | 108 | dout(10) << dendl; |
31f18b77 | 109 | |
9f95a23c | 110 | m_lock.lock(); |
28e407b8 | 111 | if (m_canceled) { |
9f95a23c TL |
112 | m_lock.unlock(); |
113 | CancelableRequest::finish(-ECANCELED); | |
28e407b8 AA |
114 | return; |
115 | } | |
116 | ||
117 | Context *ctx = create_async_context_callback( | |
9f95a23c | 118 | m_threads->work_queue, create_context_callback< |
28e407b8 | 119 | ImageSync<I>, &ImageSync<I>::handle_notify_sync_request>(this)); |
31f18b77 | 120 | m_instance_watcher->notify_sync_request(m_local_image_ctx->id, ctx); |
9f95a23c | 121 | m_lock.unlock(); |
31f18b77 FG |
122 | } |
123 | ||
124 | template <typename I> | |
125 | void ImageSync<I>::handle_notify_sync_request(int r) { | |
11fdf7f2 | 126 | dout(10) << ": r=" << r << dendl; |
31f18b77 | 127 | |
9f95a23c | 128 | m_lock.lock(); |
28e407b8 AA |
129 | if (r == 0 && m_canceled) { |
130 | r = -ECANCELED; | |
131 | } | |
9f95a23c | 132 | m_lock.unlock(); |
28e407b8 | 133 | |
31f18b77 | 134 | if (r < 0) { |
9f95a23c | 135 | CancelableRequest::finish(r); |
31f18b77 FG |
136 | return; |
137 | } | |
138 | ||
139 | send_prune_catch_up_sync_point(); | |
140 | } | |
141 | ||
7c673cae FG |
142 | template <typename I> |
143 | void ImageSync<I>::send_prune_catch_up_sync_point() { | |
144 | update_progress("PRUNE_CATCH_UP_SYNC_POINT"); | |
145 | ||
9f95a23c | 146 | if (m_sync_point_handler->get_sync_points().empty()) { |
7c673cae FG |
147 | send_create_sync_point(); |
148 | return; | |
149 | } | |
150 | ||
11fdf7f2 | 151 | dout(10) << dendl; |
7c673cae FG |
152 | |
153 | // prune will remove sync points with missing snapshots and | |
154 | // ensure we have a maximum of one sync point (in case we | |
155 | // restarted) | |
156 | Context *ctx = create_context_callback< | |
157 | ImageSync<I>, &ImageSync<I>::handle_prune_catch_up_sync_point>(this); | |
158 | SyncPointPruneRequest<I> *request = SyncPointPruneRequest<I>::create( | |
9f95a23c | 159 | m_remote_image_ctx, false, m_sync_point_handler, ctx); |
7c673cae FG |
160 | request->send(); |
161 | } | |
162 | ||
163 | template <typename I> | |
164 | void ImageSync<I>::handle_prune_catch_up_sync_point(int r) { | |
11fdf7f2 | 165 | dout(10) << ": r=" << r << dendl; |
7c673cae FG |
166 | |
167 | if (r < 0) { | |
168 | derr << ": failed to prune catch-up sync point: " | |
169 | << cpp_strerror(r) << dendl; | |
170 | finish(r); | |
171 | return; | |
172 | } | |
173 | ||
174 | send_create_sync_point(); | |
175 | } | |
176 | ||
177 | template <typename I> | |
178 | void ImageSync<I>::send_create_sync_point() { | |
179 | update_progress("CREATE_SYNC_POINT"); | |
180 | ||
181 | // TODO: when support for disconnecting laggy clients is added, | |
182 | // re-connect and create catch-up sync point | |
9f95a23c | 183 | if (!m_sync_point_handler->get_sync_points().empty()) { |
11fdf7f2 | 184 | send_copy_image(); |
7c673cae FG |
185 | return; |
186 | } | |
187 | ||
11fdf7f2 | 188 | dout(10) << dendl; |
7c673cae FG |
189 | |
190 | Context *ctx = create_context_callback< | |
191 | ImageSync<I>, &ImageSync<I>::handle_create_sync_point>(this); | |
192 | SyncPointCreateRequest<I> *request = SyncPointCreateRequest<I>::create( | |
9f95a23c | 193 | m_remote_image_ctx, m_local_mirror_uuid, m_sync_point_handler, ctx); |
7c673cae FG |
194 | request->send(); |
195 | } | |
196 | ||
197 | template <typename I> | |
198 | void ImageSync<I>::handle_create_sync_point(int r) { | |
11fdf7f2 | 199 | dout(10) << ": r=" << r << dendl; |
7c673cae FG |
200 | |
201 | if (r < 0) { | |
202 | derr << ": failed to create sync point: " << cpp_strerror(r) | |
203 | << dendl; | |
204 | finish(r); | |
205 | return; | |
206 | } | |
207 | ||
11fdf7f2 | 208 | send_copy_image(); |
7c673cae FG |
209 | } |
210 | ||
211 | template <typename I> | |
11fdf7f2 TL |
212 | void ImageSync<I>::send_copy_image() { |
213 | librados::snap_t snap_id_start = 0; | |
214 | librados::snap_t snap_id_end; | |
215 | librbd::deep_copy::ObjectNumber object_number; | |
216 | int r = 0; | |
9f95a23c TL |
217 | |
218 | m_snap_seqs_copy = m_sync_point_handler->get_snap_seqs(); | |
219 | m_sync_points_copy = m_sync_point_handler->get_sync_points(); | |
220 | ceph_assert(!m_sync_points_copy.empty()); | |
221 | auto &sync_point = m_sync_points_copy.front(); | |
222 | ||
7c673cae | 223 | { |
9f95a23c | 224 | std::shared_lock image_locker{m_remote_image_ctx->image_lock}; |
11fdf7f2 TL |
225 | snap_id_end = m_remote_image_ctx->get_snap_id( |
226 | cls::rbd::UserSnapshotNamespace(), sync_point.snap_name); | |
227 | if (snap_id_end == CEPH_NOSNAP) { | |
228 | derr << ": failed to locate snapshot: " << sync_point.snap_name << dendl; | |
229 | r = -ENOENT; | |
230 | } else if (!sync_point.from_snap_name.empty()) { | |
231 | snap_id_start = m_remote_image_ctx->get_snap_id( | |
232 | cls::rbd::UserSnapshotNamespace(), sync_point.from_snap_name); | |
233 | if (snap_id_start == CEPH_NOSNAP) { | |
234 | derr << ": failed to locate from snapshot: " | |
235 | << sync_point.from_snap_name << dendl; | |
236 | r = -ENOENT; | |
237 | } | |
7c673cae | 238 | } |
11fdf7f2 | 239 | object_number = sync_point.object_number; |
7c673cae | 240 | } |
11fdf7f2 | 241 | if (r < 0) { |
7c673cae FG |
242 | finish(r); |
243 | return; | |
244 | } | |
245 | ||
9f95a23c | 246 | m_lock.lock(); |
7c673cae | 247 | if (m_canceled) { |
9f95a23c | 248 | m_lock.unlock(); |
7c673cae FG |
249 | finish(-ECANCELED); |
250 | return; | |
251 | } | |
252 | ||
11fdf7f2 | 253 | dout(10) << dendl; |
7c673cae FG |
254 | |
255 | Context *ctx = create_context_callback< | |
256 | ImageSync<I>, &ImageSync<I>::handle_copy_image>(this); | |
1911f103 | 257 | m_image_copy_prog_handler = new ImageCopyProgressHandler(this); |
11fdf7f2 TL |
258 | m_image_copy_request = librbd::DeepCopyRequest<I>::create( |
259 | m_remote_image_ctx, m_local_image_ctx, snap_id_start, snap_id_end, | |
9f95a23c | 260 | 0, false, object_number, m_threads->work_queue, &m_snap_seqs_copy, |
1911f103 | 261 | m_image_copy_prog_handler, ctx); |
7c673cae | 262 | m_image_copy_request->get(); |
9f95a23c | 263 | m_lock.unlock(); |
7c673cae FG |
264 | |
265 | update_progress("COPY_IMAGE"); | |
266 | ||
267 | m_image_copy_request->send(); | |
268 | } | |
269 | ||
270 | template <typename I> | |
271 | void ImageSync<I>::handle_copy_image(int r) { | |
11fdf7f2 | 272 | dout(10) << ": r=" << r << dendl; |
7c673cae FG |
273 | |
274 | { | |
9f95a23c | 275 | std::scoped_lock locker{m_threads->timer_lock, m_lock}; |
7c673cae FG |
276 | m_image_copy_request->put(); |
277 | m_image_copy_request = nullptr; | |
1911f103 TL |
278 | delete m_image_copy_prog_handler; |
279 | m_image_copy_prog_handler = nullptr; | |
7c673cae FG |
280 | if (r == 0 && m_canceled) { |
281 | r = -ECANCELED; | |
282 | } | |
11fdf7f2 TL |
283 | |
284 | if (m_update_sync_ctx != nullptr) { | |
9f95a23c | 285 | m_threads->timer->cancel_event(m_update_sync_ctx); |
11fdf7f2 TL |
286 | m_update_sync_ctx = nullptr; |
287 | } | |
288 | ||
289 | if (m_updating_sync_point) { | |
290 | m_ret_val = r; | |
291 | return; | |
292 | } | |
7c673cae FG |
293 | } |
294 | ||
295 | if (r == -ECANCELED) { | |
296 | dout(10) << ": image copy canceled" << dendl; | |
297 | finish(r); | |
298 | return; | |
299 | } else if (r < 0) { | |
300 | derr << ": failed to copy image: " << cpp_strerror(r) << dendl; | |
301 | finish(r); | |
302 | return; | |
303 | } | |
304 | ||
11fdf7f2 | 305 | send_flush_sync_point(); |
7c673cae FG |
306 | } |
307 | ||
308 | template <typename I> | |
11fdf7f2 TL |
309 | void ImageSync<I>::handle_copy_image_update_progress(uint64_t object_no, |
310 | uint64_t object_count) { | |
311 | int percent = 100 * object_no / object_count; | |
312 | update_progress("COPY_IMAGE " + stringify(percent) + "%"); | |
7c673cae | 313 | |
9f95a23c | 314 | std::lock_guard locker{m_lock}; |
11fdf7f2 TL |
315 | m_image_copy_object_no = object_no; |
316 | m_image_copy_object_count = object_count; | |
7c673cae | 317 | |
11fdf7f2 TL |
318 | if (m_update_sync_ctx == nullptr && !m_updating_sync_point) { |
319 | send_update_sync_point(); | |
320 | } | |
321 | } | |
7c673cae | 322 | |
11fdf7f2 TL |
323 | template <typename I> |
324 | void ImageSync<I>::send_update_sync_point() { | |
9f95a23c | 325 | ceph_assert(ceph_mutex_is_locked(m_lock)); |
7c673cae | 326 | |
11fdf7f2 TL |
327 | m_update_sync_ctx = nullptr; |
328 | ||
329 | if (m_canceled) { | |
330 | return; | |
31f18b77 | 331 | } |
11fdf7f2 | 332 | |
9f95a23c TL |
333 | ceph_assert(!m_sync_points_copy.empty()); |
334 | auto sync_point = &m_sync_points_copy.front(); | |
11fdf7f2 | 335 | |
9f95a23c | 336 | if (sync_point->object_number && |
11fdf7f2 TL |
337 | (m_image_copy_object_no - 1) == sync_point->object_number.get()) { |
338 | // update sync point did not progress since last sync | |
31f18b77 FG |
339 | return; |
340 | } | |
341 | ||
11fdf7f2 TL |
342 | m_updating_sync_point = true; |
343 | ||
11fdf7f2 TL |
344 | if (m_image_copy_object_no > 0) { |
345 | sync_point->object_number = m_image_copy_object_no - 1; | |
346 | } | |
347 | ||
9f95a23c TL |
348 | auto ctx = create_context_callback< |
349 | ImageSync<I>, &ImageSync<I>::handle_update_sync_point>(this); | |
350 | m_sync_point_handler->update_sync_points(m_snap_seqs_copy, | |
351 | m_sync_points_copy, false, ctx); | |
7c673cae FG |
352 | } |
353 | ||
354 | template <typename I> | |
11fdf7f2 TL |
355 | void ImageSync<I>::handle_update_sync_point(int r) { |
356 | CephContext *cct = m_local_image_ctx->cct; | |
357 | ldout(cct, 20) << ": r=" << r << dendl; | |
7c673cae | 358 | |
11fdf7f2 | 359 | { |
9f95a23c | 360 | std::scoped_lock locker{m_threads->timer_lock, m_lock}; |
11fdf7f2 TL |
361 | m_updating_sync_point = false; |
362 | ||
363 | if (m_image_copy_request != nullptr) { | |
9f95a23c | 364 | m_update_sync_ctx = new LambdaContext( |
11fdf7f2 | 365 | [this](int r) { |
9f95a23c | 366 | std::lock_guard locker{m_lock}; |
11fdf7f2 TL |
367 | this->send_update_sync_point(); |
368 | }); | |
9f95a23c TL |
369 | m_threads->timer->add_event_after( |
370 | m_update_sync_point_interval, m_update_sync_ctx); | |
11fdf7f2 TL |
371 | return; |
372 | } | |
373 | } | |
374 | ||
375 | send_flush_sync_point(); | |
7c673cae FG |
376 | } |
377 | ||
378 | template <typename I> | |
11fdf7f2 TL |
379 | void ImageSync<I>::send_flush_sync_point() { |
380 | if (m_ret_val < 0) { | |
381 | finish(m_ret_val); | |
382 | return; | |
383 | } | |
7c673cae | 384 | |
11fdf7f2 TL |
385 | update_progress("FLUSH_SYNC_POINT"); |
386 | ||
9f95a23c TL |
387 | ceph_assert(!m_sync_points_copy.empty()); |
388 | auto sync_point = &m_sync_points_copy.front(); | |
389 | ||
11fdf7f2 TL |
390 | if (m_image_copy_object_no > 0) { |
391 | sync_point->object_number = m_image_copy_object_no - 1; | |
392 | } else { | |
393 | sync_point->object_number = boost::none; | |
394 | } | |
395 | ||
9f95a23c TL |
396 | auto ctx = create_context_callback< |
397 | ImageSync<I>, &ImageSync<I>::handle_flush_sync_point>(this); | |
398 | m_sync_point_handler->update_sync_points(m_snap_seqs_copy, | |
399 | m_sync_points_copy, false, ctx); | |
7c673cae FG |
400 | } |
401 | ||
402 | template <typename I> | |
11fdf7f2 TL |
403 | void ImageSync<I>::handle_flush_sync_point(int r) { |
404 | dout(10) << ": r=" << r << dendl; | |
7c673cae | 405 | |
11fdf7f2 | 406 | if (r < 0) { |
11fdf7f2 TL |
407 | derr << ": failed to update client data: " << cpp_strerror(r) |
408 | << dendl; | |
409 | finish(r); | |
410 | return; | |
7c673cae | 411 | } |
7c673cae FG |
412 | |
413 | send_prune_sync_points(); | |
414 | } | |
415 | ||
416 | template <typename I> | |
417 | void ImageSync<I>::send_prune_sync_points() { | |
11fdf7f2 | 418 | dout(10) << dendl; |
7c673cae FG |
419 | |
420 | update_progress("PRUNE_SYNC_POINTS"); | |
421 | ||
422 | Context *ctx = create_context_callback< | |
423 | ImageSync<I>, &ImageSync<I>::handle_prune_sync_points>(this); | |
424 | SyncPointPruneRequest<I> *request = SyncPointPruneRequest<I>::create( | |
9f95a23c | 425 | m_remote_image_ctx, true, m_sync_point_handler, ctx); |
7c673cae FG |
426 | request->send(); |
427 | } | |
428 | ||
429 | template <typename I> | |
430 | void ImageSync<I>::handle_prune_sync_points(int r) { | |
11fdf7f2 | 431 | dout(10) << ": r=" << r << dendl; |
7c673cae FG |
432 | |
433 | if (r < 0) { | |
434 | derr << ": failed to prune sync point: " | |
435 | << cpp_strerror(r) << dendl; | |
436 | finish(r); | |
437 | return; | |
438 | } | |
439 | ||
9f95a23c | 440 | if (!m_sync_point_handler->get_sync_points().empty()) { |
7c673cae FG |
441 | send_copy_image(); |
442 | return; | |
443 | } | |
444 | ||
445 | finish(0); | |
446 | } | |
447 | ||
448 | template <typename I> | |
449 | void ImageSync<I>::update_progress(const std::string &description) { | |
450 | dout(20) << ": " << description << dendl; | |
451 | ||
452 | if (m_progress_ctx) { | |
453 | m_progress_ctx->update_progress("IMAGE_SYNC/" + description); | |
454 | } | |
455 | } | |
456 | ||
31f18b77 FG |
457 | template <typename I> |
458 | void ImageSync<I>::finish(int r) { | |
459 | dout(20) << ": r=" << r << dendl; | |
460 | ||
461 | m_instance_watcher->notify_sync_complete(m_local_image_ctx->id); | |
9f95a23c | 462 | CancelableRequest::finish(r); |
31f18b77 FG |
463 | } |
464 | ||
7c673cae FG |
465 | } // namespace mirror |
466 | } // namespace rbd | |
467 | ||
468 | template class rbd::mirror::ImageSync<librbd::ImageCtx>; |