ceph/src/librbd/api/Migration.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "librbd/api/Migration.h"
   5 #include "include/rados/librados.hpp"
   6 #include "include/stringify.h"
   7 #include "common/dout.h"
   8 #include "common/errno.h"
   9 #include "common/Cond.h"
  10 #include "cls/rbd/cls_rbd_client.h"
  11 #include "librbd/ExclusiveLock.h"
  12 #include "librbd/ImageCtx.h"
  13 #include "librbd/ImageState.h"
  14 #include "librbd/Operations.h"
  15 #include "librbd/Utils.h"
  16 #include "librbd/api/Config.h"
  17 #include "librbd/api/Group.h"
  18 #include "librbd/api/Image.h"
  19 #include "librbd/api/Snapshot.h"
  20 #include "librbd/api/Trash.h"
  21 #include "librbd/deep_copy/MetadataCopyRequest.h"
  22 #include "librbd/deep_copy/SnapshotCopyRequest.h"
  23 #include "librbd/exclusive_lock/Policy.h"
  24 #include "librbd/image/AttachChildRequest.h"
  25 #include "librbd/image/AttachParentRequest.h"
  26 #include "librbd/image/CloneRequest.h"
  27 #include "librbd/image/CreateRequest.h"
  28 #include "librbd/image/DetachChildRequest.h"
  29 #include "librbd/image/DetachParentRequest.h"
  30 #include "librbd/image/ListWatchersRequest.h"
  31 #include "librbd/image/RemoveRequest.h"
  32 #include "librbd/internal.h"
  33 #include "librbd/io/ImageRequestWQ.h"
  34 #include "librbd/mirror/DisableRequest.h"
  35 #include "librbd/mirror/EnableRequest.h"
  36
  37 #include <boost/scope_exit.hpp>
  38
  39 #define dout_subsys ceph_subsys_rbd
  40 #undef dout_prefix
  41 #define dout_prefix *_dout << "librbd::Migration: " << __func__ << ": "
  42
  43 namespace librbd {
  44
  45 inline bool operator==(const linked_image_spec_t& rhs,
  46                        const linked_image_spec_t& lhs) {
  47   bool result = (rhs.pool_id == lhs.pool_id &&
  48                  rhs.pool_namespace == lhs.pool_namespace &&
  49                  rhs.image_id == lhs.image_id);
  50   return result;
  51 }
  52
  53 namespace api {
  54
  55 using util::create_rados_callback;
  56
  57 namespace {
  58
  59 class MigrationProgressContext : public ProgressContext {
  60 public:
  61   MigrationProgressContext(librados::IoCtx& io_ctx,
  62                            const std::string &header_oid,
  63                            cls::rbd::MigrationState state,
  64                            ProgressContext *prog_ctx)
  65     : m_io_ctx(io_ctx), m_header_oid(header_oid), m_state(state),
  66       m_prog_ctx(prog_ctx), m_cct(reinterpret_cast<CephContext*>(io_ctx.cct())),
  67       m_lock(ceph::make_mutex(
  68         util::unique_lock_name("librbd::api::MigrationProgressContext",
  69                                this))) {
  70     ceph_assert(m_prog_ctx != nullptr);
  71   }
  72
  73   ~MigrationProgressContext() {
  74     wait_for_in_flight_updates();
  75   }
  76
  77   int update_progress(uint64_t offset, uint64_t total) override {
  78     ldout(m_cct, 20) << "offset=" << offset << ", total=" << total << dendl;
  79
  80     m_prog_ctx->update_progress(offset, total);
  81
  82     std::string description = stringify(offset * 100 / total) + "% complete";
  83
  84     send_state_description_update(description);
  85
  86     return 0;
  87   }
  88
  89 private:
  90   librados::IoCtx& m_io_ctx;
  91   std::string m_header_oid;
  92   cls::rbd::MigrationState m_state;
  93   ProgressContext *m_prog_ctx;
  94
  95   CephContext* m_cct;
  96   mutable ceph::mutex m_lock;
  97   ceph::condition_variable m_cond;
  98   std::string m_state_description;
  99   bool m_pending_update = false;
 100   int m_in_flight_state_updates = 0;
 101
 102   void send_state_description_update(const std::string &description) {
 103     std::lock_guard locker{m_lock};
 104
 105     if (description == m_state_description) {
 106       return;
 107     }
 108
 109     m_state_description = description;
 110
 111     if (m_in_flight_state_updates > 0) {
 112       m_pending_update = true;
 113       return;
 114     }
 115
 116     set_state_description();
 117   }
 118
 119   void set_state_description() {
 120     ldout(m_cct, 20) << "state_description=" << m_state_description << dendl;
 121
 122     ceph_assert(ceph_mutex_is_locked(m_lock));
 123
 124     librados::ObjectWriteOperation op;
 125     cls_client::migration_set_state(&op, m_state, m_state_description);
 126
 127     using klass = MigrationProgressContext;
 128     librados::AioCompletion *comp =
 129       create_rados_callback<klass, &klass::handle_set_state_description>(this);
 130     int r = m_io_ctx.aio_operate(m_header_oid, comp, &op);
 131     ceph_assert(r == 0);
 132     comp->release();
 133
 134     m_in_flight_state_updates++;
 135   }
 136
 137   void handle_set_state_description(int r) {
 138     ldout(m_cct, 20) << "r=" << r << dendl;
 139
 140     std::lock_guard locker{m_lock};
 141
 142     m_in_flight_state_updates--;
 143
 144     if (r < 0) {
 145       lderr(m_cct) << "failed to update migration state: " << cpp_strerror(r)
 146                    << dendl;
 147     } else if (m_pending_update) {
 148       set_state_description();
 149       m_pending_update = false;
 150     } else {
 151       m_cond.notify_all();
 152     }
 153   }
 154
 155   void wait_for_in_flight_updates() {
 156     std::unique_lock locker{m_lock};
 157
 158     ldout(m_cct, 20) << "m_in_flight_state_updates="
 159                      << m_in_flight_state_updates << dendl;
 160     m_pending_update = false;
 161     m_cond.wait(locker, [this] { return m_in_flight_state_updates <= 0; });
 162   }
 163 };
 164
 165 int trash_search(librados::IoCtx &io_ctx, rbd_trash_image_source_t source,
 166                  const std::string &image_name, std::string *image_id) {
 167   std::vector<trash_image_info_t> entries;
 168
 169   int r = Trash<>::list(io_ctx, entries, false);
 170   if (r < 0) {
 171     return r;
 172   }
 173
 174   for (auto &entry : entries) {
 175     if (entry.source == source && entry.name == image_name) {
 176       *image_id = entry.id;
 177       return 0;
 178     }
 179   }
 180
 181   return -ENOENT;
 182 }
 183
 184 template <typename I>
 185 int open_source_image(librados::IoCtx& io_ctx, const std::string &image_name,
 186                       I **src_image_ctx, librados::IoCtx *dst_io_ctx,
 187                       std::string *dst_image_name, std::string *dst_image_id,
 188                       bool *flatten, bool *mirroring,
 189                       cls::rbd::MirrorImageMode *mirror_image_mode,
 190                       cls::rbd::MigrationState *state,
 191                       std::string *state_description) {
 192   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 193
 194   librados::IoCtx src_io_ctx;
 195   std::string src_image_name;
 196   std::string src_image_id;
 197   cls::rbd::MigrationSpec migration_spec;
 198   I *image_ctx = I::create(image_name, "", nullptr, io_ctx, false);
 199
 200   ldout(cct, 10) << "trying to open image by name " << io_ctx.get_pool_name()
 201                  << "/" << image_name << dendl;
 202
 203   int r = image_ctx->state->open(OPEN_FLAG_IGNORE_MIGRATING);
 204   if (r < 0) {
 205     if (r != -ENOENT) {
 206       lderr(cct) << "failed to open image: " << cpp_strerror(r) << dendl;
 207       return r;
 208     }
 209     image_ctx = nullptr;
 210   }
 211
 212   BOOST_SCOPE_EXIT_TPL(&r, &image_ctx) {
 213     if (r != 0 && image_ctx != nullptr) {
 214       image_ctx->state->close();
 215     }
 216   } BOOST_SCOPE_EXIT_END;
 217
 218   if (r == 0) {
 219     // The opened image is either a source (then just proceed) or a
 220     // destination (then look for the source image id in the migration
 221     // header).
 222
 223     r = cls_client::migration_get(&image_ctx->md_ctx, image_ctx->header_oid,
 224                                   &migration_spec);
 225
 226     if (r < 0) {
 227       lderr(cct) << "failed retrieving migration header: " << cpp_strerror(r)
 228                  << dendl;
 229       return r;
 230     }
 231
 232     ldout(cct, 10) << "migration spec: " << migration_spec << dendl;
 233
 234     if (migration_spec.header_type != cls::rbd::MIGRATION_HEADER_TYPE_SRC &&
 235         migration_spec.header_type != cls::rbd::MIGRATION_HEADER_TYPE_DST) {
 236         lderr(cct) << "unexpected migration header type: "
 237                    << migration_spec.header_type << dendl;
 238         r = -EINVAL;
 239         return r;
 240     }
 241
 242     if (migration_spec.header_type == cls::rbd::MIGRATION_HEADER_TYPE_DST) {
 243       ldout(cct, 10) << "the destination image is opened" << dendl;
 244
 245       // Close and look for the source image.
 246       r = image_ctx->state->close();
 247       image_ctx = nullptr;
 248       if (r < 0) {
 249         lderr(cct) << "failed closing image: " << cpp_strerror(r)
 250                    << dendl;
 251         return r;
 252       }
 253
 254       r = util::create_ioctx(io_ctx, "source image", migration_spec.pool_id,
 255                              migration_spec.pool_namespace, &src_io_ctx);
 256       if (r < 0) {
 257         return r;
 258       }
 259
 260       src_image_name = migration_spec.image_name;
 261       src_image_id = migration_spec.image_id;
 262     } else {
 263       ldout(cct, 10) << "the source image is opened" << dendl;
 264     }
 265   } else {
 266     assert (r == -ENOENT);
 267
 268     ldout(cct, 10) << "source image is not found. Trying trash" << dendl;
 269
 270     r = trash_search(io_ctx, RBD_TRASH_IMAGE_SOURCE_MIGRATION, image_name,
 271                      &src_image_id);
 272     if (r < 0) {
 273       lderr(cct) << "failed to determine image id: " << cpp_strerror(r)
 274                  << dendl;
 275       return r;
 276     }
 277
 278     ldout(cct, 10) << "source image id from trash: " << src_image_id << dendl;
 279
 280     src_io_ctx.dup(io_ctx);
 281   }
 282
 283   if (image_ctx == nullptr) {
 284     int flags = OPEN_FLAG_IGNORE_MIGRATING;
 285
 286     if (src_image_id.empty()) {
 287       ldout(cct, 20) << "trying to open v1 image by name "
 288                      << src_io_ctx.get_pool_name() << "/" << src_image_name
 289                      << dendl;
 290
 291       flags |= OPEN_FLAG_OLD_FORMAT;
 292     } else {
 293       ldout(cct, 20) << "trying to open v2 image by id "
 294                      << src_io_ctx.get_pool_name() << "/" << src_image_id
 295                      << dendl;
 296     }
 297
 298     image_ctx = I::create(src_image_name, src_image_id, nullptr, src_io_ctx,
 299                           false);
 300     r = image_ctx->state->open(flags);
 301     if (r < 0) {
 302       lderr(cct) << "failed to open source image " << src_io_ctx.get_pool_name()
 303                  << "/" << (src_image_id.empty() ? src_image_name : src_image_id)
 304                  << ": " << cpp_strerror(r) << dendl;
 305       image_ctx = nullptr;
 306       return r;
 307     }
 308
 309     r = cls_client::migration_get(&image_ctx->md_ctx, image_ctx->header_oid,
 310                                   &migration_spec);
 311     if (r < 0) {
 312       lderr(cct) << "failed retrieving migration header: " << cpp_strerror(r)
 313                  << dendl;
 314       return r;
 315     }
 316
 317     ldout(cct, 20) << "migration spec: " << migration_spec << dendl;
 318   }
 319
 320   r = util::create_ioctx(image_ctx->md_ctx, "source image",
 321                          migration_spec.pool_id, migration_spec.pool_namespace,
 322                          dst_io_ctx);
 323   if (r < 0) {
 324     return r;
 325   }
 326
 327   *src_image_ctx = image_ctx;
 328   *dst_image_name = migration_spec.image_name;
 329   *dst_image_id = migration_spec.image_id;
 330   *flatten = migration_spec.flatten;
 331   *mirroring = migration_spec.mirroring;
 332   *mirror_image_mode = migration_spec.mirror_image_mode;
 333   *state = migration_spec.state;
 334   *state_description = migration_spec.state_description;
 335
 336   return 0;
 337 }
 338
 339 } // anonymous namespace
 340
 341 template <typename I>
 342 int Migration<I>::prepare(librados::IoCtx& io_ctx,
 343                           const std::string &image_name,
 344                           librados::IoCtx& dest_io_ctx,
 345                           const std::string &dest_image_name_,
 346                           ImageOptions& opts) {
 347   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 348
 349   std::string dest_image_name = dest_image_name_.empty() ? image_name :
 350     dest_image_name_;
 351
 352   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << " -> "
 353                  << dest_io_ctx.get_pool_name() << "/" << dest_image_name
 354                  << ", opts=" << opts << dendl;
 355
 356   auto image_ctx = I::create(image_name, "", nullptr, io_ctx, false);
 357   int r = image_ctx->state->open(0);
 358   if (r < 0) {
 359     lderr(cct) << "failed to open image: " << cpp_strerror(r) << dendl;
 360     return r;
 361   }
 362   BOOST_SCOPE_EXIT_TPL(image_ctx) {
 363     image_ctx->state->close();
 364   } BOOST_SCOPE_EXIT_END;
 365
 366   std::list<obj_watch_t> watchers;
 367   int flags = librbd::image::LIST_WATCHERS_FILTER_OUT_MY_INSTANCE |
 368               librbd::image::LIST_WATCHERS_FILTER_OUT_MIRROR_INSTANCES;
 369   C_SaferCond on_list_watchers;
 370   auto list_watchers_request = librbd::image::ListWatchersRequest<I>::create(
 371       *image_ctx, flags, &watchers, &on_list_watchers);
 372   list_watchers_request->send();
 373   r = on_list_watchers.wait();
 374   if (r < 0) {
 375     lderr(cct) << "failed listing watchers:" << cpp_strerror(r) << dendl;
 376     return r;
 377   }
 378   if (!watchers.empty()) {
 379     lderr(cct) << "image has watchers - not migrating" << dendl;
 380     return -EBUSY;
 381   }
 382
 383   uint64_t format = 2;
 384   if (opts.get(RBD_IMAGE_OPTION_FORMAT, &format) != 0) {
 385     opts.set(RBD_IMAGE_OPTION_FORMAT, format);
 386   }
 387   if (format != 2) {
 388     lderr(cct) << "unsupported destination image format: " << format << dendl;
 389     return -EINVAL;
 390   }
 391
 392   uint64_t features;
 393   {
 394     std::shared_lock image_locker{image_ctx->image_lock};
 395     features = image_ctx->features;
 396   }
 397   opts.get(RBD_IMAGE_OPTION_FEATURES, &features);
 398   if ((features & ~RBD_FEATURES_ALL) != 0) {
 399     lderr(cct) << "librbd does not support requested features" << dendl;
 400     return -ENOSYS;
 401   }
 402   features &= ~RBD_FEATURES_INTERNAL;
 403   features |= RBD_FEATURE_MIGRATING;
 404   opts.set(RBD_IMAGE_OPTION_FEATURES, features);
 405
 406   uint64_t order = image_ctx->order;
 407   if (opts.get(RBD_IMAGE_OPTION_ORDER, &order) != 0) {
 408     opts.set(RBD_IMAGE_OPTION_ORDER, order);
 409   }
 410   r = image::CreateRequest<I>::validate_order(cct, order);
 411   if (r < 0) {
 412     return r;
 413   }
 414
 415   uint64_t stripe_unit = image_ctx->stripe_unit;
 416   if (opts.get(RBD_IMAGE_OPTION_STRIPE_UNIT, &stripe_unit) != 0) {
 417     opts.set(RBD_IMAGE_OPTION_STRIPE_UNIT, stripe_unit);
 418   }
 419   uint64_t stripe_count = image_ctx->stripe_count;
 420   if (opts.get(RBD_IMAGE_OPTION_STRIPE_COUNT, &stripe_count) != 0) {
 421     opts.set(RBD_IMAGE_OPTION_STRIPE_COUNT, stripe_count);
 422   }
 423
 424   uint64_t flatten = 0;
 425   if (opts.get(RBD_IMAGE_OPTION_FLATTEN, &flatten) == 0) {
 426     opts.unset(RBD_IMAGE_OPTION_FLATTEN);
 427   }
 428
 429   ldout(cct, 20) << "updated opts=" << opts << dendl;
 430
 431   Migration migration(image_ctx, dest_io_ctx, dest_image_name, "", opts,
 432                       flatten > 0, false, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL,
 433                       cls::rbd::MIGRATION_STATE_PREPARING, "", nullptr);
 434   r = migration.prepare();
 435
 436   features &= ~RBD_FEATURE_MIGRATING;
 437   opts.set(RBD_IMAGE_OPTION_FEATURES, features);
 438
 439   return r;
 440 }
 441
 442 template <typename I>
 443 int Migration<I>::execute(librados::IoCtx& io_ctx,
 444                           const std::string &image_name,
 445                           ProgressContext &prog_ctx) {
 446   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 447
 448   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << dendl;
 449
 450   I *image_ctx;
 451   librados::IoCtx dest_io_ctx;
 452   std::string dest_image_name;
 453   std::string dest_image_id;
 454   bool flatten;
 455   bool mirroring;
 456   cls::rbd::MirrorImageMode mirror_image_mode;
 457   cls::rbd::MigrationState state;
 458   std::string state_description;
 459
 460   int r = open_source_image(io_ctx, image_name, &image_ctx, &dest_io_ctx,
 461                             &dest_image_name, &dest_image_id, &flatten,
 462                             &mirroring, &mirror_image_mode, &state,
 463                             &state_description);
 464   if (r < 0) {
 465     return r;
 466   }
 467
 468   BOOST_SCOPE_EXIT_TPL(image_ctx) {
 469     image_ctx->state->close();
 470   } BOOST_SCOPE_EXIT_END;
 471
 472   if (state != cls::rbd::MIGRATION_STATE_PREPARED) {
 473     lderr(cct) << "current migration state is '" << state << "'"
 474                << " (should be 'prepared')" << dendl;
 475     return -EINVAL;
 476   }
 477
 478   ldout(cct, 5) << "migrating " << image_ctx->md_ctx.get_pool_name() << "/"
 479                 << image_ctx->name << " -> " << dest_io_ctx.get_pool_name()
 480                 << "/" << dest_image_name << dendl;
 481
 482   ImageOptions opts;
 483   Migration migration(image_ctx, dest_io_ctx, dest_image_name, dest_image_id,
 484                       opts, flatten, mirroring, mirror_image_mode, state,
 485                       state_description, &prog_ctx);
 486   r = migration.execute();
 487   if (r < 0) {
 488     return r;
 489   }
 490
 491   return 0;
 492 }
 493
 494 template <typename I>
 495 int Migration<I>::abort(librados::IoCtx& io_ctx, const std::string &image_name,
 496                         ProgressContext &prog_ctx) {
 497   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 498
 499   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << dendl;
 500
 501   I *image_ctx;
 502   librados::IoCtx dest_io_ctx;
 503   std::string dest_image_name;
 504   std::string dest_image_id;
 505   bool flatten;
 506   bool mirroring;
 507   cls::rbd::MirrorImageMode mirror_image_mode;
 508   cls::rbd::MigrationState state;
 509   std::string state_description;
 510
 511   int r = open_source_image(io_ctx, image_name, &image_ctx, &dest_io_ctx,
 512                             &dest_image_name, &dest_image_id, &flatten,
 513                             &mirroring, &mirror_image_mode, &state,
 514                             &state_description);
 515   if (r < 0) {
 516     return r;
 517   }
 518
 519   ldout(cct, 5) << "canceling incomplete migration "
 520                 << image_ctx->md_ctx.get_pool_name() << "/" << image_ctx->name
 521                 << " -> " << dest_io_ctx.get_pool_name() << "/" << dest_image_name
 522                 << dendl;
 523
 524   ImageOptions opts;
 525   Migration migration(image_ctx, dest_io_ctx, dest_image_name, dest_image_id,
 526                       opts, flatten, mirroring, mirror_image_mode, state,
 527                       state_description, &prog_ctx);
 528   r = migration.abort();
 529
 530   image_ctx->state->close();
 531
 532   if (r < 0) {
 533     return r;
 534   }
 535
 536   return 0;
 537 }
 538
 539 template <typename I>
 540 int Migration<I>::commit(librados::IoCtx& io_ctx,
 541                          const std::string &image_name,
 542                          ProgressContext &prog_ctx) {
 543   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 544
 545   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << dendl;
 546
 547   I *image_ctx;
 548   librados::IoCtx dest_io_ctx;
 549   std::string dest_image_name;
 550   std::string dest_image_id;
 551   bool flatten;
 552   bool mirroring;
 553   cls::rbd::MirrorImageMode mirror_image_mode;
 554   cls::rbd::MigrationState state;
 555   std::string state_description;
 556
 557   int r = open_source_image(io_ctx, image_name, &image_ctx, &dest_io_ctx,
 558                             &dest_image_name, &dest_image_id, &flatten,
 559                             &mirroring, &mirror_image_mode, &state,
 560                             &state_description);
 561   if (r < 0) {
 562     return r;
 563   }
 564
 565   if (state != cls::rbd::MIGRATION_STATE_EXECUTED) {
 566     lderr(cct) << "current migration state is '" << state << "'"
 567                << " (should be 'executed')" << dendl;
 568     image_ctx->state->close();
 569     return -EINVAL;
 570   }
 571
 572   ldout(cct, 5) << "migrating " << image_ctx->md_ctx.get_pool_name() << "/"
 573                 << image_ctx->name << " -> " << dest_io_ctx.get_pool_name()
 574                 << "/" << dest_image_name << dendl;
 575
 576   ImageOptions opts;
 577   Migration migration(image_ctx, dest_io_ctx, dest_image_name, dest_image_id,
 578                       opts, flatten, mirroring, mirror_image_mode, state,
 579                       state_description, &prog_ctx);
 580   r = migration.commit();
 581
 582   // image_ctx is closed in commit when removing src image
 583
 584   if (r < 0) {
 585     return r;
 586   }
 587
 588   return 0;
 589 }
 590
 591 template <typename I>
 592 int Migration<I>::status(librados::IoCtx& io_ctx,
 593                          const std::string &image_name,
 594                          image_migration_status_t *status) {
 595   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 596
 597   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << dendl;
 598
 599   I *image_ctx;
 600   librados::IoCtx dest_io_ctx;
 601   std::string dest_image_name;
 602   std::string dest_image_id;
 603   bool flatten;
 604   bool mirroring;
 605   cls::rbd::MirrorImageMode mirror_image_mode;
 606   cls::rbd::MigrationState state;
 607   std::string state_description;
 608
 609   int r = open_source_image(io_ctx, image_name, &image_ctx, &dest_io_ctx,
 610                             &dest_image_name, &dest_image_id, &flatten,
 611                             &mirroring, &mirror_image_mode, &state,
 612                             &state_description);
 613   if (r < 0) {
 614     return r;
 615   }
 616
 617   ldout(cct, 5) << "migrating " << image_ctx->md_ctx.get_pool_name() << "/"
 618                 << image_ctx->name << " -> " << dest_io_ctx.get_pool_name()
 619                 << "/" << dest_image_name << dendl;
 620
 621   ImageOptions opts;
 622   Migration migration(image_ctx, dest_io_ctx, dest_image_name, dest_image_id,
 623                       opts, flatten, mirroring, mirror_image_mode, state,
 624                       state_description, nullptr);
 625   r = migration.status(status);
 626
 627   image_ctx->state->close();
 628
 629   if (r < 0) {
 630     return r;
 631   }
 632
 633   return 0;
 634 }
 635
 636 template <typename I>
 637 Migration<I>::Migration(I *src_image_ctx, librados::IoCtx& dst_io_ctx,
 638                         const std::string &dstname,
 639                         const std::string &dst_image_id,
 640                         ImageOptions& opts, bool flatten, bool mirroring,
 641                         cls::rbd::MirrorImageMode mirror_image_mode,
 642                         cls::rbd::MigrationState state,
 643                         const std::string &state_description,
 644                         ProgressContext *prog_ctx)
 645   : m_cct(static_cast<CephContext *>(dst_io_ctx.cct())),
 646     m_src_image_ctx(src_image_ctx), m_dst_io_ctx(dst_io_ctx),
 647     m_src_old_format(m_src_image_ctx->old_format),
 648     m_src_image_name(m_src_image_ctx->old_format ? m_src_image_ctx->name : ""),
 649     m_src_image_id(m_src_image_ctx->id),
 650     m_src_header_oid(m_src_image_ctx->header_oid), m_dst_image_name(dstname),
 651     m_dst_image_id(dst_image_id.empty() ?
 652                    util::generate_image_id(m_dst_io_ctx) : dst_image_id),
 653     m_dst_header_oid(util::header_name(m_dst_image_id)), m_image_options(opts),
 654     m_flatten(flatten), m_mirroring(mirroring),
 655     m_mirror_image_mode(mirror_image_mode), m_prog_ctx(prog_ctx),
 656     m_src_migration_spec(cls::rbd::MIGRATION_HEADER_TYPE_SRC,
 657                          m_dst_io_ctx.get_id(), m_dst_io_ctx.get_namespace(),
 658                          m_dst_image_name, m_dst_image_id, {}, 0, mirroring,
 659                          mirror_image_mode, flatten, state, state_description),
 660     m_dst_migration_spec(cls::rbd::MIGRATION_HEADER_TYPE_DST,
 661                          src_image_ctx->md_ctx.get_id(),
 662                          src_image_ctx->md_ctx.get_namespace(),
 663                          m_src_image_ctx->name, m_src_image_ctx->id, {}, 0,
 664                          mirroring, mirror_image_mode, flatten, state,
 665                          state_description) {
 666   m_src_io_ctx.dup(src_image_ctx->md_ctx);
 667 }
 668
 669 template <typename I>
 670 int Migration<I>::prepare() {
 671   ldout(m_cct, 10) << dendl;
 672
 673   int r = validate_src_snaps();
 674   if (r < 0) {
 675     return r;
 676   }
 677
 678   r = disable_mirroring(m_src_image_ctx, &m_mirroring, &m_mirror_image_mode);
 679   if (r < 0) {
 680     return r;
 681   }
 682
 683   r = unlink_src_image();
 684   if (r < 0) {
 685     enable_mirroring(m_src_image_ctx, m_mirroring, m_mirror_image_mode);
 686     return r;
 687   }
 688
 689   r = set_migration();
 690   if (r < 0) {
 691     relink_src_image();
 692     enable_mirroring(m_src_image_ctx, m_mirroring, m_mirror_image_mode);
 693     return r;
 694   }
 695
 696   r = create_dst_image();
 697   if (r < 0) {
 698     abort();
 699     return r;
 700   }
 701
 702   ldout(m_cct, 10) << "succeeded" << dendl;
 703
 704   return 0;
 705 }
 706
 707 template <typename I>
 708 int Migration<I>::execute() {
 709   ldout(m_cct, 10) << dendl;
 710
 711   auto dst_image_ctx = I::create(m_dst_image_name, m_dst_image_id, nullptr,
 712                                  m_dst_io_ctx, false);
 713   int r = dst_image_ctx->state->open(0);
 714   if (r < 0) {
 715     lderr(m_cct) << "failed to open destination image: " << cpp_strerror(r)
 716                  << dendl;
 717     return r;
 718   }
 719
 720   BOOST_SCOPE_EXIT_TPL(dst_image_ctx) {
 721     dst_image_ctx->state->close();
 722   } BOOST_SCOPE_EXIT_END;
 723
 724   r = set_state(cls::rbd::MIGRATION_STATE_EXECUTING, "");
 725   if (r < 0) {
 726     return r;
 727   }
 728
 729   while (true) {
 730     MigrationProgressContext prog_ctx(m_src_io_ctx, m_src_header_oid,
 731                                       cls::rbd::MIGRATION_STATE_EXECUTING,
 732                                       m_prog_ctx);
 733     r = dst_image_ctx->operations->migrate(prog_ctx);
 734     if (r == -EROFS) {
 735       std::shared_lock owner_locker{dst_image_ctx->owner_lock};
 736       if (dst_image_ctx->exclusive_lock != nullptr &&
 737           !dst_image_ctx->exclusive_lock->accept_ops()) {
 738         ldout(m_cct, 5) << "lost exclusive lock, retrying remote" << dendl;
 739         continue;
 740       }
 741     }
 742     break;
 743   }
 744   if (r < 0) {
 745     lderr(m_cct) << "migration failed: " << cpp_strerror(r) << dendl;
 746     return r;
 747   }
 748
 749   r = set_state(cls::rbd::MIGRATION_STATE_EXECUTED, "");
 750   if (r < 0) {
 751     return r;
 752   }
 753
 754   dst_image_ctx->notify_update();
 755
 756   ldout(m_cct, 10) << "succeeded" << dendl;
 757
 758   return 0;
 759 }
 760
 761 template <typename I>
 762 int Migration<I>::abort() {
 763   ldout(m_cct, 10) << dendl;
 764
 765   int r;
 766
 767   m_src_image_ctx->owner_lock.lock_shared();
 768   if (m_src_image_ctx->exclusive_lock != nullptr &&
 769       !m_src_image_ctx->exclusive_lock->is_lock_owner()) {
 770     C_SaferCond ctx;
 771     m_src_image_ctx->exclusive_lock->acquire_lock(&ctx);
 772     m_src_image_ctx->owner_lock.unlock_shared();
 773     r = ctx.wait();
 774     if (r < 0) {
 775       lderr(m_cct) << "error acquiring exclusive lock: " << cpp_strerror(r)
 776                    << dendl;
 777       return r;
 778     }
 779   } else {
 780     m_src_image_ctx->owner_lock.unlock_shared();
 781   }
 782
 783   group_info_t group_info;
 784   group_info.pool = -1;
 785
 786   auto dst_image_ctx = I::create(m_dst_image_name, m_dst_image_id, nullptr,
 787                                  m_dst_io_ctx, false);
 788   r = dst_image_ctx->state->open(OPEN_FLAG_IGNORE_MIGRATING);
 789   if (r < 0) {
 790     ldout(m_cct, 1) << "failed to open destination image: " << cpp_strerror(r)
 791                     << dendl;
 792   } else {
 793     ldout(m_cct, 10) << "relinking children" << dendl;
 794
 795     r = relink_children(dst_image_ctx, m_src_image_ctx);
 796     if (r < 0) {
 797       return r;
 798     }
 799
 800     ldout(m_cct, 10) << "removing dst image snapshots" << dendl;
 801
 802     BOOST_SCOPE_EXIT_TPL(&dst_image_ctx) {
 803       if (dst_image_ctx != nullptr) {
 804         dst_image_ctx->state->close();
 805       }
 806     } BOOST_SCOPE_EXIT_END;
 807
 808     std::vector<librbd::snap_info_t> snaps;
 809     r = Snapshot<I>::list(dst_image_ctx, snaps);
 810     if (r < 0) {
 811       lderr(m_cct) << "failed listing snapshots: " << cpp_strerror(r)
 812                    << dendl;
 813       return r;
 814     }
 815
 816     for (auto &snap : snaps) {
 817       librbd::NoOpProgressContext prog_ctx;
 818       int r = Snapshot<I>::remove(dst_image_ctx, snap.name.c_str(),
 819                                   RBD_SNAP_REMOVE_UNPROTECT, prog_ctx);
 820       if (r < 0) {
 821         lderr(m_cct) << "failed removing snapshot: " << cpp_strerror(r)
 822                      << dendl;
 823         return r;
 824       }
 825     }
 826
 827     ldout(m_cct, 10) << "removing group" << dendl;
 828
 829     r = remove_group(dst_image_ctx, &group_info);
 830     if (r < 0 && r != -ENOENT) {
 831       return r;
 832     }
 833
 834     ldout(m_cct, 10) << "removing dst image" << dendl;
 835
 836     ceph_assert(dst_image_ctx->ignore_migrating);
 837
 838     ThreadPool *thread_pool;
 839     ContextWQ *op_work_queue;
 840     ImageCtx::get_thread_pool_instance(m_cct, &thread_pool, &op_work_queue);
 841     C_SaferCond on_remove;
 842     auto req = librbd::image::RemoveRequest<>::create(
 843       m_dst_io_ctx, dst_image_ctx, false, false, *m_prog_ctx, op_work_queue,
 844       &on_remove);
 845     req->send();
 846     r = on_remove.wait();
 847
 848     dst_image_ctx = nullptr;
 849
 850     if (r < 0) {
 851       lderr(m_cct) << "failed removing destination image '"
 852                    << m_dst_io_ctx.get_pool_name() << "/" << m_dst_image_name
 853                    << " (" << m_dst_image_id << ")': " << cpp_strerror(r)
 854                    << dendl;
 855       // not fatal
 856     }
 857   }
 858
 859   r = relink_src_image();
 860   if (r < 0) {
 861     return r;
 862   }
 863
 864   r = add_group(m_src_image_ctx, group_info);
 865   if (r < 0) {
 866     return r;
 867   }
 868
 869   r = remove_migration(m_src_image_ctx);
 870   if (r < 0) {
 871     return r;
 872   }
 873
 874   r = enable_mirroring(m_src_image_ctx, m_mirroring, m_mirror_image_mode);
 875   if (r < 0) {
 876     return r;
 877   }
 878
 879   ldout(m_cct, 10) << "succeeded" << dendl;
 880
 881   return 0;
 882 }
 883
 884 template <typename I>
 885 int Migration<I>::commit() {
 886   ldout(m_cct, 10) << dendl;
 887
 888   BOOST_SCOPE_EXIT_TPL(&m_src_image_ctx) {
 889     if (m_src_image_ctx != nullptr) {
 890       m_src_image_ctx->state->close();
 891     }
 892   } BOOST_SCOPE_EXIT_END;
 893
 894   auto dst_image_ctx = I::create(m_dst_image_name, m_dst_image_id, nullptr,
 895                                  m_dst_io_ctx, false);
 896   int r = dst_image_ctx->state->open(0);
 897   if (r < 0) {
 898     lderr(m_cct) << "failed to open destination image: " << cpp_strerror(r)
 899                  << dendl;
 900     return r;
 901   }
 902
 903   BOOST_SCOPE_EXIT_TPL(dst_image_ctx) {
 904     dst_image_ctx->state->close();
 905   } BOOST_SCOPE_EXIT_END;
 906
 907   r = remove_migration(dst_image_ctx);
 908   if (r < 0) {
 909     return r;
 910   }
 911
 912   r = remove_src_image();
 913   if (r < 0) {
 914     return r;
 915   }
 916
 917   r = enable_mirroring(dst_image_ctx, m_mirroring, m_mirror_image_mode);
 918   if (r < 0) {
 919     return r;
 920   }
 921
 922   ldout(m_cct, 10) << "succeeded" << dendl;
 923
 924   return 0;
 925 }
 926
 927 template <typename I>
 928 int Migration<I>::status(image_migration_status_t *status) {
 929   ldout(m_cct, 10) << dendl;
 930
 931   status->source_pool_id = m_dst_migration_spec.pool_id;
 932   status->source_pool_namespace = m_dst_migration_spec.pool_namespace;
 933   status->source_image_name = m_dst_migration_spec.image_name;
 934   status->source_image_id = m_dst_migration_spec.image_id;
 935   status->dest_pool_id = m_src_migration_spec.pool_id;
 936   status->dest_pool_namespace = m_src_migration_spec.pool_namespace;
 937   status->dest_image_name = m_src_migration_spec.image_name;
 938   status->dest_image_id = m_src_migration_spec.image_id;
 939
 940   switch (m_src_migration_spec.state) {
 941   case cls::rbd::MIGRATION_STATE_ERROR:
 942     status->state = RBD_IMAGE_MIGRATION_STATE_ERROR;
 943     break;
 944   case cls::rbd::MIGRATION_STATE_PREPARING:
 945     status->state = RBD_IMAGE_MIGRATION_STATE_PREPARING;
 946     break;
 947   case cls::rbd::MIGRATION_STATE_PREPARED:
 948     status->state = RBD_IMAGE_MIGRATION_STATE_PREPARED;
 949     break;
 950   case cls::rbd::MIGRATION_STATE_EXECUTING:
 951     status->state = RBD_IMAGE_MIGRATION_STATE_EXECUTING;
 952     break;
 953   case cls::rbd::MIGRATION_STATE_EXECUTED:
 954     status->state = RBD_IMAGE_MIGRATION_STATE_EXECUTED;
 955     break;
 956   default:
 957     status->state = RBD_IMAGE_MIGRATION_STATE_UNKNOWN;
 958     break;
 959   }
 960
 961   status->state_description = m_src_migration_spec.state_description;
 962
 963   return 0;
 964 }
 965
 966 template <typename I>
 967 int Migration<I>::set_state(cls::rbd::MigrationState state,
 968                             const std::string &description) {
 969   int r = cls_client::migration_set_state(&m_src_io_ctx, m_src_header_oid,
 970                                           state, description);
 971   if (r < 0) {
 972     lderr(m_cct) << "failed to set source migration header: " << cpp_strerror(r)
 973                  << dendl;
 974     return r;
 975   }
 976
 977   r = cls_client::migration_set_state(&m_dst_io_ctx, m_dst_header_oid, state,
 978                                       description);
 979   if (r < 0) {
 980     lderr(m_cct) << "failed to set destination migration header: "
 981                  << cpp_strerror(r) << dendl;
 982     return r;
 983   }
 984
 985   return 0;
 986 }
 987
 988 template <typename I>
 989 int Migration<I>::list_src_snaps(std::vector<librbd::snap_info_t> *snaps) {
 990   ldout(m_cct, 10) << dendl;
 991
 992   int r = Snapshot<I>::list(m_src_image_ctx, *snaps);
 993   if (r < 0) {
 994     lderr(m_cct) << "failed listing snapshots: " << cpp_strerror(r) << dendl;
 995     return r;
 996   }
 997
 998   for (auto &snap : *snaps) {
 999     librbd::snap_namespace_type_t namespace_type;
1000     r = Snapshot<I>::get_namespace_type(m_src_image_ctx, snap.id,
1001                                         &namespace_type);
1002     if (r < 0) {
1003       lderr(m_cct) << "error getting snap namespace type: " << cpp_strerror(r)
1004                    << dendl;
1005       return r;
1006     }
1007
1008     if (namespace_type != RBD_SNAP_NAMESPACE_TYPE_USER) {
1009       if (namespace_type == RBD_SNAP_NAMESPACE_TYPE_TRASH) {
1010         lderr(m_cct) << "image has snapshots with linked clones that must be "
1011                      << "deleted or flattened before the image can be migrated"
1012                      << dendl;
1013       } else {
1014         lderr(m_cct) << "image has non-user type snapshots "
1015                      << "that are not supported by migration" << dendl;
1016       }
1017       return -EBUSY;
1018     }
1019   }
1020
1021   return 0;
1022 }
1023
1024 template <typename I>
1025 int Migration<I>::validate_src_snaps() {
1026   ldout(m_cct, 10) << dendl;
1027
1028   std::vector<librbd::snap_info_t> snaps;
1029   int r = list_src_snaps(&snaps);
1030   if (r < 0) {
1031     return r;
1032   }
1033
1034   uint64_t dst_features = 0;
1035   r = m_image_options.get(RBD_IMAGE_OPTION_FEATURES, &dst_features);
1036   ceph_assert(r == 0);
1037
1038   if (!m_src_image_ctx->test_features(RBD_FEATURE_LAYERING)) {
1039     return 0;
1040   }
1041
1042   for (auto &snap : snaps) {
1043     std::shared_lock image_locker{m_src_image_ctx->image_lock};
1044     cls::rbd::ParentImageSpec parent_spec{m_src_image_ctx->md_ctx.get_id(),
1045                                           m_src_image_ctx->md_ctx.get_namespace(),
1046                                           m_src_image_ctx->id, snap.id};
1047     std::vector<librbd::linked_image_spec_t> child_images;
1048     r = api::Image<I>::list_children(m_src_image_ctx, parent_spec,
1049                                      &child_images);
1050     if (r < 0) {
1051       lderr(m_cct) << "failed listing children: " << cpp_strerror(r)
1052                    << dendl;
1053       return r;
1054     }
1055     if (!child_images.empty()) {
1056       ldout(m_cct, 1) << m_src_image_ctx->name << "@" << snap.name
1057                       << " has children" << dendl;
1058
1059       if ((dst_features & RBD_FEATURE_LAYERING) == 0) {
1060         lderr(m_cct) << "can't migrate to destination without layering feature: "
1061                      << "image has children" << dendl;
1062         return -EINVAL;
1063       }
1064     }
1065   }
1066
1067   return 0;
1068 }
1069
1070
1071 template <typename I>
1072 int Migration<I>::set_migration() {
1073   ldout(m_cct, 10) << dendl;
1074
1075   m_src_image_ctx->ignore_migrating = true;
1076
1077   int r = cls_client::migration_set(&m_src_io_ctx, m_src_header_oid,
1078                                     m_src_migration_spec);
1079   if (r < 0) {
1080     lderr(m_cct) << "failed to set migration header: " << cpp_strerror(r)
1081                  << dendl;
1082     return r;
1083   }
1084
1085   m_src_image_ctx->notify_update();
1086
1087   return 0;
1088 }
1089
1090 template <typename I>
1091 int Migration<I>::remove_migration(I *image_ctx) {
1092   ldout(m_cct, 10) << dendl;
1093
1094   int r;
1095
1096   r = cls_client::migration_remove(&image_ctx->md_ctx, image_ctx->header_oid);
1097   if (r == -ENOENT) {
1098     r = 0;
1099   }
1100   if (r < 0) {
1101     lderr(m_cct) << "failed removing migration header: " << cpp_strerror(r)
1102                  << dendl;
1103     return r;
1104   }
1105
1106   image_ctx->notify_update();
1107
1108   return 0;
1109 }
1110
1111 template <typename I>
1112 int Migration<I>::unlink_src_image() {
1113   if (m_src_old_format) {
1114     return v1_unlink_src_image();
1115   } else {
1116     return v2_unlink_src_image();
1117   }
1118 }
1119
1120 template <typename I>
1121 int Migration<I>::v1_unlink_src_image() {
1122   ldout(m_cct, 10) << dendl;
1123
1124   int r = tmap_rm(m_src_io_ctx, m_src_image_name);
1125   if (r < 0) {
1126     lderr(m_cct) << "failed removing " << m_src_image_name << " from tmap: "
1127                  << cpp_strerror(r) << dendl;
1128     return r;
1129   }
1130
1131   return 0;
1132 }
1133
1134 template <typename I>
1135 int Migration<I>::v2_unlink_src_image() {
1136   ldout(m_cct, 10) << dendl;
1137
1138   m_src_image_ctx->owner_lock.lock_shared();
1139   if (m_src_image_ctx->exclusive_lock != nullptr &&
1140       m_src_image_ctx->exclusive_lock->is_lock_owner()) {
1141     C_SaferCond ctx;
1142     m_src_image_ctx->exclusive_lock->release_lock(&ctx);
1143     m_src_image_ctx->owner_lock.unlock_shared();
1144     int r = ctx.wait();
1145      if (r < 0) {
1146       lderr(m_cct) << "error releasing exclusive lock: " << cpp_strerror(r)
1147                    << dendl;
1148       return r;
1149      }
1150   } else {
1151     m_src_image_ctx->owner_lock.unlock_shared();
1152   }
1153
1154   int r = Trash<I>::move(m_src_io_ctx, RBD_TRASH_IMAGE_SOURCE_MIGRATION,
1155                          m_src_image_ctx->name, 0);
1156   if (r < 0) {
1157     lderr(m_cct) << "failed moving image to trash: " << cpp_strerror(r)
1158                  << dendl;
1159     return r;
1160   }
1161
1162   return 0;
1163 }
1164
1165 template <typename I>
1166 int Migration<I>::relink_src_image() {
1167   if (m_src_old_format) {
1168     return v1_relink_src_image();
1169   } else {
1170     return v2_relink_src_image();
1171   }
1172 }
1173
1174 template <typename I>
1175 int Migration<I>::v1_relink_src_image() {
1176   ldout(m_cct, 10) << dendl;
1177
1178   int r = tmap_set(m_src_io_ctx, m_src_image_name);
1179   if (r < 0) {
1180     lderr(m_cct) << "failed adding " << m_src_image_name << " to tmap: "
1181                  << cpp_strerror(r) << dendl;
1182     return r;
1183   }
1184
1185   return 0;
1186 }
1187
1188 template <typename I>
1189 int Migration<I>::v2_relink_src_image() {
1190   ldout(m_cct, 10) << dendl;
1191
1192   int r = Trash<I>::restore(m_src_io_ctx,
1193                             {cls::rbd::TRASH_IMAGE_SOURCE_MIGRATION},
1194                             m_src_image_ctx->id, m_src_image_ctx->name);
1195   if (r < 0) {
1196     lderr(m_cct) << "failed restoring image from trash: " << cpp_strerror(r)
1197                  << dendl;
1198     return r;
1199   }
1200
1201   return 0;
1202 }
1203
1204 template <typename I>
1205 int Migration<I>::create_dst_image() {
1206   ldout(m_cct, 10) << dendl;
1207
1208   uint64_t size;
1209   cls::rbd::ParentImageSpec parent_spec;
1210   {
1211     std::shared_lock image_locker{m_src_image_ctx->image_lock};
1212     size = m_src_image_ctx->size;
1213
1214     // use oldest snapshot or HEAD for parent spec
1215     if (!m_src_image_ctx->snap_info.empty()) {
1216       parent_spec = m_src_image_ctx->snap_info.begin()->second.parent.spec;
1217     } else {
1218       parent_spec = m_src_image_ctx->parent_md.spec;
1219     }
1220   }
1221
1222   ThreadPool *thread_pool;
1223   ContextWQ *op_work_queue;
1224   ImageCtx::get_thread_pool_instance(m_cct, &thread_pool, &op_work_queue);
1225
1226   ConfigProxy config{m_cct->_conf};
1227   api::Config<I>::apply_pool_overrides(m_dst_io_ctx, &config);
1228
1229   int r;
1230   C_SaferCond on_create;
1231   librados::IoCtx parent_io_ctx;
1232   if (parent_spec.pool_id == -1) {
1233     auto *req = image::CreateRequest<I>::create(
1234       config, m_dst_io_ctx, m_dst_image_name, m_dst_image_id, size,
1235       m_image_options, true /* skip_mirror_enable */,
1236       cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, "", "", op_work_queue, &on_create);
1237     req->send();
1238   } else {
1239     r = util::create_ioctx(m_src_image_ctx->md_ctx, "destination image",
1240                            parent_spec.pool_id, parent_spec.pool_namespace,
1241                            &parent_io_ctx);
1242     if (r < 0) {
1243       return r;
1244     }
1245
1246     auto *req = image::CloneRequest<I>::create(
1247       config, parent_io_ctx, parent_spec.image_id, "", {}, parent_spec.snap_id,
1248       m_dst_io_ctx, m_dst_image_name, m_dst_image_id, m_image_options,
1249       cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, "", "", op_work_queue, &on_create);
1250     req->send();
1251   }
1252
1253   r = on_create.wait();
1254   if (r < 0) {
1255     lderr(m_cct) << "header creation failed: " << cpp_strerror(r) << dendl;
1256     return r;
1257   }
1258
1259   auto dst_image_ctx = I::create(m_dst_image_name, m_dst_image_id, nullptr,
1260                                  m_dst_io_ctx, false);
1261
1262   r = dst_image_ctx->state->open(OPEN_FLAG_IGNORE_MIGRATING);
1263   if (r < 0) {
1264     lderr(m_cct) << "failed to open newly created header: " << cpp_strerror(r)
1265                  << dendl;
1266     return r;
1267   }
1268
1269   BOOST_SCOPE_EXIT_TPL(dst_image_ctx) {
1270     dst_image_ctx->state->close();
1271   } BOOST_SCOPE_EXIT_END;
1272
1273   {
1274     std::shared_lock owner_locker{dst_image_ctx->owner_lock};
1275     r = dst_image_ctx->operations->prepare_image_update(
1276       exclusive_lock::OPERATION_REQUEST_TYPE_GENERAL, true);
1277     if (r < 0) {
1278       lderr(m_cct) << "cannot obtain exclusive lock" << dendl;
1279       return r;
1280     }
1281     if (dst_image_ctx->exclusive_lock != nullptr) {
1282       dst_image_ctx->exclusive_lock->block_requests(0);
1283     }
1284   }
1285
1286   SnapSeqs snap_seqs;
1287
1288   C_SaferCond on_snapshot_copy;
1289   auto snapshot_copy_req = librbd::deep_copy::SnapshotCopyRequest<I>::create(
1290       m_src_image_ctx, dst_image_ctx, 0, CEPH_NOSNAP, 0, m_flatten,
1291       m_src_image_ctx->op_work_queue, &snap_seqs, &on_snapshot_copy);
1292   snapshot_copy_req->send();
1293   r = on_snapshot_copy.wait();
1294   if (r < 0) {
1295     lderr(m_cct) << "failed to copy snapshots: " << cpp_strerror(r) << dendl;
1296     return r;
1297   }
1298
1299   C_SaferCond on_metadata_copy;
1300   auto metadata_copy_req = librbd::deep_copy::MetadataCopyRequest<I>::create(
1301       m_src_image_ctx, dst_image_ctx, &on_metadata_copy);
1302   metadata_copy_req->send();
1303   r = on_metadata_copy.wait();
1304   if (r < 0) {
1305     lderr(m_cct) << "failed to copy metadata: " << cpp_strerror(r) << dendl;
1306     return r;
1307   }
1308
1309   m_dst_migration_spec = {cls::rbd::MIGRATION_HEADER_TYPE_DST,
1310                           m_src_io_ctx.get_id(), m_src_io_ctx.get_namespace(),
1311                           m_src_image_name, m_src_image_id, snap_seqs, size,
1312                           m_mirroring, m_mirror_image_mode, m_flatten,
1313                           cls::rbd::MIGRATION_STATE_PREPARING, ""};
1314
1315   r = cls_client::migration_set(&m_dst_io_ctx, m_dst_header_oid,
1316                                 m_dst_migration_spec);
1317   if (r < 0) {
1318     lderr(m_cct) << "failed to set migration header: " << cpp_strerror(r)
1319                  << dendl;
1320     return r;
1321   }
1322
1323   r = update_group(m_src_image_ctx, dst_image_ctx);
1324   if (r < 0) {
1325     return r;
1326   }
1327
1328   r = set_state(cls::rbd::MIGRATION_STATE_PREPARED, "");
1329   if (r < 0) {
1330     return r;
1331   }
1332
1333   r = dst_image_ctx->state->refresh();
1334   if (r < 0) {
1335     lderr(m_cct) << "failed to refresh destination image: " << cpp_strerror(r)
1336                  << dendl;
1337     return r;
1338   }
1339
1340   r = relink_children(m_src_image_ctx, dst_image_ctx);
1341   if (r < 0) {
1342     return r;
1343   }
1344
1345   return 0;
1346 }
1347
1348 template <typename I>
1349 int Migration<I>::remove_group(I *image_ctx, group_info_t *group_info) {
1350   int r = librbd::api::Group<I>::image_get_group(image_ctx, group_info);
1351   if (r < 0) {
1352     lderr(m_cct) << "failed to get image group: " << cpp_strerror(r) << dendl;
1353     return r;
1354   }
1355
1356   if (group_info->pool == -1) {
1357     return -ENOENT;
1358   }
1359
1360   ceph_assert(!image_ctx->id.empty());
1361
1362   ldout(m_cct, 10) << dendl;
1363
1364   IoCtx group_ioctx;
1365   r = util::create_ioctx(image_ctx->md_ctx, "group", group_info->pool, {},
1366                          &group_ioctx);
1367   if (r < 0) {
1368     return r;
1369   }
1370
1371   r = librbd::api::Group<I>::image_remove_by_id(group_ioctx,
1372                                                 group_info->name.c_str(),
1373                                                 image_ctx->md_ctx,
1374                                                 image_ctx->id.c_str());
1375   if (r < 0) {
1376     lderr(m_cct) << "failed to remove image from group: " << cpp_strerror(r)
1377                  << dendl;
1378     return r;
1379   }
1380
1381   return 0;
1382 }
1383
1384 template <typename I>
1385 int Migration<I>::add_group(I *image_ctx, group_info_t &group_info) {
1386   if (group_info.pool == -1) {
1387     return 0;
1388   }
1389
1390   ldout(m_cct, 10) << dendl;
1391
1392   IoCtx group_ioctx;
1393   int r = util::create_ioctx(image_ctx->md_ctx, "group", group_info.pool, {},
1394                              &group_ioctx);
1395   if (r < 0) {
1396     return r;
1397   }
1398
1399   r = librbd::api::Group<I>::image_add(group_ioctx, group_info.name.c_str(),
1400                                        image_ctx->md_ctx,
1401                                        image_ctx->name.c_str());
1402   if (r < 0) {
1403     lderr(m_cct) << "failed to add image to group: " << cpp_strerror(r)
1404                  << dendl;
1405     return r;
1406   }
1407
1408   return 0;
1409 }
1410
1411 template <typename I>
1412 int Migration<I>::update_group(I *from_image_ctx, I *to_image_ctx) {
1413   ldout(m_cct, 10) << dendl;
1414
1415   group_info_t group_info;
1416
1417   int r = remove_group(from_image_ctx, &group_info);
1418   if (r < 0) {
1419     return r == -ENOENT ? 0 : r;
1420   }
1421
1422   r = add_group(to_image_ctx, group_info);
1423   if (r < 0) {
1424     return r;
1425   }
1426
1427   return 0;
1428 }
1429
1430 template <typename I>
1431 int Migration<I>::disable_mirroring(
1432     I *image_ctx, bool *was_enabled,
1433     cls::rbd::MirrorImageMode *mirror_image_mode) {
1434   *was_enabled = false;
1435
1436   cls::rbd::MirrorImage mirror_image;
1437   int r = cls_client::mirror_image_get(&image_ctx->md_ctx, image_ctx->id,
1438                                        &mirror_image);
1439   if (r == -ENOENT) {
1440     ldout(m_cct, 10) << "mirroring is not enabled for this image" << dendl;
1441     return 0;
1442   }
1443
1444   if (r < 0) {
1445     lderr(m_cct) << "failed to retrieve mirror image: " << cpp_strerror(r)
1446                  << dendl;
1447     return r;
1448   }
1449
1450   if (mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) {
1451     *was_enabled = true;
1452     *mirror_image_mode = mirror_image.mode;
1453   }
1454
1455   ldout(m_cct, 10) << dendl;
1456
1457   C_SaferCond ctx;
1458   auto req = mirror::DisableRequest<I>::create(image_ctx, false, true, &ctx);
1459   req->send();
1460   r = ctx.wait();
1461   if (r < 0) {
1462     lderr(m_cct) << "failed to disable mirroring: " << cpp_strerror(r)
1463                  << dendl;
1464     return r;
1465   }
1466
1467   m_src_migration_spec.mirroring = true;
1468
1469   return 0;
1470 }
1471
1472 template <typename I>
1473 int Migration<I>::enable_mirroring(
1474     I *image_ctx, bool was_enabled,
1475     cls::rbd::MirrorImageMode mirror_image_mode) {
1476   cls::rbd::MirrorMode mirror_mode;
1477   int r = cls_client::mirror_mode_get(&image_ctx->md_ctx, &mirror_mode);
1478   if (r < 0 && r != -ENOENT) {
1479     lderr(m_cct) << "failed to retrieve mirror mode: " << cpp_strerror(r)
1480                  << dendl;
1481     return r;
1482   }
1483
1484   if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) {
1485     ldout(m_cct, 10) << "mirroring is not enabled for destination pool"
1486                      << dendl;
1487     return 0;
1488   }
1489   if (mirror_mode == cls::rbd::MIRROR_MODE_IMAGE && !was_enabled) {
1490     ldout(m_cct, 10) << "mirroring is not enabled for image" << dendl;
1491     return 0;
1492   }
1493
1494   ldout(m_cct, 10) << dendl;
1495
1496   C_SaferCond ctx;
1497   auto req = mirror::EnableRequest<I>::create(
1498     image_ctx, mirror_image_mode, &ctx);
1499   req->send();
1500   r = ctx.wait();
1501   if (r < 0) {
1502     lderr(m_cct) << "failed to enable mirroring: " << cpp_strerror(r)
1503                  << dendl;
1504     return r;
1505   }
1506
1507   return 0;
1508 }
1509
1510 // When relinking children we should be careful as it my be interrupted
1511 // at any moment by some reason and we may end up in an inconsistent
1512 // state, which we have to be able to fix with "migration abort". Below
1513 // are all possible states during migration (P1 - sourse parent, P2 -
1514 // destination parent, C - child):
1515 //
1516 //   P1  P2    P1  P2    P1  P2    P1  P2
1517 //   ^\         \  ^      \ /^        /^
1518 //    \v         v/        v/        v/
1519 //     C         C         C         C
1520 //
1521 //     1         2         3         4
1522 //
1523 // (1) and (4) are the initial and the final consistent states. (2)
1524 // and (3) are intermediate inconsistent states that have to be fixed
1525 // by relink_children running in "migration abort" mode. For this, it
1526 // scans P2 for all children attached and relinks (fixes) states (3)
1527 // and (4) to state (1). Then it scans P1 for remaining children and
1528 // fixes the states (2).
1529
1530 template <typename I>
1531 int Migration<I>::relink_children(I *from_image_ctx, I *to_image_ctx) {
1532   ldout(m_cct, 10) << dendl;
1533
1534   std::vector<librbd::snap_info_t> snaps;
1535   int r = list_src_snaps(&snaps);
1536   if (r < 0) {
1537     return r;
1538   }
1539
1540   bool migration_abort = (to_image_ctx == m_src_image_ctx);
1541
1542   for (auto it = snaps.begin(); it != snaps.end(); it++) {
1543     auto &snap = *it;
1544     std::vector<librbd::linked_image_spec_t> src_child_images;
1545
1546     if (from_image_ctx != m_src_image_ctx) {
1547       ceph_assert(migration_abort);
1548
1549       // We run list snaps against the src image to get only those snapshots
1550       // that are migrated. If the "from" image is not the src image
1551       // (abort migration case), we need to remap snap ids.
1552       // Also collect the list of the children currently attached to the
1553       // source, so we could make a proper decision later about relinking.
1554
1555       std::shared_lock src_image_locker{to_image_ctx->image_lock};
1556       cls::rbd::ParentImageSpec src_parent_spec{to_image_ctx->md_ctx.get_id(),
1557                                                 to_image_ctx->md_ctx.get_namespace(),
1558                                                 to_image_ctx->id, snap.id};
1559       r = api::Image<I>::list_children(to_image_ctx, src_parent_spec,
1560                                        &src_child_images);
1561       if (r < 0) {
1562         lderr(m_cct) << "failed listing children: " << cpp_strerror(r)
1563                      << dendl;
1564         return r;
1565       }
1566
1567       std::shared_lock image_locker{from_image_ctx->image_lock};
1568       snap.id = from_image_ctx->get_snap_id(cls::rbd::UserSnapshotNamespace(),
1569                                             snap.name);
1570       if (snap.id == CEPH_NOSNAP) {
1571         ldout(m_cct, 5) << "skipping snapshot " << snap.name << dendl;
1572         continue;
1573       }
1574     }
1575
1576     std::vector<librbd::linked_image_spec_t> child_images;
1577     {
1578       std::shared_lock image_locker{from_image_ctx->image_lock};
1579       cls::rbd::ParentImageSpec parent_spec{from_image_ctx->md_ctx.get_id(),
1580                                             from_image_ctx->md_ctx.get_namespace(),
1581                                             from_image_ctx->id, snap.id};
1582       r = api::Image<I>::list_children(from_image_ctx, parent_spec,
1583                                        &child_images);
1584       if (r < 0) {
1585         lderr(m_cct) << "failed listing children: " << cpp_strerror(r)
1586                      << dendl;
1587         return r;
1588       }
1589     }
1590
1591     for (auto &child_image : child_images) {
1592       r = relink_child(from_image_ctx, to_image_ctx, snap, child_image,
1593                        migration_abort, true);
1594       if (r < 0) {
1595         return r;
1596       }
1597
1598       src_child_images.erase(std::remove(src_child_images.begin(),
1599                                          src_child_images.end(), child_image),
1600                              src_child_images.end());
1601     }
1602
1603     for (auto &child_image : src_child_images) {
1604       r = relink_child(from_image_ctx, to_image_ctx, snap, child_image,
1605                        migration_abort, false);
1606       if (r < 0) {
1607         return r;
1608       }
1609     }
1610   }
1611
1612   return 0;
1613 }
1614
1615 template <typename I>
1616 int Migration<I>::relink_child(I *from_image_ctx, I *to_image_ctx,
1617                                const librbd::snap_info_t &from_snap,
1618                                const librbd::linked_image_spec_t &child_image,
1619                                bool migration_abort, bool reattach_child) {
1620   ldout(m_cct, 10) << from_snap.name << " " << child_image.pool_name << "/"
1621                    << child_image.pool_namespace << "/"
1622                    << child_image.image_name << " (migration_abort="
1623                    << migration_abort << ", reattach_child=" << reattach_child
1624                    << ")" << dendl;
1625
1626   librados::snap_t to_snap_id;
1627   {
1628     std::shared_lock image_locker{to_image_ctx->image_lock};
1629     to_snap_id = to_image_ctx->get_snap_id(cls::rbd::UserSnapshotNamespace(),
1630                                              from_snap.name);
1631     if (to_snap_id == CEPH_NOSNAP) {
1632       lderr(m_cct) << "no snapshot " << from_snap.name << " on destination image"
1633                    << dendl;
1634       return -ENOENT;
1635     }
1636   }
1637
1638   librados::IoCtx child_io_ctx;
1639   int r = util::create_ioctx(to_image_ctx->md_ctx,
1640                              "child image " + child_image.image_name,
1641                              child_image.pool_id, child_image.pool_namespace,
1642                              &child_io_ctx);
1643   if (r < 0) {
1644     return r;
1645   }
1646
1647   I *child_image_ctx = I::create("", child_image.image_id, nullptr,
1648                                  child_io_ctx, false);
1649   r = child_image_ctx->state->open(OPEN_FLAG_SKIP_OPEN_PARENT);
1650   if (r < 0) {
1651     lderr(m_cct) << "failed to open child image: " << cpp_strerror(r) << dendl;
1652     return r;
1653   }
1654   BOOST_SCOPE_EXIT_TPL(child_image_ctx) {
1655     child_image_ctx->state->close();
1656   } BOOST_SCOPE_EXIT_END;
1657
1658   uint32_t clone_format = 1;
1659   if (child_image_ctx->test_op_features(RBD_OPERATION_FEATURE_CLONE_CHILD)) {
1660     clone_format = 2;
1661   }
1662
1663   cls::rbd::ParentImageSpec parent_spec;
1664   uint64_t parent_overlap;
1665   {
1666     std::shared_lock image_locker{child_image_ctx->image_lock};
1667
1668     // use oldest snapshot or HEAD for parent spec
1669     if (!child_image_ctx->snap_info.empty()) {
1670       parent_spec = child_image_ctx->snap_info.begin()->second.parent.spec;
1671       parent_overlap = child_image_ctx->snap_info.begin()->second.parent.overlap;
1672     } else {
1673       parent_spec = child_image_ctx->parent_md.spec;
1674       parent_overlap = child_image_ctx->parent_md.overlap;
1675     }
1676   }
1677
1678   if (migration_abort &&
1679       parent_spec.pool_id == to_image_ctx->md_ctx.get_id() &&
1680       parent_spec.pool_namespace == to_image_ctx->md_ctx.get_namespace() &&
1681       parent_spec.image_id == to_image_ctx->id &&
1682       parent_spec.snap_id == to_snap_id) {
1683     ldout(m_cct, 10) << "no need for parent re-attach" << dendl;
1684   } else {
1685     if (parent_spec.pool_id != from_image_ctx->md_ctx.get_id() ||
1686         parent_spec.pool_namespace != from_image_ctx->md_ctx.get_namespace() ||
1687         parent_spec.image_id != from_image_ctx->id ||
1688         parent_spec.snap_id != from_snap.id) {
1689       lderr(m_cct) << "parent is not source image: " << parent_spec.pool_id
1690                    << "/" << parent_spec.pool_namespace << "/"
1691                    << parent_spec.image_id << "@" << parent_spec.snap_id
1692                    << dendl;
1693       return -ESTALE;
1694     }
1695
1696     parent_spec.pool_id = to_image_ctx->md_ctx.get_id();
1697     parent_spec.pool_namespace = to_image_ctx->md_ctx.get_namespace();
1698     parent_spec.image_id = to_image_ctx->id;
1699     parent_spec.snap_id = to_snap_id;
1700
1701     C_SaferCond on_reattach_parent;
1702     auto reattach_parent_req = image::AttachParentRequest<I>::create(
1703       *child_image_ctx, parent_spec, parent_overlap, true, &on_reattach_parent);
1704     reattach_parent_req->send();
1705     r = on_reattach_parent.wait();
1706     if (r < 0) {
1707       lderr(m_cct) << "failed to re-attach parent: " << cpp_strerror(r) << dendl;
1708       return r;
1709     }
1710   }
1711
1712   if (reattach_child) {
1713     C_SaferCond on_reattach_child;
1714     auto reattach_child_req = image::AttachChildRequest<I>::create(
1715       child_image_ctx, to_image_ctx, to_snap_id, from_image_ctx, from_snap.id,
1716       clone_format, &on_reattach_child);
1717     reattach_child_req->send();
1718     r = on_reattach_child.wait();
1719     if (r < 0) {
1720       lderr(m_cct) << "failed to re-attach child: " << cpp_strerror(r) << dendl;
1721       return r;
1722     }
1723   }
1724
1725   child_image_ctx->notify_update();
1726
1727   return 0;
1728 }
1729
1730 template <typename I>
1731 int Migration<I>::remove_src_image() {
1732   ldout(m_cct, 10) << dendl;
1733
1734   std::vector<librbd::snap_info_t> snaps;
1735   int r = list_src_snaps(&snaps);
1736   if (r < 0) {
1737     return r;
1738   }
1739
1740   for (auto it = snaps.rbegin(); it != snaps.rend(); it++) {
1741     auto &snap = *it;
1742
1743     librbd::NoOpProgressContext prog_ctx;
1744     int r = Snapshot<I>::remove(m_src_image_ctx, snap.name.c_str(),
1745                                 RBD_SNAP_REMOVE_UNPROTECT, prog_ctx);
1746     if (r < 0) {
1747       lderr(m_cct) << "failed removing source image snapshot '" << snap.name
1748                    << "': " << cpp_strerror(r) << dendl;
1749       return r;
1750     }
1751   }
1752
1753   ceph_assert(m_src_image_ctx->ignore_migrating);
1754
1755   ThreadPool *thread_pool;
1756   ContextWQ *op_work_queue;
1757   ImageCtx::get_thread_pool_instance(m_cct, &thread_pool, &op_work_queue);
1758   C_SaferCond on_remove;
1759   auto req = librbd::image::RemoveRequest<I>::create(
1760       m_src_io_ctx, m_src_image_ctx, false, true, *m_prog_ctx, op_work_queue,
1761       &on_remove);
1762   req->send();
1763   r = on_remove.wait();
1764
1765   m_src_image_ctx = nullptr;
1766
1767   // For old format image it will return -ENOENT due to expected
1768   // tmap_rm failure at the end.
1769   if (r < 0 && r != -ENOENT) {
1770     lderr(m_cct) << "failed removing source image: " << cpp_strerror(r)
1771                  << dendl;
1772     return r;
1773   }
1774
1775   if (!m_src_image_id.empty()) {
1776     r = cls_client::trash_remove(&m_src_io_ctx, m_src_image_id);
1777     if (r < 0 && r != -ENOENT) {
1778       lderr(m_cct) << "error removing image " << m_src_image_id
1779                    << " from rbd_trash object" << dendl;
1780     }
1781   }
1782
1783   return 0;
1784 }
1785
1786 } // namespace api
1787 } // namespace librbd
1788
1789 template class librbd::api::Migration<librbd::ImageCtx>;