ceph/src/librbd/api/Migration.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "librbd/api/Migration.h"
   5 #include "include/rados/librados.hpp"
   6 #include "include/stringify.h"
   7 #include "common/dout.h"
   8 #include "common/errno.h"
   9 #include "cls/rbd/cls_rbd_client.h"
  10 #include "librbd/ExclusiveLock.h"
  11 #include "librbd/ImageCtx.h"
  12 #include "librbd/ImageState.h"
  13 #include "librbd/Operations.h"
  14 #include "librbd/Utils.h"
  15 #include "librbd/api/Config.h"
  16 #include "librbd/api/Group.h"
  17 #include "librbd/api/Image.h"
  18 #include "librbd/api/Snapshot.h"
  19 #include "librbd/api/Trash.h"
  20 #include "librbd/deep_copy/MetadataCopyRequest.h"
  21 #include "librbd/deep_copy/SnapshotCopyRequest.h"
  22 #include "librbd/exclusive_lock/Policy.h"
  23 #include "librbd/image/AttachChildRequest.h"
  24 #include "librbd/image/AttachParentRequest.h"
  25 #include "librbd/image/CloneRequest.h"
  26 #include "librbd/image/CreateRequest.h"
  27 #include "librbd/image/DetachChildRequest.h"
  28 #include "librbd/image/DetachParentRequest.h"
  29 #include "librbd/image/ListWatchersRequest.h"
  30 #include "librbd/image/RemoveRequest.h"
  31 #include "librbd/internal.h"
  32 #include "librbd/io/ImageRequestWQ.h"
  33 #include "librbd/mirror/DisableRequest.h"
  34 #include "librbd/mirror/EnableRequest.h"
  35
  36 #include <boost/scope_exit.hpp>
  37
  38 #define dout_subsys ceph_subsys_rbd
  39 #undef dout_prefix
  40 #define dout_prefix *_dout << "librbd::Migration: " << __func__ << ": "
  41
  42 namespace librbd {
  43
  44 inline bool operator==(const linked_image_spec_t& rhs,
  45                        const linked_image_spec_t& lhs) {
  46   bool result = (rhs.pool_id == lhs.pool_id &&
  47                  rhs.pool_namespace == lhs.pool_namespace &&
  48                  rhs.image_id == lhs.image_id);
  49   return result;
  50 }
  51
  52 namespace api {
  53
  54 using util::create_rados_callback;
  55
  56 namespace {
  57
  58 class MigrationProgressContext : public ProgressContext {
  59 public:
  60   MigrationProgressContext(librados::IoCtx& io_ctx,
  61                            const std::string &header_oid,
  62                            cls::rbd::MigrationState state,
  63                            ProgressContext *prog_ctx)
  64     : m_io_ctx(io_ctx), m_header_oid(header_oid), m_state(state),
  65       m_prog_ctx(prog_ctx), m_cct(reinterpret_cast<CephContext*>(io_ctx.cct())),
  66       m_lock(util::unique_lock_name("librbd::api::MigrationProgressContext",
  67                                     this)) {
  68     ceph_assert(m_prog_ctx != nullptr);
  69   }
  70
  71   ~MigrationProgressContext() {
  72     wait_for_in_flight_updates();
  73   }
  74
  75   int update_progress(uint64_t offset, uint64_t total) override {
  76     ldout(m_cct, 20) << "offset=" << offset << ", total=" << total << dendl;
  77
  78     m_prog_ctx->update_progress(offset, total);
  79
  80     std::string description = stringify(offset * 100 / total) + "% complete";
  81
  82     send_state_description_update(description);
  83
  84     return 0;
  85   }
  86
  87 private:
  88   librados::IoCtx& m_io_ctx;
  89   std::string m_header_oid;
  90   cls::rbd::MigrationState m_state;
  91   ProgressContext *m_prog_ctx;
  92
  93   CephContext* m_cct;
  94   mutable Mutex m_lock;
  95   Cond m_cond;
  96   std::string m_state_description;
  97   bool m_pending_update = false;
  98   int m_in_flight_state_updates = 0;
  99
 100   void send_state_description_update(const std::string &description) {
 101     Mutex::Locker locker(m_lock);
 102
 103     if (description == m_state_description) {
 104       return;
 105     }
 106
 107     m_state_description = description;
 108
 109     if (m_in_flight_state_updates > 0) {
 110       m_pending_update = true;
 111       return;
 112     }
 113
 114     set_state_description();
 115   }
 116
 117   void set_state_description() {
 118     ldout(m_cct, 20) << "state_description=" << m_state_description << dendl;
 119
 120     ceph_assert(m_lock.is_locked());
 121
 122     librados::ObjectWriteOperation op;
 123     cls_client::migration_set_state(&op, m_state, m_state_description);
 124
 125     using klass = MigrationProgressContext;
 126     librados::AioCompletion *comp =
 127       create_rados_callback<klass, &klass::handle_set_state_description>(this);
 128     int r = m_io_ctx.aio_operate(m_header_oid, comp, &op);
 129     ceph_assert(r == 0);
 130     comp->release();
 131
 132     m_in_flight_state_updates++;
 133   }
 134
 135   void handle_set_state_description(int r) {
 136     ldout(m_cct, 20) << "r=" << r << dendl;
 137
 138     Mutex::Locker locker(m_lock);
 139
 140     m_in_flight_state_updates--;
 141
 142     if (r < 0) {
 143       lderr(m_cct) << "failed to update migration state: " << cpp_strerror(r)
 144                    << dendl;
 145     } else if (m_pending_update) {
 146       set_state_description();
 147       m_pending_update = false;
 148     } else {
 149       m_cond.Signal();
 150     }
 151   }
 152
 153   void wait_for_in_flight_updates() {
 154     Mutex::Locker locker(m_lock);
 155
 156     ldout(m_cct, 20) << "m_in_flight_state_updates="
 157                      << m_in_flight_state_updates << dendl;
 158
 159     m_pending_update = false;
 160     while (m_in_flight_state_updates > 0) {
 161       m_cond.Wait(m_lock);
 162     }
 163   }
 164 };
 165
 166 int trash_search(librados::IoCtx &io_ctx, rbd_trash_image_source_t source,
 167                  const std::string &image_name, std::string *image_id) {
 168   std::vector<trash_image_info_t> entries;
 169
 170   int r = Trash<>::list(io_ctx, entries, false);
 171   if (r < 0) {
 172     return r;
 173   }
 174
 175   for (auto &entry : entries) {
 176     if (entry.source == source && entry.name == image_name) {
 177       *image_id = entry.id;
 178       return 0;
 179     }
 180   }
 181
 182   return -ENOENT;
 183 }
 184
 185 template <typename I>
 186 int open_source_image(librados::IoCtx& io_ctx, const std::string &image_name,
 187                       I **src_image_ctx, librados::IoCtx *dst_io_ctx,
 188                       std::string *dst_image_name, std::string *dst_image_id,
 189                       bool *flatten, bool *mirroring,
 190                       cls::rbd::MigrationState *state,
 191                       std::string *state_description) {
 192   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 193
 194   librados::IoCtx src_io_ctx;
 195   std::string src_image_name;
 196   std::string src_image_id;
 197   cls::rbd::MigrationSpec migration_spec;
 198   I *image_ctx = I::create(image_name, "", nullptr, io_ctx, false);
 199
 200   ldout(cct, 10) << "trying to open image by name " << io_ctx.get_pool_name()
 201                  << "/" << image_name << dendl;
 202
 203   int r = image_ctx->state->open(OPEN_FLAG_IGNORE_MIGRATING);
 204   if (r < 0) {
 205     if (r != -ENOENT) {
 206       lderr(cct) << "failed to open image: " << cpp_strerror(r) << dendl;
 207       return r;
 208     }
 209     image_ctx = nullptr;
 210   }
 211
 212   BOOST_SCOPE_EXIT_TPL(&r, &image_ctx) {
 213     if (r != 0 && image_ctx != nullptr) {
 214       image_ctx->state->close();
 215     }
 216   } BOOST_SCOPE_EXIT_END;
 217
 218   if (r == 0) {
 219     // The opened image is either a source (then just proceed) or a
 220     // destination (then look for the source image id in the migration
 221     // header).
 222
 223     r = cls_client::migration_get(&image_ctx->md_ctx, image_ctx->header_oid,
 224                                   &migration_spec);
 225
 226     if (r < 0) {
 227       lderr(cct) << "failed retrieving migration header: " << cpp_strerror(r)
 228                  << dendl;
 229       return r;
 230     }
 231
 232     ldout(cct, 10) << "migration spec: " << migration_spec << dendl;
 233
 234     if (migration_spec.header_type != cls::rbd::MIGRATION_HEADER_TYPE_SRC &&
 235         migration_spec.header_type != cls::rbd::MIGRATION_HEADER_TYPE_DST) {
 236         lderr(cct) << "unexpected migration header type: "
 237                    << migration_spec.header_type << dendl;
 238         r = -EINVAL;
 239         return r;
 240     }
 241
 242     if (migration_spec.header_type == cls::rbd::MIGRATION_HEADER_TYPE_DST) {
 243       ldout(cct, 10) << "the destination image is opened" << dendl;
 244
 245       // Close and look for the source image.
 246       r = image_ctx->state->close();
 247       image_ctx = nullptr;
 248       if (r < 0) {
 249         lderr(cct) << "failed closing image: " << cpp_strerror(r)
 250                    << dendl;
 251         return r;
 252       }
 253
 254       r = util::create_ioctx(io_ctx, "source image", migration_spec.pool_id,
 255                              migration_spec.pool_namespace, &src_io_ctx);
 256       if (r < 0) {
 257         return r;
 258       }
 259
 260       src_image_name = migration_spec.image_name;
 261       src_image_id = migration_spec.image_id;
 262     } else {
 263       ldout(cct, 10) << "the source image is opened" << dendl;
 264     }
 265   } else {
 266     assert (r == -ENOENT);
 267
 268     ldout(cct, 10) << "source image is not found. Trying trash" << dendl;
 269
 270     r = trash_search(io_ctx, RBD_TRASH_IMAGE_SOURCE_MIGRATION, image_name,
 271                      &src_image_id);
 272     if (r < 0) {
 273       lderr(cct) << "failed to determine image id: " << cpp_strerror(r)
 274                  << dendl;
 275       return r;
 276     }
 277
 278     ldout(cct, 10) << "source image id from trash: " << src_image_id << dendl;
 279
 280     src_io_ctx.dup(io_ctx);
 281   }
 282
 283   if (image_ctx == nullptr) {
 284     int flags = OPEN_FLAG_IGNORE_MIGRATING;
 285
 286     if (src_image_id.empty()) {
 287       ldout(cct, 20) << "trying to open v1 image by name "
 288                      << src_io_ctx.get_pool_name() << "/" << src_image_name
 289                      << dendl;
 290
 291       flags |= OPEN_FLAG_OLD_FORMAT;
 292     } else {
 293       ldout(cct, 20) << "trying to open v2 image by id "
 294                      << src_io_ctx.get_pool_name() << "/" << src_image_id
 295                      << dendl;
 296     }
 297
 298     image_ctx = I::create(src_image_name, src_image_id, nullptr, src_io_ctx,
 299                           false);
 300     r = image_ctx->state->open(flags);
 301     if (r < 0) {
 302       lderr(cct) << "failed to open source image " << src_io_ctx.get_pool_name()
 303                  << "/" << (src_image_id.empty() ? src_image_name : src_image_id)
 304                  << ": " << cpp_strerror(r) << dendl;
 305       image_ctx = nullptr;
 306       return r;
 307     }
 308
 309     r = cls_client::migration_get(&image_ctx->md_ctx, image_ctx->header_oid,
 310                                   &migration_spec);
 311     if (r < 0) {
 312       lderr(cct) << "failed retrieving migration header: " << cpp_strerror(r)
 313                  << dendl;
 314       return r;
 315     }
 316
 317     ldout(cct, 20) << "migration spec: " << migration_spec << dendl;
 318   }
 319
 320   r = util::create_ioctx(image_ctx->md_ctx, "source image",
 321                          migration_spec.pool_id, migration_spec.pool_namespace,
 322                          dst_io_ctx);
 323   if (r < 0) {
 324     return r;
 325   }
 326
 327   *src_image_ctx = image_ctx;
 328   *dst_image_name = migration_spec.image_name;
 329   *dst_image_id = migration_spec.image_id;
 330   *flatten = migration_spec.flatten;
 331   *mirroring = migration_spec.mirroring;
 332   *state = migration_spec.state;
 333   *state_description = migration_spec.state_description;
 334
 335   return 0;
 336 }
 337
 338 } // anonymous namespace
 339
 340 template <typename I>
 341 int Migration<I>::prepare(librados::IoCtx& io_ctx,
 342                           const std::string &image_name,
 343                           librados::IoCtx& dest_io_ctx,
 344                           const std::string &dest_image_name_,
 345                           ImageOptions& opts) {
 346   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 347
 348   std::string dest_image_name = dest_image_name_.empty() ? image_name :
 349     dest_image_name_;
 350
 351   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << " -> "
 352                  << dest_io_ctx.get_pool_name() << "/" << dest_image_name
 353                  << ", opts=" << opts << dendl;
 354
 355   auto image_ctx = I::create(image_name, "", nullptr, io_ctx, false);
 356   int r = image_ctx->state->open(0);
 357   if (r < 0) {
 358     lderr(cct) << "failed to open image: " << cpp_strerror(r) << dendl;
 359     return r;
 360   }
 361   BOOST_SCOPE_EXIT_TPL(image_ctx) {
 362     image_ctx->state->close();
 363   } BOOST_SCOPE_EXIT_END;
 364
 365   std::list<obj_watch_t> watchers;
 366   int flags = librbd::image::LIST_WATCHERS_FILTER_OUT_MY_INSTANCE |
 367               librbd::image::LIST_WATCHERS_FILTER_OUT_MIRROR_INSTANCES;
 368   C_SaferCond on_list_watchers;
 369   auto list_watchers_request = librbd::image::ListWatchersRequest<I>::create(
 370       *image_ctx, flags, &watchers, &on_list_watchers);
 371   list_watchers_request->send();
 372   r = on_list_watchers.wait();
 373   if (r < 0) {
 374     lderr(cct) << "failed listing watchers:" << cpp_strerror(r) << dendl;
 375     return r;
 376   }
 377   if (!watchers.empty()) {
 378     lderr(cct) << "image has watchers - not migrating" << dendl;
 379     return -EBUSY;
 380   }
 381
 382   uint64_t format = 2;
 383   if (opts.get(RBD_IMAGE_OPTION_FORMAT, &format) != 0) {
 384     opts.set(RBD_IMAGE_OPTION_FORMAT, format);
 385   }
 386   if (format != 2) {
 387     lderr(cct) << "unsupported destination image format: " << format << dendl;
 388     return -EINVAL;
 389   }
 390
 391   uint64_t features;
 392   {
 393     RWLock::RLocker snap_locker(image_ctx->snap_lock);
 394     features = image_ctx->features;
 395   }
 396   opts.get(RBD_IMAGE_OPTION_FEATURES, &features);
 397   if ((features & ~RBD_FEATURES_ALL) != 0) {
 398     lderr(cct) << "librbd does not support requested features" << dendl;
 399     return -ENOSYS;
 400   }
 401   features &= ~RBD_FEATURES_INTERNAL;
 402   features |= RBD_FEATURE_MIGRATING;
 403   opts.set(RBD_IMAGE_OPTION_FEATURES, features);
 404
 405   uint64_t order = image_ctx->order;
 406   if (opts.get(RBD_IMAGE_OPTION_ORDER, &order) != 0) {
 407     opts.set(RBD_IMAGE_OPTION_ORDER, order);
 408   }
 409   r = image::CreateRequest<I>::validate_order(cct, order);
 410   if (r < 0) {
 411     return r;
 412   }
 413
 414   uint64_t stripe_unit = image_ctx->stripe_unit;
 415   if (opts.get(RBD_IMAGE_OPTION_STRIPE_UNIT, &stripe_unit) != 0) {
 416     opts.set(RBD_IMAGE_OPTION_STRIPE_UNIT, stripe_unit);
 417   }
 418   uint64_t stripe_count = image_ctx->stripe_count;
 419   if (opts.get(RBD_IMAGE_OPTION_STRIPE_COUNT, &stripe_count) != 0) {
 420     opts.set(RBD_IMAGE_OPTION_STRIPE_COUNT, stripe_count);
 421   }
 422
 423   uint64_t flatten = 0;
 424   if (opts.get(RBD_IMAGE_OPTION_FLATTEN, &flatten) == 0) {
 425     opts.unset(RBD_IMAGE_OPTION_FLATTEN);
 426   }
 427
 428   ldout(cct, 20) << "updated opts=" << opts << dendl;
 429
 430   Migration migration(image_ctx, dest_io_ctx, dest_image_name, "", opts, flatten > 0,
 431                       false, cls::rbd::MIGRATION_STATE_PREPARING, "", nullptr);
 432   r = migration.prepare();
 433
 434   features &= ~RBD_FEATURE_MIGRATING;
 435   opts.set(RBD_IMAGE_OPTION_FEATURES, features);
 436
 437   return r;
 438 }
 439
 440 template <typename I>
 441 int Migration<I>::execute(librados::IoCtx& io_ctx,
 442                           const std::string &image_name,
 443                           ProgressContext &prog_ctx) {
 444   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 445
 446   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << dendl;
 447
 448   I *image_ctx;
 449   librados::IoCtx dest_io_ctx;
 450   std::string dest_image_name;
 451   std::string dest_image_id;
 452   bool flatten;
 453   bool mirroring;
 454   cls::rbd::MigrationState state;
 455   std::string state_description;
 456
 457   int r = open_source_image(io_ctx, image_name, &image_ctx, &dest_io_ctx,
 458                             &dest_image_name, &dest_image_id, &flatten,
 459                             &mirroring, &state, &state_description);
 460   if (r < 0) {
 461     return r;
 462   }
 463
 464   BOOST_SCOPE_EXIT_TPL(image_ctx) {
 465     image_ctx->state->close();
 466   } BOOST_SCOPE_EXIT_END;
 467
 468   if (state != cls::rbd::MIGRATION_STATE_PREPARED) {
 469     lderr(cct) << "current migration state is '" << state << "'"
 470                << " (should be 'prepared')" << dendl;
 471     return -EINVAL;
 472   }
 473
 474   ldout(cct, 5) << "migrating " << image_ctx->md_ctx.get_pool_name() << "/"
 475                 << image_ctx->name << " -> " << dest_io_ctx.get_pool_name()
 476                 << "/" << dest_image_name << dendl;
 477
 478   ImageOptions opts;
 479   Migration migration(image_ctx, dest_io_ctx, dest_image_name, dest_image_id,
 480                       opts, flatten, mirroring, state, state_description,
 481                       &prog_ctx);
 482   r = migration.execute();
 483   if (r < 0) {
 484     return r;
 485   }
 486
 487   return 0;
 488 }
 489
 490 template <typename I>
 491 int Migration<I>::abort(librados::IoCtx& io_ctx, const std::string &image_name,
 492                         ProgressContext &prog_ctx) {
 493   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 494
 495   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << dendl;
 496
 497   I *image_ctx;
 498   librados::IoCtx dest_io_ctx;
 499   std::string dest_image_name;
 500   std::string dest_image_id;
 501   bool flatten;
 502   bool mirroring;
 503   cls::rbd::MigrationState state;
 504   std::string state_description;
 505
 506   int r = open_source_image(io_ctx, image_name, &image_ctx, &dest_io_ctx,
 507                             &dest_image_name, &dest_image_id, &flatten,
 508                             &mirroring, &state, &state_description);
 509   if (r < 0) {
 510     return r;
 511   }
 512
 513   ldout(cct, 5) << "canceling incomplete migration "
 514                 << image_ctx->md_ctx.get_pool_name() << "/" << image_ctx->name
 515                 << " -> " << dest_io_ctx.get_pool_name() << "/" << dest_image_name
 516                 << dendl;
 517
 518   ImageOptions opts;
 519   Migration migration(image_ctx, dest_io_ctx, dest_image_name, dest_image_id,
 520                       opts, flatten, mirroring, state, state_description,
 521                       &prog_ctx);
 522   r = migration.abort();
 523
 524   image_ctx->state->close();
 525
 526   if (r < 0) {
 527     return r;
 528   }
 529
 530   return 0;
 531 }
 532
 533 template <typename I>
 534 int Migration<I>::commit(librados::IoCtx& io_ctx,
 535                          const std::string &image_name,
 536                          ProgressContext &prog_ctx) {
 537   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 538
 539   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << dendl;
 540
 541   I *image_ctx;
 542   librados::IoCtx dest_io_ctx;
 543   std::string dest_image_name;
 544   std::string dest_image_id;
 545   bool flatten;
 546   bool mirroring;
 547   cls::rbd::MigrationState state;
 548   std::string state_description;
 549
 550   int r = open_source_image(io_ctx, image_name, &image_ctx, &dest_io_ctx,
 551                             &dest_image_name, &dest_image_id, &flatten,
 552                             &mirroring, &state, &state_description);
 553   if (r < 0) {
 554     return r;
 555   }
 556
 557   if (state != cls::rbd::MIGRATION_STATE_EXECUTED) {
 558     lderr(cct) << "current migration state is '" << state << "'"
 559                << " (should be 'executed')" << dendl;
 560     image_ctx->state->close();
 561     return -EINVAL;
 562   }
 563
 564   ldout(cct, 5) << "migrating " << image_ctx->md_ctx.get_pool_name() << "/"
 565                 << image_ctx->name << " -> " << dest_io_ctx.get_pool_name()
 566                 << "/" << dest_image_name << dendl;
 567
 568   ImageOptions opts;
 569   Migration migration(image_ctx, dest_io_ctx, dest_image_name, dest_image_id,
 570                       opts, flatten, mirroring, state, state_description,
 571                       &prog_ctx);
 572   r = migration.commit();
 573
 574   // image_ctx is closed in commit when removing src image
 575
 576   if (r < 0) {
 577     return r;
 578   }
 579
 580   return 0;
 581 }
 582
 583 template <typename I>
 584 int Migration<I>::status(librados::IoCtx& io_ctx,
 585                          const std::string &image_name,
 586                          image_migration_status_t *status) {
 587   CephContext* cct = reinterpret_cast<CephContext *>(io_ctx.cct());
 588
 589   ldout(cct, 10) << io_ctx.get_pool_name() << "/" << image_name << dendl;
 590
 591   I *image_ctx;
 592   librados::IoCtx dest_io_ctx;
 593   std::string dest_image_name;
 594   std::string dest_image_id;
 595   bool flatten;
 596   bool mirroring;
 597   cls::rbd::MigrationState state;
 598   std::string state_description;
 599
 600   int r = open_source_image(io_ctx, image_name, &image_ctx, &dest_io_ctx,
 601                             &dest_image_name, &dest_image_id, &flatten,
 602                             &mirroring, &state, &state_description);
 603   if (r < 0) {
 604     return r;
 605   }
 606
 607   ldout(cct, 5) << "migrating " << image_ctx->md_ctx.get_pool_name() << "/"
 608                 << image_ctx->name << " -> " << dest_io_ctx.get_pool_name()
 609                 << "/" << dest_image_name << dendl;
 610
 611   ImageOptions opts;
 612   Migration migration(image_ctx, dest_io_ctx, dest_image_name, dest_image_id,
 613                       opts, flatten, mirroring, state, state_description,
 614                       nullptr);
 615   r = migration.status(status);
 616
 617   image_ctx->state->close();
 618
 619   if (r < 0) {
 620     return r;
 621   }
 622
 623   return 0;
 624 }
 625
 626 template <typename I>
 627 Migration<I>::Migration(I *src_image_ctx, librados::IoCtx& dst_io_ctx,
 628                         const std::string &dstname,
 629                         const std::string &dst_image_id,
 630                         ImageOptions& opts, bool flatten, bool mirroring,
 631                         cls::rbd::MigrationState state,
 632                         const std::string &state_description,
 633                         ProgressContext *prog_ctx)
 634   : m_cct(static_cast<CephContext *>(dst_io_ctx.cct())),
 635     m_src_image_ctx(src_image_ctx), m_dst_io_ctx(dst_io_ctx),
 636     m_src_old_format(m_src_image_ctx->old_format),
 637     m_src_image_name(m_src_image_ctx->old_format ? m_src_image_ctx->name : ""),
 638     m_src_image_id(m_src_image_ctx->id),
 639     m_src_header_oid(m_src_image_ctx->header_oid), m_dst_image_name(dstname),
 640     m_dst_image_id(dst_image_id.empty() ?
 641                    util::generate_image_id(m_dst_io_ctx) : dst_image_id),
 642     m_dst_header_oid(util::header_name(m_dst_image_id)), m_image_options(opts),
 643     m_flatten(flatten), m_mirroring(mirroring), m_prog_ctx(prog_ctx),
 644     m_src_migration_spec(cls::rbd::MIGRATION_HEADER_TYPE_SRC,
 645                          m_dst_io_ctx.get_id(), m_dst_io_ctx.get_namespace(),
 646                          m_dst_image_name, m_dst_image_id, {}, 0, flatten,
 647                          mirroring, state, state_description),
 648     m_dst_migration_spec(cls::rbd::MIGRATION_HEADER_TYPE_DST,
 649                          src_image_ctx->md_ctx.get_id(),
 650                          src_image_ctx->md_ctx.get_namespace(),
 651                          m_src_image_ctx->name, m_src_image_ctx->id, {}, 0,
 652                          flatten, mirroring, state, state_description) {
 653   m_src_io_ctx.dup(src_image_ctx->md_ctx);
 654 }
 655
 656 template <typename I>
 657 int Migration<I>::prepare() {
 658   ldout(m_cct, 10) << dendl;
 659
 660   int r = validate_src_snaps();
 661   if (r < 0) {
 662     return r;
 663   }
 664
 665   r = disable_mirroring(m_src_image_ctx, &m_mirroring);
 666   if (r < 0) {
 667     return r;
 668   }
 669
 670   r = unlink_src_image();
 671   if (r < 0) {
 672     enable_mirroring(m_src_image_ctx, m_mirroring);
 673     return r;
 674   }
 675
 676   r = set_migration();
 677   if (r < 0) {
 678     relink_src_image();
 679     enable_mirroring(m_src_image_ctx, m_mirroring);
 680     return r;
 681   }
 682
 683   r = create_dst_image();
 684   if (r < 0) {
 685     abort();
 686     return r;
 687   }
 688
 689   ldout(m_cct, 10) << "succeeded" << dendl;
 690
 691   return 0;
 692 }
 693
 694 template <typename I>
 695 int Migration<I>::execute() {
 696   ldout(m_cct, 10) << dendl;
 697
 698   auto dst_image_ctx = I::create(m_dst_image_name, m_dst_image_id, nullptr,
 699                                  m_dst_io_ctx, false);
 700   int r = dst_image_ctx->state->open(0);
 701   if (r < 0) {
 702     lderr(m_cct) << "failed to open destination image: " << cpp_strerror(r)
 703                  << dendl;
 704     return r;
 705   }
 706
 707   BOOST_SCOPE_EXIT_TPL(dst_image_ctx) {
 708     dst_image_ctx->state->close();
 709   } BOOST_SCOPE_EXIT_END;
 710
 711   r = set_state(cls::rbd::MIGRATION_STATE_EXECUTING, "");
 712   if (r < 0) {
 713     return r;
 714   }
 715
 716   while (true) {
 717     MigrationProgressContext prog_ctx(m_src_io_ctx, m_src_header_oid,
 718                                       cls::rbd::MIGRATION_STATE_EXECUTING,
 719                                       m_prog_ctx);
 720     r = dst_image_ctx->operations->migrate(prog_ctx);
 721     if (r == -EROFS) {
 722       RWLock::RLocker owner_locker(dst_image_ctx->owner_lock);
 723       if (dst_image_ctx->exclusive_lock != nullptr &&
 724           !dst_image_ctx->exclusive_lock->accept_ops()) {
 725         ldout(m_cct, 5) << "lost exclusive lock, retrying remote" << dendl;
 726         continue;
 727       }
 728     }
 729     break;
 730   }
 731   if (r < 0) {
 732     lderr(m_cct) << "migration failed: " << cpp_strerror(r) << dendl;
 733     return r;
 734   }
 735
 736   r = set_state(cls::rbd::MIGRATION_STATE_EXECUTED, "");
 737   if (r < 0) {
 738     return r;
 739   }
 740
 741   dst_image_ctx->notify_update();
 742
 743   ldout(m_cct, 10) << "succeeded" << dendl;
 744
 745   return 0;
 746 }
 747
 748 template <typename I>
 749 int Migration<I>::abort() {
 750   ldout(m_cct, 10) << dendl;
 751
 752   int r;
 753
 754   m_src_image_ctx->owner_lock.get_read();
 755   if (m_src_image_ctx->exclusive_lock != nullptr &&
 756       !m_src_image_ctx->exclusive_lock->is_lock_owner()) {
 757     C_SaferCond ctx;
 758     m_src_image_ctx->exclusive_lock->acquire_lock(&ctx);
 759     m_src_image_ctx->owner_lock.put_read();
 760     r = ctx.wait();
 761     if (r < 0) {
 762       lderr(m_cct) << "error acquiring exclusive lock: " << cpp_strerror(r)
 763                    << dendl;
 764       return r;
 765     }
 766   } else {
 767     m_src_image_ctx->owner_lock.put_read();
 768   }
 769
 770   group_info_t group_info;
 771   group_info.pool = -1;
 772
 773   auto dst_image_ctx = I::create(m_dst_image_name, m_dst_image_id, nullptr,
 774                                  m_dst_io_ctx, false);
 775   r = dst_image_ctx->state->open(OPEN_FLAG_IGNORE_MIGRATING);
 776   if (r < 0) {
 777     ldout(m_cct, 1) << "failed to open destination image: " << cpp_strerror(r)
 778                     << dendl;
 779   } else {
 780     ldout(m_cct, 10) << "relinking children" << dendl;
 781
 782     r = relink_children(dst_image_ctx, m_src_image_ctx);
 783     if (r < 0) {
 784       return r;
 785     }
 786
 787     ldout(m_cct, 10) << "removing dst image snapshots" << dendl;
 788
 789     BOOST_SCOPE_EXIT_TPL(&dst_image_ctx) {
 790       if (dst_image_ctx != nullptr) {
 791         dst_image_ctx->state->close();
 792       }
 793     } BOOST_SCOPE_EXIT_END;
 794
 795     std::vector<librbd::snap_info_t> snaps;
 796     r = snap_list(dst_image_ctx, snaps);
 797     if (r < 0) {
 798       lderr(m_cct) << "failed listing snapshots: " << cpp_strerror(r)
 799                    << dendl;
 800       return r;
 801     }
 802
 803     for (auto &snap : snaps) {
 804       librbd::NoOpProgressContext prog_ctx;
 805       int r = snap_remove(dst_image_ctx, snap.name.c_str(),
 806                           RBD_SNAP_REMOVE_UNPROTECT, prog_ctx);
 807       if (r < 0) {
 808         lderr(m_cct) << "failed removing snapshot: " << cpp_strerror(r)
 809                      << dendl;
 810         return r;
 811       }
 812     }
 813
 814     ldout(m_cct, 10) << "removing group" << dendl;
 815
 816     r = remove_group(dst_image_ctx, &group_info);
 817     if (r < 0 && r != -ENOENT) {
 818       return r;
 819     }
 820
 821     ldout(m_cct, 10) << "removing dst image" << dendl;
 822
 823     ceph_assert(dst_image_ctx->ignore_migrating);
 824
 825     ThreadPool *thread_pool;
 826     ContextWQ *op_work_queue;
 827     ImageCtx::get_thread_pool_instance(m_cct, &thread_pool, &op_work_queue);
 828     C_SaferCond on_remove;
 829     auto req = librbd::image::RemoveRequest<>::create(
 830       m_dst_io_ctx, dst_image_ctx, false, false, *m_prog_ctx, op_work_queue,
 831       &on_remove);
 832     req->send();
 833     r = on_remove.wait();
 834
 835     dst_image_ctx = nullptr;
 836
 837     if (r < 0) {
 838       lderr(m_cct) << "failed removing destination image '"
 839                    << m_dst_io_ctx.get_pool_name() << "/" << m_dst_image_name
 840                    << " (" << m_dst_image_id << ")': " << cpp_strerror(r)
 841                    << dendl;
 842       // not fatal
 843     }
 844   }
 845
 846   r = relink_src_image();
 847   if (r < 0) {
 848     return r;
 849   }
 850
 851   r = add_group(m_src_image_ctx, group_info);
 852   if (r < 0) {
 853     return r;
 854   }
 855
 856   r = remove_migration(m_src_image_ctx);
 857   if (r < 0) {
 858     return r;
 859   }
 860
 861   r = enable_mirroring(m_src_image_ctx, m_mirroring);
 862   if (r < 0) {
 863     return r;
 864   }
 865
 866   ldout(m_cct, 10) << "succeeded" << dendl;
 867
 868   return 0;
 869 }
 870
 871 template <typename I>
 872 int Migration<I>::commit() {
 873   ldout(m_cct, 10) << dendl;
 874
 875   BOOST_SCOPE_EXIT_TPL(&m_src_image_ctx) {
 876     if (m_src_image_ctx != nullptr) {
 877       m_src_image_ctx->state->close();
 878     }
 879   } BOOST_SCOPE_EXIT_END;
 880
 881   auto dst_image_ctx = I::create(m_dst_image_name, m_dst_image_id, nullptr,
 882                                  m_dst_io_ctx, false);
 883   int r = dst_image_ctx->state->open(0);
 884   if (r < 0) {
 885     lderr(m_cct) << "failed to open destination image: " << cpp_strerror(r)
 886                  << dendl;
 887     return r;
 888   }
 889
 890   BOOST_SCOPE_EXIT_TPL(dst_image_ctx) {
 891     dst_image_ctx->state->close();
 892   } BOOST_SCOPE_EXIT_END;
 893
 894   r = remove_migration(dst_image_ctx);
 895   if (r < 0) {
 896     return r;
 897   }
 898
 899   r = remove_src_image();
 900   if (r < 0) {
 901     return r;
 902   }
 903
 904   r = enable_mirroring(dst_image_ctx, m_mirroring);
 905   if (r < 0) {
 906     return r;
 907   }
 908
 909   ldout(m_cct, 10) << "succeeded" << dendl;
 910
 911   return 0;
 912 }
 913
 914 template <typename I>
 915 int Migration<I>::status(image_migration_status_t *status) {
 916   ldout(m_cct, 10) << dendl;
 917
 918   status->source_pool_id = m_dst_migration_spec.pool_id;
 919   status->source_pool_namespace = m_dst_migration_spec.pool_namespace;
 920   status->source_image_name = m_dst_migration_spec.image_name;
 921   status->source_image_id = m_dst_migration_spec.image_id;
 922   status->dest_pool_id = m_src_migration_spec.pool_id;
 923   status->dest_pool_namespace = m_src_migration_spec.pool_namespace;
 924   status->dest_image_name = m_src_migration_spec.image_name;
 925   status->dest_image_id = m_src_migration_spec.image_id;
 926
 927   switch (m_src_migration_spec.state) {
 928   case cls::rbd::MIGRATION_STATE_ERROR:
 929     status->state = RBD_IMAGE_MIGRATION_STATE_ERROR;
 930     break;
 931   case cls::rbd::MIGRATION_STATE_PREPARING:
 932     status->state = RBD_IMAGE_MIGRATION_STATE_PREPARING;
 933     break;
 934   case cls::rbd::MIGRATION_STATE_PREPARED:
 935     status->state = RBD_IMAGE_MIGRATION_STATE_PREPARED;
 936     break;
 937   case cls::rbd::MIGRATION_STATE_EXECUTING:
 938     status->state = RBD_IMAGE_MIGRATION_STATE_EXECUTING;
 939     break;
 940   case cls::rbd::MIGRATION_STATE_EXECUTED:
 941     status->state = RBD_IMAGE_MIGRATION_STATE_EXECUTED;
 942     break;
 943   default:
 944     status->state = RBD_IMAGE_MIGRATION_STATE_UNKNOWN;
 945     break;
 946   }
 947
 948   status->state_description = m_src_migration_spec.state_description;
 949
 950   return 0;
 951 }
 952
 953 template <typename I>
 954 int Migration<I>::set_state(cls::rbd::MigrationState state,
 955                             const std::string &description) {
 956   int r = cls_client::migration_set_state(&m_src_io_ctx, m_src_header_oid,
 957                                           state, description);
 958   if (r < 0) {
 959     lderr(m_cct) << "failed to set source migration header: " << cpp_strerror(r)
 960                  << dendl;
 961     return r;
 962   }
 963
 964   r = cls_client::migration_set_state(&m_dst_io_ctx, m_dst_header_oid, state,
 965                                       description);
 966   if (r < 0) {
 967     lderr(m_cct) << "failed to set destination migration header: "
 968                  << cpp_strerror(r) << dendl;
 969     return r;
 970   }
 971
 972   return 0;
 973 }
 974
 975 template <typename I>
 976 int Migration<I>::list_src_snaps(std::vector<librbd::snap_info_t> *snaps) {
 977   ldout(m_cct, 10) << dendl;
 978
 979   int r = snap_list(m_src_image_ctx, *snaps);
 980   if (r < 0) {
 981     lderr(m_cct) << "failed listing snapshots: " << cpp_strerror(r) << dendl;
 982     return r;
 983   }
 984
 985   for (auto &snap : *snaps) {
 986     librbd::snap_namespace_type_t namespace_type;
 987     r = Snapshot<I>::get_namespace_type(m_src_image_ctx, snap.id,
 988                                         &namespace_type);
 989     if (r < 0) {
 990       lderr(m_cct) << "error getting snap namespace type: " << cpp_strerror(r)
 991                    << dendl;
 992       return r;
 993     }
 994
 995     if (namespace_type != RBD_SNAP_NAMESPACE_TYPE_USER) {
 996       if (namespace_type == RBD_SNAP_NAMESPACE_TYPE_TRASH) {
 997         lderr(m_cct) << "image has snapshots with linked clones that must be "
 998                      << "deleted or flattened before the image can be migrated"
 999                      << dendl;
1000       } else {
1001         lderr(m_cct) << "image has non-user type snapshots "
1002                      << "that are not supported by migration" << dendl;
1003       }
1004       return -EBUSY;
1005     }
1006   }
1007
1008   return 0;
1009 }
1010
1011 template <typename I>
1012 int Migration<I>::validate_src_snaps() {
1013   ldout(m_cct, 10) << dendl;
1014
1015   std::vector<librbd::snap_info_t> snaps;
1016   int r = list_src_snaps(&snaps);
1017   if (r < 0) {
1018     return r;
1019   }
1020
1021   uint64_t dst_features = 0;
1022   r = m_image_options.get(RBD_IMAGE_OPTION_FEATURES, &dst_features);
1023   ceph_assert(r == 0);
1024
1025   if (!m_src_image_ctx->test_features(RBD_FEATURE_LAYERING)) {
1026     return 0;
1027   }
1028
1029   for (auto &snap : snaps) {
1030     RWLock::RLocker snap_locker(m_src_image_ctx->snap_lock);
1031     cls::rbd::ParentImageSpec parent_spec{m_src_image_ctx->md_ctx.get_id(),
1032                                           m_src_image_ctx->md_ctx.get_namespace(),
1033                                           m_src_image_ctx->id, snap.id};
1034     std::vector<librbd::linked_image_spec_t> child_images;
1035     r = api::Image<I>::list_children(m_src_image_ctx, parent_spec,
1036                                      &child_images);
1037     if (r < 0) {
1038       lderr(m_cct) << "failed listing children: " << cpp_strerror(r)
1039                    << dendl;
1040       return r;
1041     }
1042     if (!child_images.empty()) {
1043       ldout(m_cct, 1) << m_src_image_ctx->name << "@" << snap.name
1044                       << " has children" << dendl;
1045
1046       if ((dst_features & RBD_FEATURE_LAYERING) == 0) {
1047         lderr(m_cct) << "can't migrate to destination without layering feature: "
1048                      << "image has children" << dendl;
1049         return -EINVAL;
1050       }
1051     }
1052   }
1053
1054   return 0;
1055 }
1056
1057
1058 template <typename I>
1059 int Migration<I>::set_migration() {
1060   ldout(m_cct, 10) << dendl;
1061
1062   m_src_image_ctx->ignore_migrating = true;
1063
1064   int r = cls_client::migration_set(&m_src_io_ctx, m_src_header_oid,
1065                                     m_src_migration_spec);
1066   if (r < 0) {
1067     lderr(m_cct) << "failed to set migration header: " << cpp_strerror(r)
1068                  << dendl;
1069     return r;
1070   }
1071
1072   m_src_image_ctx->notify_update();
1073
1074   return 0;
1075 }
1076
1077 template <typename I>
1078 int Migration<I>::remove_migration(I *image_ctx) {
1079   ldout(m_cct, 10) << dendl;
1080
1081   int r;
1082
1083   r = cls_client::migration_remove(&image_ctx->md_ctx, image_ctx->header_oid);
1084   if (r == -ENOENT) {
1085     r = 0;
1086   }
1087   if (r < 0) {
1088     lderr(m_cct) << "failed removing migration header: " << cpp_strerror(r)
1089                  << dendl;
1090     return r;
1091   }
1092
1093   image_ctx->notify_update();
1094
1095   return 0;
1096 }
1097
1098 template <typename I>
1099 int Migration<I>::unlink_src_image() {
1100   if (m_src_old_format) {
1101     return v1_unlink_src_image();
1102   } else {
1103     return v2_unlink_src_image();
1104   }
1105 }
1106
1107 template <typename I>
1108 int Migration<I>::v1_unlink_src_image() {
1109   ldout(m_cct, 10) << dendl;
1110
1111   int r = tmap_rm(m_src_io_ctx, m_src_image_name);
1112   if (r < 0) {
1113     lderr(m_cct) << "failed removing " << m_src_image_name << " from tmap: "
1114                  << cpp_strerror(r) << dendl;
1115     return r;
1116   }
1117
1118   return 0;
1119 }
1120
1121 template <typename I>
1122 int Migration<I>::v2_unlink_src_image() {
1123   ldout(m_cct, 10) << dendl;
1124
1125   m_src_image_ctx->owner_lock.get_read();
1126   if (m_src_image_ctx->exclusive_lock != nullptr &&
1127       m_src_image_ctx->exclusive_lock->is_lock_owner()) {
1128     C_SaferCond ctx;
1129     m_src_image_ctx->exclusive_lock->release_lock(&ctx);
1130     m_src_image_ctx->owner_lock.put_read();
1131     int r = ctx.wait();
1132      if (r < 0) {
1133       lderr(m_cct) << "error releasing exclusive lock: " << cpp_strerror(r)
1134                    << dendl;
1135       return r;
1136      }
1137   } else {
1138     m_src_image_ctx->owner_lock.put_read();
1139   }
1140
1141   int r = Trash<I>::move(m_src_io_ctx, RBD_TRASH_IMAGE_SOURCE_MIGRATION,
1142                          m_src_image_ctx->name, 0);
1143   if (r < 0) {
1144     lderr(m_cct) << "failed moving image to trash: " << cpp_strerror(r)
1145                  << dendl;
1146     return r;
1147   }
1148
1149   return 0;
1150 }
1151
1152 template <typename I>
1153 int Migration<I>::relink_src_image() {
1154   if (m_src_old_format) {
1155     return v1_relink_src_image();
1156   } else {
1157     return v2_relink_src_image();
1158   }
1159 }
1160
1161 template <typename I>
1162 int Migration<I>::v1_relink_src_image() {
1163   ldout(m_cct, 10) << dendl;
1164
1165   int r = tmap_set(m_src_io_ctx, m_src_image_name);
1166   if (r < 0) {
1167     lderr(m_cct) << "failed adding " << m_src_image_name << " to tmap: "
1168                  << cpp_strerror(r) << dendl;
1169     return r;
1170   }
1171
1172   return 0;
1173 }
1174
1175 template <typename I>
1176 int Migration<I>::v2_relink_src_image() {
1177   ldout(m_cct, 10) << dendl;
1178
1179   int r = Trash<I>::restore(m_src_io_ctx,
1180                             {cls::rbd::TRASH_IMAGE_SOURCE_MIGRATION},
1181                             m_src_image_ctx->id, m_src_image_ctx->name);
1182   if (r < 0) {
1183     lderr(m_cct) << "failed restoring image from trash: " << cpp_strerror(r)
1184                  << dendl;
1185     return r;
1186   }
1187
1188   return 0;
1189 }
1190
1191 template <typename I>
1192 int Migration<I>::create_dst_image() {
1193   ldout(m_cct, 10) << dendl;
1194
1195   uint64_t size;
1196   cls::rbd::ParentImageSpec parent_spec;
1197   {
1198     RWLock::RLocker snap_locker(m_src_image_ctx->snap_lock);
1199     RWLock::RLocker parent_locker(m_src_image_ctx->parent_lock);
1200     size = m_src_image_ctx->size;
1201
1202     // use oldest snapshot or HEAD for parent spec
1203     if (!m_src_image_ctx->snap_info.empty()) {
1204       parent_spec = m_src_image_ctx->snap_info.begin()->second.parent.spec;
1205     } else {
1206       parent_spec = m_src_image_ctx->parent_md.spec;
1207     }
1208   }
1209
1210   ThreadPool *thread_pool;
1211   ContextWQ *op_work_queue;
1212   ImageCtx::get_thread_pool_instance(m_cct, &thread_pool, &op_work_queue);
1213
1214   ConfigProxy config{m_cct->_conf};
1215   api::Config<I>::apply_pool_overrides(m_dst_io_ctx, &config);
1216
1217   int r;
1218   C_SaferCond on_create;
1219   librados::IoCtx parent_io_ctx;
1220   if (parent_spec.pool_id == -1) {
1221     auto *req = image::CreateRequest<I>::create(
1222       config, m_dst_io_ctx, m_dst_image_name, m_dst_image_id, size,
1223       m_image_options, "", "", true /* skip_mirror_enable */, op_work_queue,
1224       &on_create);
1225     req->send();
1226   } else {
1227     r = util::create_ioctx(m_src_image_ctx->md_ctx, "destination image",
1228                            parent_spec.pool_id, parent_spec.pool_namespace,
1229                            &parent_io_ctx);
1230     if (r < 0) {
1231       return r;
1232     }
1233
1234     auto *req = image::CloneRequest<I>::create(
1235       config, parent_io_ctx, parent_spec.image_id, "", parent_spec.snap_id,
1236       m_dst_io_ctx, m_dst_image_name, m_dst_image_id, m_image_options, "", "",
1237       op_work_queue, &on_create);
1238     req->send();
1239   }
1240
1241   r = on_create.wait();
1242   if (r < 0) {
1243     lderr(m_cct) << "header creation failed: " << cpp_strerror(r) << dendl;
1244     return r;
1245   }
1246
1247   auto dst_image_ctx = I::create(m_dst_image_name, m_dst_image_id, nullptr,
1248                                  m_dst_io_ctx, false);
1249
1250   r = dst_image_ctx->state->open(OPEN_FLAG_IGNORE_MIGRATING);
1251   if (r < 0) {
1252     lderr(m_cct) << "failed to open newly created header: " << cpp_strerror(r)
1253                  << dendl;
1254     return r;
1255   }
1256
1257   BOOST_SCOPE_EXIT_TPL(dst_image_ctx) {
1258     dst_image_ctx->state->close();
1259   } BOOST_SCOPE_EXIT_END;
1260
1261   {
1262     RWLock::RLocker owner_locker(dst_image_ctx->owner_lock);
1263     r = dst_image_ctx->operations->prepare_image_update(
1264       exclusive_lock::OPERATION_REQUEST_TYPE_GENERAL, true);
1265     if (r < 0) {
1266       lderr(m_cct) << "cannot obtain exclusive lock" << dendl;
1267       return r;
1268     }
1269     if (dst_image_ctx->exclusive_lock != nullptr) {
1270       dst_image_ctx->exclusive_lock->block_requests(0);
1271     }
1272   }
1273
1274   SnapSeqs snap_seqs;
1275
1276   C_SaferCond on_snapshot_copy;
1277   auto snapshot_copy_req = librbd::deep_copy::SnapshotCopyRequest<I>::create(
1278       m_src_image_ctx, dst_image_ctx, CEPH_NOSNAP, m_flatten,
1279       m_src_image_ctx->op_work_queue, &snap_seqs, &on_snapshot_copy);
1280   snapshot_copy_req->send();
1281   r = on_snapshot_copy.wait();
1282   if (r < 0) {
1283     lderr(m_cct) << "failed to copy snapshots: " << cpp_strerror(r) << dendl;
1284     return r;
1285   }
1286
1287   C_SaferCond on_metadata_copy;
1288   auto metadata_copy_req = librbd::deep_copy::MetadataCopyRequest<I>::create(
1289       m_src_image_ctx, dst_image_ctx, &on_metadata_copy);
1290   metadata_copy_req->send();
1291   r = on_metadata_copy.wait();
1292   if (r < 0) {
1293     lderr(m_cct) << "failed to copy metadata: " << cpp_strerror(r) << dendl;
1294     return r;
1295   }
1296
1297   m_dst_migration_spec = {cls::rbd::MIGRATION_HEADER_TYPE_DST,
1298                           m_src_io_ctx.get_id(), m_src_io_ctx.get_namespace(),
1299                           m_src_image_name, m_src_image_id, snap_seqs, size,
1300                           m_flatten, m_mirroring,
1301                           cls::rbd::MIGRATION_STATE_PREPARING, ""};
1302
1303   r = cls_client::migration_set(&m_dst_io_ctx, m_dst_header_oid,
1304                                 m_dst_migration_spec);
1305   if (r < 0) {
1306     lderr(m_cct) << "failed to set migration header: " << cpp_strerror(r)
1307                  << dendl;
1308     return r;
1309   }
1310
1311   r = update_group(m_src_image_ctx, dst_image_ctx);
1312   if (r < 0) {
1313     return r;
1314   }
1315
1316   r = set_state(cls::rbd::MIGRATION_STATE_PREPARED, "");
1317   if (r < 0) {
1318     return r;
1319   }
1320
1321   r = dst_image_ctx->state->refresh();
1322   if (r < 0) {
1323     lderr(m_cct) << "failed to refresh destination image: " << cpp_strerror(r)
1324                  << dendl;
1325     return r;
1326   }
1327
1328   r = relink_children(m_src_image_ctx, dst_image_ctx);
1329   if (r < 0) {
1330     return r;
1331   }
1332
1333   return 0;
1334 }
1335
1336 template <typename I>
1337 int Migration<I>::remove_group(I *image_ctx, group_info_t *group_info) {
1338   int r = librbd::api::Group<I>::image_get_group(image_ctx, group_info);
1339   if (r < 0) {
1340     lderr(m_cct) << "failed to get image group: " << cpp_strerror(r) << dendl;
1341     return r;
1342   }
1343
1344   if (group_info->pool == -1) {
1345     return -ENOENT;
1346   }
1347
1348   ceph_assert(!image_ctx->id.empty());
1349
1350   ldout(m_cct, 10) << dendl;
1351
1352   IoCtx group_ioctx;
1353   r = util::create_ioctx(image_ctx->md_ctx, "group", group_info->pool, {},
1354                          &group_ioctx);
1355   if (r < 0) {
1356     return r;
1357   }
1358
1359   r = librbd::api::Group<I>::image_remove_by_id(group_ioctx,
1360                                                 group_info->name.c_str(),
1361                                                 image_ctx->md_ctx,
1362                                                 image_ctx->id.c_str());
1363   if (r < 0) {
1364     lderr(m_cct) << "failed to remove image from group: " << cpp_strerror(r)
1365                  << dendl;
1366     return r;
1367   }
1368
1369   return 0;
1370 }
1371
1372 template <typename I>
1373 int Migration<I>::add_group(I *image_ctx, group_info_t &group_info) {
1374   if (group_info.pool == -1) {
1375     return 0;
1376   }
1377
1378   ldout(m_cct, 10) << dendl;
1379
1380   IoCtx group_ioctx;
1381   int r = util::create_ioctx(image_ctx->md_ctx, "group", group_info.pool, {},
1382                              &group_ioctx);
1383   if (r < 0) {
1384     return r;
1385   }
1386
1387   r = librbd::api::Group<I>::image_add(group_ioctx, group_info.name.c_str(),
1388                                        image_ctx->md_ctx,
1389                                        image_ctx->name.c_str());
1390   if (r < 0) {
1391     lderr(m_cct) << "failed to add image to group: " << cpp_strerror(r)
1392                  << dendl;
1393     return r;
1394   }
1395
1396   return 0;
1397 }
1398
1399 template <typename I>
1400 int Migration<I>::update_group(I *from_image_ctx, I *to_image_ctx) {
1401   ldout(m_cct, 10) << dendl;
1402
1403   group_info_t group_info;
1404
1405   int r = remove_group(from_image_ctx, &group_info);
1406   if (r < 0) {
1407     return r == -ENOENT ? 0 : r;
1408   }
1409
1410   r = add_group(to_image_ctx, group_info);
1411   if (r < 0) {
1412     return r;
1413   }
1414
1415   return 0;
1416 }
1417
1418 template <typename I>
1419 int Migration<I>::disable_mirroring(I *image_ctx, bool *was_enabled) {
1420   *was_enabled = false;
1421
1422   if (!image_ctx->test_features(RBD_FEATURE_JOURNALING)) {
1423     return 0;
1424   }
1425
1426   cls::rbd::MirrorImage mirror_image;
1427   int r = cls_client::mirror_image_get(&image_ctx->md_ctx, image_ctx->id,
1428                                        &mirror_image);
1429   if (r == -ENOENT) {
1430     ldout(m_cct, 10) << "mirroring is not enabled for this image" << dendl;
1431     return 0;
1432   }
1433
1434   if (r < 0) {
1435     lderr(m_cct) << "failed to retrieve mirror image: " << cpp_strerror(r)
1436                  << dendl;
1437     return r;
1438   }
1439
1440   if (mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) {
1441     *was_enabled = true;
1442   }
1443
1444   ldout(m_cct, 10) << dendl;
1445
1446   C_SaferCond ctx;
1447   auto req = mirror::DisableRequest<I>::create(image_ctx, false, true, &ctx);
1448   req->send();
1449   r = ctx.wait();
1450   if (r < 0) {
1451     lderr(m_cct) << "failed to disable mirroring: " << cpp_strerror(r)
1452                  << dendl;
1453     return r;
1454   }
1455
1456   m_src_migration_spec.mirroring = true;
1457
1458   return 0;
1459 }
1460
1461 template <typename I>
1462 int Migration<I>::enable_mirroring(I *image_ctx, bool was_enabled) {
1463
1464   if (!image_ctx->test_features(RBD_FEATURE_JOURNALING)) {
1465     return 0;
1466   }
1467
1468   cls::rbd::MirrorMode mirror_mode;
1469   int r = cls_client::mirror_mode_get(&image_ctx->md_ctx, &mirror_mode);
1470   if (r < 0 && r != -ENOENT) {
1471     lderr(m_cct) << "failed to retrieve mirror mode: " << cpp_strerror(r)
1472                  << dendl;
1473     return r;
1474   }
1475
1476   if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) {
1477     ldout(m_cct, 10) << "mirroring is not enabled for destination pool"
1478                      << dendl;
1479     return 0;
1480   }
1481   if (mirror_mode == cls::rbd::MIRROR_MODE_IMAGE && !was_enabled) {
1482     ldout(m_cct, 10) << "mirroring is not enabled for image" << dendl;
1483     return 0;
1484   }
1485
1486   ldout(m_cct, 10) << dendl;
1487
1488   C_SaferCond ctx;
1489   auto req = mirror::EnableRequest<I>::create(image_ctx->md_ctx, image_ctx->id,
1490                                               "", image_ctx->op_work_queue,
1491                                               &ctx);
1492   req->send();
1493   r = ctx.wait();
1494   if (r < 0) {
1495     lderr(m_cct) << "failed to enable mirroring: " << cpp_strerror(r)
1496                  << dendl;
1497     return r;
1498   }
1499
1500   return 0;
1501 }
1502
1503 // When relinking children we should be careful as it my be interrupted
1504 // at any moment by some reason and we may end up in an inconsistent
1505 // state, which we have to be able to fix with "migration abort". Below
1506 // are all possible states during migration (P1 - sourse parent, P2 -
1507 // destination parent, C - child):
1508 //
1509 //   P1  P2    P1  P2    P1  P2    P1  P2
1510 //   ^\         \  ^      \ /^        /^
1511 //    \v         v/        v/        v/
1512 //     C         C         C         C
1513 //
1514 //     1         2         3         4
1515 //
1516 // (1) and (4) are the initial and the final consistent states. (2)
1517 // and (3) are intermediate inconsistent states that have to be fixed
1518 // by relink_children running in "migration abort" mode. For this, it
1519 // scans P2 for all children attached and relinks (fixes) states (3)
1520 // and (4) to state (1). Then it scans P1 for remaining children and
1521 // fixes the states (2).
1522
1523 template <typename I>
1524 int Migration<I>::relink_children(I *from_image_ctx, I *to_image_ctx) {
1525   ldout(m_cct, 10) << dendl;
1526
1527   std::vector<librbd::snap_info_t> snaps;
1528   int r = list_src_snaps(&snaps);
1529   if (r < 0) {
1530     return r;
1531   }
1532
1533   bool migration_abort = (to_image_ctx == m_src_image_ctx);
1534
1535   for (auto it = snaps.begin(); it != snaps.end(); it++) {
1536     auto &snap = *it;
1537     std::vector<librbd::linked_image_spec_t> src_child_images;
1538
1539     if (from_image_ctx != m_src_image_ctx) {
1540       ceph_assert(migration_abort);
1541
1542       // We run list snaps against the src image to get only those snapshots
1543       // that are migrated. If the "from" image is not the src image
1544       // (abort migration case), we need to remap snap ids.
1545       // Also collect the list of the children currently attached to the
1546       // source, so we could make a proper decision later about relinking.
1547
1548       RWLock::RLocker src_snap_locker(to_image_ctx->snap_lock);
1549       cls::rbd::ParentImageSpec src_parent_spec{to_image_ctx->md_ctx.get_id(),
1550                                                 to_image_ctx->md_ctx.get_namespace(),
1551                                                 to_image_ctx->id, snap.id};
1552       r = api::Image<I>::list_children(to_image_ctx, src_parent_spec,
1553                                        &src_child_images);
1554       if (r < 0) {
1555         lderr(m_cct) << "failed listing children: " << cpp_strerror(r)
1556                      << dendl;
1557         return r;
1558       }
1559
1560       RWLock::RLocker snap_locker(from_image_ctx->snap_lock);
1561       snap.id = from_image_ctx->get_snap_id(cls::rbd::UserSnapshotNamespace(),
1562                                             snap.name);
1563       if (snap.id == CEPH_NOSNAP) {
1564         ldout(m_cct, 5) << "skipping snapshot " << snap.name << dendl;
1565         continue;
1566       }
1567     }
1568
1569     std::vector<librbd::linked_image_spec_t> child_images;
1570     {
1571       RWLock::RLocker snap_locker(from_image_ctx->snap_lock);
1572       cls::rbd::ParentImageSpec parent_spec{from_image_ctx->md_ctx.get_id(),
1573                                             from_image_ctx->md_ctx.get_namespace(),
1574                                             from_image_ctx->id, snap.id};
1575       r = api::Image<I>::list_children(from_image_ctx, parent_spec,
1576                                        &child_images);
1577       if (r < 0) {
1578         lderr(m_cct) << "failed listing children: " << cpp_strerror(r)
1579                      << dendl;
1580         return r;
1581       }
1582     }
1583
1584     for (auto &child_image : child_images) {
1585       r = relink_child(from_image_ctx, to_image_ctx, snap, child_image,
1586                        migration_abort, true);
1587       if (r < 0) {
1588         return r;
1589       }
1590
1591       src_child_images.erase(std::remove(src_child_images.begin(),
1592                                          src_child_images.end(), child_image),
1593                              src_child_images.end());
1594     }
1595
1596     for (auto &child_image : src_child_images) {
1597       r = relink_child(from_image_ctx, to_image_ctx, snap, child_image,
1598                        migration_abort, false);
1599       if (r < 0) {
1600         return r;
1601       }
1602     }
1603   }
1604
1605   return 0;
1606 }
1607
1608 template <typename I>
1609 int Migration<I>::relink_child(I *from_image_ctx, I *to_image_ctx,
1610                                const librbd::snap_info_t &from_snap,
1611                                const librbd::linked_image_spec_t &child_image,
1612                                bool migration_abort, bool reattach_child) {
1613   ldout(m_cct, 10) << from_snap.name << " " << child_image.pool_name << "/"
1614                    << child_image.pool_namespace << "/"
1615                    << child_image.image_name << " (migration_abort="
1616                    << migration_abort << ", reattach_child=" << reattach_child
1617                    << ")" << dendl;
1618
1619   librados::snap_t to_snap_id;
1620   {
1621     RWLock::RLocker snap_locker(to_image_ctx->snap_lock);
1622     to_snap_id = to_image_ctx->get_snap_id(cls::rbd::UserSnapshotNamespace(),
1623                                              from_snap.name);
1624     if (to_snap_id == CEPH_NOSNAP) {
1625       lderr(m_cct) << "no snapshot " << from_snap.name << " on destination image"
1626                    << dendl;
1627       return -ENOENT;
1628     }
1629   }
1630
1631   librados::IoCtx child_io_ctx;
1632   int r = util::create_ioctx(to_image_ctx->md_ctx,
1633                              "child image " + child_image.image_name,
1634                              child_image.pool_id, child_image.pool_namespace,
1635                              &child_io_ctx);
1636   if (r < 0) {
1637     return r;
1638   }
1639
1640   I *child_image_ctx = I::create("", child_image.image_id, nullptr,
1641                                  child_io_ctx, false);
1642   r = child_image_ctx->state->open(OPEN_FLAG_SKIP_OPEN_PARENT);
1643   if (r < 0) {
1644     lderr(m_cct) << "failed to open child image: " << cpp_strerror(r) << dendl;
1645     return r;
1646   }
1647   BOOST_SCOPE_EXIT_TPL(child_image_ctx) {
1648     child_image_ctx->state->close();
1649   } BOOST_SCOPE_EXIT_END;
1650
1651   uint32_t clone_format = 1;
1652   if (child_image_ctx->test_op_features(RBD_OPERATION_FEATURE_CLONE_CHILD)) {
1653     clone_format = 2;
1654   }
1655
1656   cls::rbd::ParentImageSpec parent_spec;
1657   uint64_t parent_overlap;
1658   {
1659     RWLock::RLocker snap_locker(child_image_ctx->snap_lock);
1660     RWLock::RLocker parent_locker(child_image_ctx->parent_lock);
1661
1662     // use oldest snapshot or HEAD for parent spec
1663     if (!child_image_ctx->snap_info.empty()) {
1664       parent_spec = child_image_ctx->snap_info.begin()->second.parent.spec;
1665       parent_overlap = child_image_ctx->snap_info.begin()->second.parent.overlap;
1666     } else {
1667       parent_spec = child_image_ctx->parent_md.spec;
1668       parent_overlap = child_image_ctx->parent_md.overlap;
1669     }
1670   }
1671
1672   if (migration_abort &&
1673       parent_spec.pool_id == to_image_ctx->md_ctx.get_id() &&
1674       parent_spec.pool_namespace == to_image_ctx->md_ctx.get_namespace() &&
1675       parent_spec.image_id == to_image_ctx->id &&
1676       parent_spec.snap_id == to_snap_id) {
1677     ldout(m_cct, 10) << "no need for parent re-attach" << dendl;
1678   } else {
1679     if (parent_spec.pool_id != from_image_ctx->md_ctx.get_id() ||
1680         parent_spec.pool_namespace != from_image_ctx->md_ctx.get_namespace() ||
1681         parent_spec.image_id != from_image_ctx->id ||
1682         parent_spec.snap_id != from_snap.id) {
1683       lderr(m_cct) << "parent is not source image: " << parent_spec.pool_id
1684                    << "/" << parent_spec.pool_namespace << "/"
1685                    << parent_spec.image_id << "@" << parent_spec.snap_id
1686                    << dendl;
1687       return -ESTALE;
1688     }
1689
1690     parent_spec.pool_id = to_image_ctx->md_ctx.get_id();
1691     parent_spec.pool_namespace = to_image_ctx->md_ctx.get_namespace();
1692     parent_spec.image_id = to_image_ctx->id;
1693     parent_spec.snap_id = to_snap_id;
1694
1695     C_SaferCond on_reattach_parent;
1696     auto reattach_parent_req = image::AttachParentRequest<I>::create(
1697       *child_image_ctx, parent_spec, parent_overlap, true, &on_reattach_parent);
1698     reattach_parent_req->send();
1699     r = on_reattach_parent.wait();
1700     if (r < 0) {
1701       lderr(m_cct) << "failed to re-attach parent: " << cpp_strerror(r) << dendl;
1702       return r;
1703     }
1704   }
1705
1706   if (reattach_child) {
1707     C_SaferCond on_reattach_child;
1708     auto reattach_child_req = image::AttachChildRequest<I>::create(
1709       child_image_ctx, to_image_ctx, to_snap_id, from_image_ctx, from_snap.id,
1710       clone_format, &on_reattach_child);
1711     reattach_child_req->send();
1712     r = on_reattach_child.wait();
1713     if (r < 0) {
1714       lderr(m_cct) << "failed to re-attach child: " << cpp_strerror(r) << dendl;
1715       return r;
1716     }
1717   }
1718
1719   child_image_ctx->notify_update();
1720
1721   return 0;
1722 }
1723
1724 template <typename I>
1725 int Migration<I>::remove_src_image() {
1726   ldout(m_cct, 10) << dendl;
1727
1728   std::vector<librbd::snap_info_t> snaps;
1729   int r = list_src_snaps(&snaps);
1730   if (r < 0) {
1731     return r;
1732   }
1733
1734   for (auto it = snaps.rbegin(); it != snaps.rend(); it++) {
1735     auto &snap = *it;
1736
1737     librbd::NoOpProgressContext prog_ctx;
1738     int r = snap_remove(m_src_image_ctx, snap.name.c_str(),
1739                         RBD_SNAP_REMOVE_UNPROTECT, prog_ctx);
1740     if (r < 0) {
1741       lderr(m_cct) << "failed removing source image snapshot '" << snap.name
1742                    << "': " << cpp_strerror(r) << dendl;
1743       return r;
1744     }
1745   }
1746
1747   ceph_assert(m_src_image_ctx->ignore_migrating);
1748
1749   ThreadPool *thread_pool;
1750   ContextWQ *op_work_queue;
1751   ImageCtx::get_thread_pool_instance(m_cct, &thread_pool, &op_work_queue);
1752   C_SaferCond on_remove;
1753   auto req = librbd::image::RemoveRequest<I>::create(
1754       m_src_io_ctx, m_src_image_ctx, false, true, *m_prog_ctx, op_work_queue,
1755       &on_remove);
1756   req->send();
1757   r = on_remove.wait();
1758
1759   m_src_image_ctx = nullptr;
1760
1761   // For old format image it will return -ENOENT due to expected
1762   // tmap_rm failure at the end.
1763   if (r < 0 && r != -ENOENT) {
1764     lderr(m_cct) << "failed removing source image: " << cpp_strerror(r)
1765                  << dendl;
1766     return r;
1767   }
1768
1769   if (!m_src_image_id.empty()) {
1770     r = cls_client::trash_remove(&m_src_io_ctx, m_src_image_id);
1771     if (r < 0 && r != -ENOENT) {
1772       lderr(m_cct) << "error removing image " << m_src_image_id
1773                    << " from rbd_trash object" << dendl;
1774     }
1775   }
1776
1777   return 0;
1778 }
1779
1780 } // namespace api
1781 } // namespace librbd
1782
1783 template class librbd::api::Migration<librbd::ImageCtx>;