ceph/src/test/librbd/test_mirroring.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 SUSE LINUX GmbH
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14 #include "test/librbd/test_fixture.h"
  15 #include "test/librbd/test_support.h"
  16 #include "librbd/ExclusiveLock.h"
  17 #include "librbd/ImageState.h"
  18 #include "librbd/ImageWatcher.h"
  19 #include "librbd/internal.h"
  20 #include "librbd/ObjectMap.h"
  21 #include "librbd/Operations.h"
  22 #include "librbd/api/Image.h"
  23 #include "librbd/api/Namespace.h"
  24 #include "librbd/io/AioCompletion.h"
  25 #include "librbd/io/ImageRequest.h"
  26 #include "librbd/journal/Types.h"
  27 #include "librbd/mirror/snapshot/GetImageStateRequest.h"
  28 #include "librbd/mirror/snapshot/RemoveImageStateRequest.h"
  29 #include "librbd/mirror/snapshot/SetImageStateRequest.h"
  30 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
  31 #include "journal/Journaler.h"
  32 #include "journal/Settings.h"
  33 #include "common/Cond.h"
  34 #include <boost/scope_exit.hpp>
  35 #include <boost/assign/list_of.hpp>
  36 #include <utility>
  37 #include <vector>
  38
  39 using namespace std;
  40
  41 void register_test_mirroring() {
  42 }
  43
  44 namespace librbd {
  45
  46 static bool operator==(const mirror_peer_site_t& lhs,
  47                        const mirror_peer_site_t& rhs) {
  48   return (lhs.uuid == rhs.uuid &&
  49           lhs.direction == rhs.direction &&
  50           lhs.site_name == rhs.site_name &&
  51           lhs.client_name == rhs.client_name &&
  52           lhs.last_seen == rhs.last_seen);
  53 }
  54
  55 static std::ostream& operator<<(std::ostream& os,
  56                                 const mirror_peer_site_t& rhs) {
  57   os << "uuid=" << rhs.uuid << ", "
  58      << "direction=" << rhs.direction << ", "
  59      << "site_name=" << rhs.site_name << ", "
  60      << "client_name=" << rhs.client_name << ", "
  61      << "last_seen=" << rhs.last_seen;
  62   return os;
  63 }
  64
  65 };
  66
  67 class TestMirroring : public TestFixture {
  68 public:
  69
  70   TestMirroring() {}
  71
  72
  73   void TearDown() override {
  74     unlock_image();
  75
  76     TestFixture::TearDown();
  77   }
  78
  79   void SetUp() override {
  80     ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), m_ioctx));
  81   }
  82
  83   std::string image_name = "mirrorimg1";
  84
  85   int get_local_mirror_image_site_status(
  86       const librbd::mirror_image_global_status_t& status,
  87       librbd::mirror_image_site_status_t* local_status) {
  88     auto it = std::find_if(status.site_statuses.begin(),
  89                            status.site_statuses.end(),
  90                            [](auto& site_status) {
  91         return (site_status.mirror_uuid ==
  92                   RBD_MIRROR_IMAGE_STATUS_LOCAL_MIRROR_UUID);
  93       });
  94     if (it == status.site_statuses.end()) {
  95       return -ENOENT;
  96     }
  97
  98     *local_status = *it;
  99     return 0;
 100   }
 101
 102   void check_mirror_image_enable(
 103       rbd_mirror_mode_t mirror_mode, uint64_t features, int expected_r,
 104       rbd_mirror_image_state_t mirror_state,
 105       rbd_mirror_image_mode_t mirror_image_mode = RBD_MIRROR_IMAGE_MODE_JOURNAL) {
 106
 107     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 108
 109     int order = 20;
 110     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
 111     librbd::Image image;
 112     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 113
 114     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 115
 116     ASSERT_EQ(expected_r, image.mirror_image_enable2(mirror_image_mode));
 117
 118     librbd::mirror_image_info_t mirror_image;
 119     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 120     ASSERT_EQ(mirror_state, mirror_image.state);
 121
 122     if (mirror_image.state == RBD_MIRROR_IMAGE_ENABLED) {
 123       librbd::mirror_image_mode_t mode;
 124       ASSERT_EQ(0, image.mirror_image_get_mode(&mode));
 125       ASSERT_EQ(mirror_image_mode, mode);
 126     }
 127
 128     librbd::mirror_image_global_status_t status;
 129     ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 130     librbd::mirror_image_site_status_t local_status;
 131     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 132     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 133
 134     std::string instance_id;
 135     ASSERT_EQ(mirror_state == RBD_MIRROR_IMAGE_ENABLED ? -ENOENT : -EINVAL,
 136               image.mirror_image_get_instance_id(&instance_id));
 137
 138     ASSERT_EQ(0, image.close());
 139     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 140     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 141   }
 142
 143   void check_mirror_image_disable(rbd_mirror_mode_t mirror_mode,
 144                                   uint64_t features,
 145                                   int expected_r,
 146                                   rbd_mirror_image_state_t mirror_state) {
 147
 148     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 149
 150     int order = 20;
 151     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
 152     librbd::Image image;
 153     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 154
 155     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 156
 157     ASSERT_EQ(expected_r, image.mirror_image_disable(false));
 158
 159     librbd::mirror_image_info_t mirror_image;
 160     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 161     ASSERT_EQ(mirror_state, mirror_image.state);
 162
 163     librbd::mirror_image_global_status_t status;
 164     ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 165     librbd::mirror_image_site_status_t local_status;
 166     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 167     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 168
 169     std::string instance_id;
 170     ASSERT_EQ(mirror_state == RBD_MIRROR_IMAGE_ENABLED ? -ENOENT : -EINVAL,
 171               image.mirror_image_get_instance_id(&instance_id));
 172
 173     ASSERT_EQ(0, image.close());
 174     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 175     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 176   }
 177
 178   void check_mirroring_status(size_t *images_count) {
 179     std::map<std::string, librbd::mirror_image_global_status_t> images;
 180     ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, "", 4096,
 181                                                        &images));
 182
 183     std::map<librbd::mirror_image_status_state_t, int> states;
 184     ASSERT_EQ(0, m_rbd.mirror_image_status_summary(m_ioctx, &states));
 185     size_t states_count = 0;
 186     for (auto &s : states) {
 187       states_count += s.second;
 188     }
 189     ASSERT_EQ(images.size(), states_count);
 190
 191     *images_count = images.size();
 192
 193     std::map<std::string, std::string> instance_ids;
 194     ASSERT_EQ(0, m_rbd.mirror_image_instance_id_list(m_ioctx, "", 4096,
 195                                                      &instance_ids));
 196     ASSERT_TRUE(instance_ids.empty());
 197   }
 198
 199   void check_mirroring_on_create(uint64_t features,
 200                                  rbd_mirror_mode_t mirror_mode,
 201                                  rbd_mirror_image_state_t mirror_state) {
 202
 203     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 204
 205     size_t mirror_images_count = 0;
 206     check_mirroring_status(&mirror_images_count);
 207
 208     int order = 20;
 209     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
 210     librbd::Image image;
 211     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 212
 213     librbd::mirror_image_info_t mirror_image;
 214     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 215     ASSERT_EQ(mirror_state, mirror_image.state);
 216
 217     librbd::mirror_image_global_status_t status;
 218     ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 219     librbd::mirror_image_site_status_t local_status;
 220     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 221     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 222
 223     size_t mirror_images_new_count = 0;
 224     check_mirroring_status(&mirror_images_new_count);
 225     if (mirror_mode == RBD_MIRROR_MODE_POOL &&
 226         mirror_state == RBD_MIRROR_IMAGE_ENABLED) {
 227       ASSERT_EQ(mirror_images_new_count, mirror_images_count + 1);
 228     } else {
 229       ASSERT_EQ(mirror_images_new_count, mirror_images_count);
 230     }
 231
 232     ASSERT_EQ(0, image.close());
 233     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 234     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 235
 236     check_mirroring_status(&mirror_images_new_count);
 237     ASSERT_EQ(mirror_images_new_count, mirror_images_count);
 238   }
 239
 240   void check_mirroring_on_update_features(
 241       uint64_t init_features, bool enable, bool enable_mirroring,
 242       uint64_t features, int expected_r, rbd_mirror_mode_t mirror_mode,
 243       rbd_mirror_image_state_t mirror_state,
 244       rbd_mirror_image_mode_t mirror_image_mode = RBD_MIRROR_IMAGE_MODE_JOURNAL) {
 245
 246     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 247
 248     int order = 20;
 249     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, init_features, &order));
 250     librbd::Image image;
 251     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 252
 253     if (enable_mirroring) {
 254       ASSERT_EQ(0, image.mirror_image_enable2(mirror_image_mode));
 255     }
 256
 257     ASSERT_EQ(expected_r, image.update_features(features, enable));
 258
 259     librbd::mirror_image_info_t mirror_image;
 260     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 261     ASSERT_EQ(mirror_state, mirror_image.state);
 262
 263     if (mirror_image.state == RBD_MIRROR_IMAGE_ENABLED) {
 264       librbd::mirror_image_mode_t mode;
 265       ASSERT_EQ(0, image.mirror_image_get_mode(&mode));
 266       ASSERT_EQ(mirror_image_mode, mode);
 267     }
 268
 269     librbd::mirror_image_global_status_t status;
 270     ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 271     librbd::mirror_image_site_status_t local_status;
 272     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 273     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 274
 275     ASSERT_EQ(0, image.close());
 276     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 277     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 278   }
 279
 280   void setup_images_with_mirror_mode(rbd_mirror_mode_t mirror_mode,
 281                                         std::vector<uint64_t>& features_vec) {
 282
 283     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 284
 285     int id = 1;
 286     int order = 20;
 287     for (const auto& features : features_vec) {
 288       std::stringstream img_name("img_");
 289       img_name << id++;
 290       std::string img_name_str = img_name.str();
 291       ASSERT_EQ(0, m_rbd.create2(m_ioctx, img_name_str.c_str(), 2048, features, &order));
 292     }
 293   }
 294
 295   void check_mirroring_on_mirror_mode_set(rbd_mirror_mode_t mirror_mode,
 296                             std::vector<rbd_mirror_image_state_t>& states_vec) {
 297
 298     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 299
 300     std::vector< std::tuple<std::string, rbd_mirror_image_state_t> > images;
 301     int id = 1;
 302     for (const auto& mirror_state : states_vec) {
 303       std::stringstream img_name("img_");
 304       img_name << id++;
 305       std::string img_name_str = img_name.str();
 306       librbd::Image image;
 307       ASSERT_EQ(0, m_rbd.open(m_ioctx, image, img_name_str.c_str()));
 308       images.push_back(std::make_tuple(img_name_str, mirror_state));
 309
 310       librbd::mirror_image_info_t mirror_image;
 311       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 312       ASSERT_EQ(mirror_state, mirror_image.state);
 313
 314       librbd::mirror_image_global_status_t status;
 315       ASSERT_EQ(0, image.mirror_image_get_global_status(&status,
 316                                                         sizeof(status)));
 317       librbd::mirror_image_site_status_t local_status;
 318       ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 319       ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 320
 321       ASSERT_EQ(0, image.close());
 322       ASSERT_EQ(0, m_rbd.remove(m_ioctx, img_name_str.c_str()));
 323     }
 324   }
 325
 326   void check_remove_image(rbd_mirror_mode_t mirror_mode, uint64_t features,
 327                           bool enable_mirroring, bool demote = false) {
 328
 329     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 330
 331     int order = 20;
 332     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 333               &order));
 334     librbd::Image image;
 335     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 336
 337     if (enable_mirroring) {
 338       ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL));
 339     }
 340
 341     if (demote) {
 342       ASSERT_EQ(0, image.mirror_image_demote());
 343       ASSERT_EQ(0, image.mirror_image_disable(true));
 344     }
 345
 346     image.close();
 347     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 348     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 349   }
 350
 351   void check_trash_move_restore(rbd_mirror_mode_t mirror_mode,
 352                                 bool enable_mirroring) {
 353
 354     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 355
 356     int order = 20;
 357     uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 358     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 359                                &order));
 360     librbd::Image image;
 361     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 362
 363     if (enable_mirroring) {
 364       ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL));
 365     }
 366
 367     std::string image_id;
 368     ASSERT_EQ(0, image.get_id(&image_id));
 369     image.close();
 370
 371     ASSERT_EQ(0, m_rbd.trash_move(m_ioctx, image_name.c_str(), 100));
 372
 373     ASSERT_EQ(0, m_rbd.open_by_id(m_ioctx, image, image_id.c_str(), NULL));
 374
 375     librbd::mirror_image_info_t mirror_image;
 376     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 377     ASSERT_EQ(mirror_image.state, RBD_MIRROR_IMAGE_DISABLED);
 378
 379     ASSERT_EQ(0, m_rbd.trash_restore(m_ioctx, image_id.c_str(), ""));
 380
 381     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 382     if (mirror_mode == RBD_MIRROR_MODE_POOL) {
 383       ASSERT_EQ(mirror_image.state, RBD_MIRROR_IMAGE_ENABLED);
 384     } else {
 385       ASSERT_EQ(mirror_image.state, RBD_MIRROR_IMAGE_DISABLED);
 386     }
 387
 388     image.close();
 389     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 390     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 391   }
 392
 393   void setup_mirror_peer(librados::IoCtx &io_ctx, librbd::Image &image) {
 394     ASSERT_EQ(0, image.snap_create("sync-point-snap"));
 395
 396     std::string image_id;
 397     ASSERT_EQ(0, get_image_id(image, &image_id));
 398
 399     librbd::journal::MirrorPeerClientMeta peer_client_meta(
 400       "remote-image-id", {{{}, "sync-point-snap", boost::none}}, {});
 401     librbd::journal::ClientData client_data(peer_client_meta);
 402
 403     journal::Journaler journaler(io_ctx, image_id, "peer-client", {}, nullptr);
 404     C_SaferCond init_ctx;
 405     journaler.init(&init_ctx);
 406     ASSERT_EQ(-ENOENT, init_ctx.wait());
 407
 408     bufferlist client_data_bl;
 409     encode(client_data, client_data_bl);
 410     ASSERT_EQ(0, journaler.register_client(client_data_bl));
 411
 412     C_SaferCond shut_down_ctx;
 413     journaler.shut_down(&shut_down_ctx);
 414     ASSERT_EQ(0, shut_down_ctx.wait());
 415   }
 416
 417 };
 418
 419 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeImage) {
 420   uint64_t features = 0;
 421   features |= RBD_FEATURE_OBJECT_MAP;
 422   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 423   features |= RBD_FEATURE_JOURNALING;
 424   check_mirror_image_enable(RBD_MIRROR_MODE_IMAGE, features, 0,
 425       RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_MODE_JOURNAL);
 426 }
 427
 428 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModePool) {
 429   uint64_t features = 0;
 430   features |= RBD_FEATURE_OBJECT_MAP;
 431   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 432   features |= RBD_FEATURE_JOURNALING;
 433   check_mirror_image_enable(RBD_MIRROR_MODE_POOL, features, -EINVAL,
 434       RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_MODE_JOURNAL);
 435 }
 436
 437 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeDisabled) {
 438   uint64_t features = 0;
 439   features |= RBD_FEATURE_OBJECT_MAP;
 440   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 441   features |= RBD_FEATURE_JOURNALING;
 442   check_mirror_image_enable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 443       RBD_MIRROR_IMAGE_DISABLED);
 444 }
 445
 446 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModeImage) {
 447   uint64_t features = 0;
 448   features |= RBD_FEATURE_OBJECT_MAP;
 449   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 450   features |= RBD_FEATURE_JOURNALING;
 451   check_mirror_image_disable(RBD_MIRROR_MODE_IMAGE, features, 0,
 452       RBD_MIRROR_IMAGE_DISABLED);
 453 }
 454
 455 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModeImage_NoObjectMap) {
 456   uint64_t features = 0;
 457   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 458   features |= RBD_FEATURE_JOURNALING;
 459   check_mirror_image_disable(RBD_MIRROR_MODE_IMAGE, features, 0,
 460       RBD_MIRROR_IMAGE_DISABLED);
 461 }
 462
 463 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModePool) {
 464   uint64_t features = 0;
 465   features |= RBD_FEATURE_OBJECT_MAP;
 466   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 467   features |= RBD_FEATURE_JOURNALING;
 468   check_mirror_image_disable(RBD_MIRROR_MODE_POOL, features, -EINVAL,
 469       RBD_MIRROR_IMAGE_ENABLED);
 470 }
 471
 472 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModeDisabled) {
 473   uint64_t features = 0;
 474   features |= RBD_FEATURE_OBJECT_MAP;
 475   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 476   features |= RBD_FEATURE_JOURNALING;
 477   check_mirror_image_disable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 478       RBD_MIRROR_IMAGE_DISABLED);
 479 }
 480
 481 TEST_F(TestMirroring, DisableImageMirrorWithPeer) {
 482   REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
 483
 484   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 485
 486   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 487   int order = 20;
 488   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 489                              &order));
 490
 491   librbd::Image image;
 492   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 493   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL));
 494
 495   setup_mirror_peer(m_ioctx, image);
 496
 497   ASSERT_EQ(0, image.mirror_image_disable(false));
 498
 499   std::vector<librbd::snap_info_t> snaps;
 500   ASSERT_EQ(0, image.snap_list(snaps));
 501   ASSERT_TRUE(snaps.empty());
 502
 503   librbd::mirror_image_info_t mirror_image;
 504   ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 505                                            sizeof(mirror_image)));
 506   ASSERT_EQ(RBD_MIRROR_IMAGE_DISABLED, mirror_image.state);
 507
 508   librbd::mirror_image_global_status_t status;
 509   ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 510   librbd::mirror_image_site_status_t local_status;
 511   ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 512   ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 513
 514   ASSERT_EQ(0, image.close());
 515   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 516   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 517 }
 518
 519 TEST_F(TestMirroring, DisableJournalingWithPeer) {
 520   REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
 521
 522   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 523
 524   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 525   int order = 20;
 526   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 527                              &order));
 528
 529   librbd::Image image;
 530   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 531
 532   setup_mirror_peer(m_ioctx, image);
 533
 534   ASSERT_EQ(0, image.update_features(RBD_FEATURE_JOURNALING, false));
 535
 536   std::vector<librbd::snap_info_t> snaps;
 537   ASSERT_EQ(0, image.snap_list(snaps));
 538   ASSERT_TRUE(snaps.empty());
 539
 540   librbd::mirror_image_info_t mirror_image;
 541   ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 542                                            sizeof(mirror_image)));
 543   ASSERT_EQ(RBD_MIRROR_IMAGE_DISABLED, mirror_image.state);
 544
 545   librbd::mirror_image_global_status_t status;
 546   ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 547   librbd::mirror_image_site_status_t local_status;
 548   ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 549   ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 550
 551   ASSERT_EQ(0, image.close());
 552   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 553   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 554 }
 555
 556 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling) {
 557   uint64_t features = 0;
 558   features |= RBD_FEATURE_OBJECT_MAP;
 559   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 560   check_mirror_image_enable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 561       RBD_MIRROR_IMAGE_DISABLED);
 562 }
 563
 564 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModePool_WithoutJournaling) {
 565   uint64_t features = 0;
 566   features |= RBD_FEATURE_OBJECT_MAP;
 567   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 568   check_mirror_image_enable(RBD_MIRROR_MODE_POOL, features, -EINVAL,
 569       RBD_MIRROR_IMAGE_DISABLED);
 570 }
 571
 572 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeImage_WithoutJournaling) {
 573   uint64_t features = 0;
 574   features |= RBD_FEATURE_OBJECT_MAP;
 575   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 576   check_mirror_image_enable(RBD_MIRROR_MODE_IMAGE, features, 0,
 577       RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_MODE_SNAPSHOT);
 578 }
 579
 580 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock) {
 581   uint64_t features = 0;
 582   check_mirror_image_enable(RBD_MIRROR_MODE_IMAGE, features, 0,
 583       RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_MODE_SNAPSHOT);
 584 }
 585
 586 TEST_F(TestMirroring, CreateImage_In_MirrorModeDisabled) {
 587   uint64_t features = 0;
 588   features |= RBD_FEATURE_OBJECT_MAP;
 589   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 590   features |= RBD_FEATURE_JOURNALING;
 591   check_mirroring_on_create(features, RBD_MIRROR_MODE_DISABLED,
 592                             RBD_MIRROR_IMAGE_DISABLED);
 593 }
 594
 595 TEST_F(TestMirroring, CreateImage_In_MirrorModeImage) {
 596   uint64_t features = 0;
 597   features |= RBD_FEATURE_OBJECT_MAP;
 598   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 599   features |= RBD_FEATURE_JOURNALING;
 600   check_mirroring_on_create(features, RBD_MIRROR_MODE_IMAGE,
 601                             RBD_MIRROR_IMAGE_DISABLED);
 602 }
 603
 604 TEST_F(TestMirroring, CreateImage_In_MirrorModePool) {
 605   uint64_t features = 0;
 606   features |= RBD_FEATURE_OBJECT_MAP;
 607   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 608   features |= RBD_FEATURE_JOURNALING;
 609   check_mirroring_on_create(features, RBD_MIRROR_MODE_POOL,
 610                             RBD_MIRROR_IMAGE_ENABLED);
 611 }
 612
 613 TEST_F(TestMirroring, CreateImage_In_MirrorModePool_WithoutJournaling) {
 614   uint64_t features = 0;
 615   features |= RBD_FEATURE_OBJECT_MAP;
 616   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 617   check_mirroring_on_create(features, RBD_MIRROR_MODE_POOL,
 618                             RBD_MIRROR_IMAGE_DISABLED);
 619 }
 620
 621 TEST_F(TestMirroring, CreateImage_In_MirrorModeImage_WithoutJournaling) {
 622   uint64_t features = 0;
 623   features |= RBD_FEATURE_OBJECT_MAP;
 624   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 625   check_mirroring_on_create(features, RBD_MIRROR_MODE_IMAGE,
 626                             RBD_MIRROR_IMAGE_DISABLED);
 627 }
 628
 629 TEST_F(TestMirroring, EnableJournaling_In_MirrorModeDisabled) {
 630   uint64_t init_features = 0;
 631   init_features |= RBD_FEATURE_OBJECT_MAP;
 632   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 633   uint64_t features = RBD_FEATURE_JOURNALING;
 634   check_mirroring_on_update_features(init_features, true, false, features, 0,
 635                       RBD_MIRROR_MODE_DISABLED, RBD_MIRROR_IMAGE_DISABLED);
 636 }
 637
 638 TEST_F(TestMirroring, EnableJournaling_In_MirrorModeImage) {
 639   uint64_t init_features = 0;
 640   init_features |= RBD_FEATURE_OBJECT_MAP;
 641   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 642   uint64_t features = RBD_FEATURE_JOURNALING;
 643   check_mirroring_on_update_features(init_features, true, false, features, 0,
 644                       RBD_MIRROR_MODE_IMAGE, RBD_MIRROR_IMAGE_DISABLED);
 645 }
 646
 647 TEST_F(TestMirroring, EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled) {
 648   uint64_t init_features = 0;
 649   init_features |= RBD_FEATURE_OBJECT_MAP;
 650   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 651   uint64_t features = RBD_FEATURE_JOURNALING;
 652   check_mirroring_on_update_features(init_features, true, true, features,
 653                       0, RBD_MIRROR_MODE_IMAGE, RBD_MIRROR_IMAGE_ENABLED,
 654                       RBD_MIRROR_IMAGE_MODE_SNAPSHOT);
 655 }
 656
 657 TEST_F(TestMirroring, EnableJournaling_In_MirrorModePool) {
 658   uint64_t init_features = 0;
 659   init_features |= RBD_FEATURE_OBJECT_MAP;
 660   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 661   uint64_t features = RBD_FEATURE_JOURNALING;
 662   check_mirroring_on_update_features(init_features, true, false, features, 0,
 663                       RBD_MIRROR_MODE_POOL, RBD_MIRROR_IMAGE_ENABLED,
 664                       RBD_MIRROR_IMAGE_MODE_JOURNAL);
 665 }
 666
 667 TEST_F(TestMirroring, DisableJournaling_In_MirrorModePool) {
 668   uint64_t init_features = 0;
 669   init_features |= RBD_FEATURE_OBJECT_MAP;
 670   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 671   init_features |= RBD_FEATURE_JOURNALING;
 672   uint64_t features = RBD_FEATURE_JOURNALING;
 673   check_mirroring_on_update_features(init_features, false, false, features, 0,
 674                       RBD_MIRROR_MODE_POOL, RBD_MIRROR_IMAGE_DISABLED);
 675 }
 676
 677 TEST_F(TestMirroring, DisableJournaling_In_MirrorModeImage) {
 678   uint64_t init_features = 0;
 679   init_features |= RBD_FEATURE_OBJECT_MAP;
 680   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 681   init_features |= RBD_FEATURE_JOURNALING;
 682   uint64_t features = RBD_FEATURE_JOURNALING;
 683   check_mirroring_on_update_features(init_features, false, true, features,
 684                       -EINVAL, RBD_MIRROR_MODE_IMAGE, RBD_MIRROR_IMAGE_ENABLED,
 685                       RBD_MIRROR_IMAGE_MODE_JOURNAL);
 686 }
 687
 688 TEST_F(TestMirroring, MirrorModeSet_DisabledMode_To_PoolMode) {
 689   std::vector<uint64_t> features_vec;
 690   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 691   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 692
 693   setup_images_with_mirror_mode(RBD_MIRROR_MODE_DISABLED, features_vec);
 694
 695   std::vector<rbd_mirror_image_state_t> states_vec;
 696   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 697   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 698   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_POOL, states_vec);
 699 }
 700
 701 TEST_F(TestMirroring, MirrorModeSet_PoolMode_To_DisabledMode) {
 702   std::vector<uint64_t> features_vec;
 703   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 704   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 705
 706   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 707
 708   std::vector<rbd_mirror_image_state_t> states_vec;
 709   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 710   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 711   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_DISABLED, states_vec);
 712 }
 713
 714 TEST_F(TestMirroring, MirrorModeSet_DisabledMode_To_ImageMode) {
 715   std::vector<uint64_t> features_vec;
 716   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 717   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 718
 719   setup_images_with_mirror_mode(RBD_MIRROR_MODE_DISABLED, features_vec);
 720
 721   std::vector<rbd_mirror_image_state_t> states_vec;
 722   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 723   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 724   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_IMAGE, states_vec);
 725 }
 726
 727
 728 TEST_F(TestMirroring, MirrorModeSet_PoolMode_To_ImageMode) {
 729   std::vector<uint64_t> features_vec;
 730   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 731   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 732
 733   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 734
 735   std::vector<rbd_mirror_image_state_t> states_vec;
 736   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 737   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 738   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_IMAGE, states_vec);
 739 }
 740
 741 TEST_F(TestMirroring, MirrorModeSet_ImageMode_To_PoolMode) {
 742   std::vector<uint64_t> features_vec;
 743   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 744   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 745
 746   setup_images_with_mirror_mode(RBD_MIRROR_MODE_IMAGE, features_vec);
 747
 748   std::vector<rbd_mirror_image_state_t> states_vec;
 749   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 750   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 751   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_POOL, states_vec);
 752 }
 753
 754 TEST_F(TestMirroring, MirrorModeSet_ImageMode_To_DisabledMode) {
 755   std::vector<uint64_t> features_vec;
 756   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 757   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 758
 759   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 760
 761   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 762   ASSERT_EQ(-EINVAL, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 763   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 764
 765   std::vector<rbd_mirror_image_state_t> states_vec;
 766   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 767   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 768   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_DISABLED, states_vec);
 769 }
 770
 771 TEST_F(TestMirroring, RemoveImage_With_MirrorImageEnabled) {
 772   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 773                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 774                      true);
 775 }
 776
 777 TEST_F(TestMirroring, RemoveImage_With_MirrorImageDisabled) {
 778   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 779                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 780                      false);
 781 }
 782
 783 TEST_F(TestMirroring, RemoveImage_With_ImageWithoutJournal) {
 784   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 785                      RBD_FEATURE_EXCLUSIVE_LOCK,
 786                      false);
 787 }
 788
 789 TEST_F(TestMirroring, RemoveImage_With_MirrorImageDemoted) {
 790   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 791                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 792                      true, true);
 793 }
 794
 795 TEST_F(TestMirroring, TrashMoveRestore_PoolMode) {
 796   check_trash_move_restore(RBD_MIRROR_MODE_POOL, false);
 797 }
 798
 799 TEST_F(TestMirroring, TrashMoveRestore_ImageMode_MirroringDisabled) {
 800   check_trash_move_restore(RBD_MIRROR_MODE_IMAGE, false);
 801 }
 802
 803 TEST_F(TestMirroring, TrashMoveRestore_ImageMode_MirroringEnabled) {
 804   check_trash_move_restore(RBD_MIRROR_MODE_IMAGE, true);
 805 }
 806
 807 TEST_F(TestMirroring, MirrorStatusList) {
 808   std::vector<uint64_t>
 809       features_vec(5, RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 810   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 811
 812   std::string last_read = "";
 813   std::map<std::string, librbd::mirror_image_global_status_t> images;
 814   ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, last_read, 2,
 815                                                      &images));
 816   ASSERT_EQ(2U, images.size());
 817
 818   last_read = images.rbegin()->first;
 819   images.clear();
 820   ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, last_read, 2,
 821                                                      &images));
 822   ASSERT_EQ(2U, images.size());
 823
 824   last_read = images.rbegin()->first;
 825   images.clear();
 826   ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, last_read, 4096,
 827                                                      &images));
 828   ASSERT_EQ(1U, images.size());
 829
 830   last_read = images.rbegin()->first;
 831   images.clear();
 832   ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, last_read, 4096,
 833                                                      &images));
 834   ASSERT_EQ(0U, images.size());
 835 }
 836
 837 TEST_F(TestMirroring, RemoveBootstrapped)
 838 {
 839   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 840
 841   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 842   int order = 20;
 843   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 844                              &order));
 845   librbd::Image image;
 846   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 847
 848   librbd::NoOpProgressContext no_op;
 849   ASSERT_EQ(-EBUSY, librbd::api::Image<>::remove(m_ioctx, image_name, no_op));
 850
 851   // simulate the image is open by rbd-mirror bootstrap
 852   uint64_t handle;
 853   struct MirrorWatcher : public librados::WatchCtx2 {
 854     explicit MirrorWatcher(librados::IoCtx &ioctx) : m_ioctx(ioctx) {
 855     }
 856     void handle_notify(uint64_t notify_id, uint64_t cookie,
 857                                uint64_t notifier_id, bufferlist& bl) override {
 858       // received IMAGE_UPDATED notification from remove
 859       m_notified = true;
 860       m_ioctx.notify_ack(RBD_MIRRORING, notify_id, cookie, bl);
 861     }
 862     void handle_error(uint64_t cookie, int err) override {
 863     }
 864     librados::IoCtx &m_ioctx;
 865     bool m_notified = false;
 866   } watcher(m_ioctx);
 867   ASSERT_EQ(0, m_ioctx.create(RBD_MIRRORING, false));
 868   ASSERT_EQ(0, m_ioctx.watch2(RBD_MIRRORING, &handle, &watcher));
 869   // now remove should succeed
 870   ASSERT_EQ(0, librbd::api::Image<>::remove(m_ioctx, image_name, no_op));
 871   ASSERT_EQ(0, m_ioctx.unwatch2(handle));
 872   ASSERT_TRUE(watcher.m_notified);
 873   ASSERT_EQ(0, image.close());
 874 }
 875
 876 TEST_F(TestMirroring, AioPromoteDemote) {
 877   std::list<std::string> image_names;
 878   for (size_t idx = 0; idx < 10; ++idx) {
 879     image_names.push_back(get_temp_image_name());
 880   }
 881
 882   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 883
 884   // create mirror images
 885   int order = 20;
 886   std::list<librbd::Image> images;
 887   for (auto &image_name : image_names) {
 888     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
 889                                RBD_FEATURE_EXCLUSIVE_LOCK |
 890                                  RBD_FEATURE_JOURNALING,
 891                                &order));
 892
 893     images.emplace_back();
 894     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
 895     ASSERT_EQ(0, images.back().mirror_image_enable2(
 896                 RBD_MIRROR_IMAGE_MODE_JOURNAL));
 897   }
 898
 899   // demote all images
 900   std::list<librbd::RBD::AioCompletion *> aio_comps;
 901   for (auto &image : images) {
 902     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 903     ASSERT_EQ(0, image.aio_mirror_image_demote(aio_comps.back()));
 904   }
 905   for (auto aio_comp : aio_comps) {
 906     ASSERT_EQ(0, aio_comp->wait_for_complete());
 907     ASSERT_EQ(1, aio_comp->is_complete());
 908     ASSERT_EQ(0, aio_comp->get_return_value());
 909     aio_comp->release();
 910   }
 911   aio_comps.clear();
 912
 913   // verify demotions
 914   for (auto &image : images) {
 915     librbd::mirror_image_info_t mirror_image;
 916     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 917                                              sizeof(mirror_image)));
 918     ASSERT_FALSE(mirror_image.primary);
 919   }
 920
 921   // promote all images
 922   for (auto &image : images) {
 923     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 924     ASSERT_EQ(0, image.aio_mirror_image_promote(false, aio_comps.back()));
 925   }
 926   for (auto aio_comp : aio_comps) {
 927     ASSERT_EQ(0, aio_comp->wait_for_complete());
 928     ASSERT_EQ(1, aio_comp->is_complete());
 929     ASSERT_EQ(0, aio_comp->get_return_value());
 930     aio_comp->release();
 931   }
 932
 933   // verify promotions
 934   for (auto &image : images) {
 935     librbd::mirror_image_info_t mirror_image;
 936     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 937                                              sizeof(mirror_image)));
 938     ASSERT_TRUE(mirror_image.primary);
 939   }
 940 }
 941
 942 TEST_F(TestMirroring, AioGetInfo) {
 943   std::list<std::string> image_names;
 944   for (size_t idx = 0; idx < 10; ++idx) {
 945     image_names.push_back(get_temp_image_name());
 946   }
 947
 948   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 949
 950   // create mirror images
 951   int order = 20;
 952   std::list<librbd::Image> images;
 953   for (auto &image_name : image_names) {
 954     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
 955                                RBD_FEATURE_EXCLUSIVE_LOCK |
 956                                  RBD_FEATURE_JOURNALING,
 957                                &order));
 958
 959     images.emplace_back();
 960     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
 961   }
 962
 963   std::list<librbd::RBD::AioCompletion *> aio_comps;
 964   std::list<librbd::mirror_image_info_t> infos;
 965   for (auto &image : images) {
 966     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 967     infos.emplace_back();
 968     ASSERT_EQ(0, image.aio_mirror_image_get_info(&infos.back(),
 969                                                  sizeof(infos.back()),
 970                                                  aio_comps.back()));
 971   }
 972   for (auto aio_comp : aio_comps) {
 973     ASSERT_EQ(0, aio_comp->wait_for_complete());
 974     ASSERT_EQ(1, aio_comp->is_complete());
 975     ASSERT_EQ(0, aio_comp->get_return_value());
 976     aio_comp->release();
 977   }
 978   aio_comps.clear();
 979
 980   for (auto &info : infos) {
 981     ASSERT_NE("", info.global_id);
 982     ASSERT_EQ(RBD_MIRROR_IMAGE_ENABLED, info.state);
 983     ASSERT_TRUE(info.primary);
 984   }
 985 }
 986
 987 TEST_F(TestMirroring, AioGetStatus) {
 988   std::list<std::string> image_names;
 989   for (size_t idx = 0; idx < 10; ++idx) {
 990     image_names.push_back(get_temp_image_name());
 991   }
 992
 993   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 994
 995   // create mirror images
 996   int order = 20;
 997   std::list<librbd::Image> images;
 998   for (auto &image_name : image_names) {
 999     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
1000                                RBD_FEATURE_EXCLUSIVE_LOCK |
1001                                  RBD_FEATURE_JOURNALING,
1002                                &order));
1003
1004     images.emplace_back();
1005     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
1006   }
1007
1008   std::list<librbd::RBD::AioCompletion *> aio_comps;
1009   std::list<librbd::mirror_image_global_status_t> statuses;
1010   for (auto &image : images) {
1011     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
1012     statuses.emplace_back();
1013     ASSERT_EQ(0, image.aio_mirror_image_get_global_status(
1014                    &statuses.back(), sizeof(statuses.back()),
1015                    aio_comps.back()));
1016   }
1017   for (auto aio_comp : aio_comps) {
1018     ASSERT_EQ(0, aio_comp->wait_for_complete());
1019     ASSERT_EQ(1, aio_comp->is_complete());
1020     ASSERT_EQ(0, aio_comp->get_return_value());
1021     aio_comp->release();
1022   }
1023   aio_comps.clear();
1024
1025   for (auto &status : statuses) {
1026     ASSERT_NE("", status.name);
1027     ASSERT_NE("", status.info.global_id);
1028     ASSERT_EQ(RBD_MIRROR_IMAGE_ENABLED, status.info.state);
1029     ASSERT_TRUE(status.info.primary);
1030
1031     librbd::mirror_image_site_status_t local_status;
1032     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
1033     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
1034     ASSERT_EQ("status not found", local_status.description);
1035     ASSERT_FALSE(local_status.up);
1036     ASSERT_EQ(0, local_status.last_update);
1037   }
1038 }
1039
1040 TEST_F(TestMirroring, SiteName) {
1041   REQUIRE(!is_librados_test_stub(_rados));
1042
1043   const std::string expected_site_name("us-east-1a");
1044   ASSERT_EQ(0, m_rbd.mirror_site_name_set(_rados, expected_site_name));
1045
1046   std::string site_name;
1047   ASSERT_EQ(0, m_rbd.mirror_site_name_get(_rados, &site_name));
1048   ASSERT_EQ(expected_site_name, site_name);
1049
1050   ASSERT_EQ(0, m_rbd.mirror_site_name_set(_rados, ""));
1051
1052   std::string fsid;
1053   ASSERT_EQ(0, _rados.cluster_fsid(&fsid));
1054   ASSERT_EQ(0, m_rbd.mirror_site_name_get(_rados, &site_name));
1055   ASSERT_EQ(fsid, site_name);
1056 }
1057
1058 TEST_F(TestMirroring, Bootstrap) {
1059   REQUIRE(!is_librados_test_stub(_rados));
1060
1061   std::string token_b64;
1062   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1063   ASSERT_EQ(-EINVAL, m_rbd.mirror_peer_bootstrap_create(m_ioctx, &token_b64));
1064
1065   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
1066   ASSERT_EQ(0, m_rbd.mirror_peer_bootstrap_create(m_ioctx, &token_b64));
1067
1068   bufferlist token_b64_bl;
1069   token_b64_bl.append(token_b64);
1070
1071   bufferlist token_bl;
1072   token_bl.decode_base64(token_b64_bl);
1073
1074   // cannot import token into same cluster
1075   ASSERT_EQ(-EINVAL,
1076             m_rbd.mirror_peer_bootstrap_import(
1077               m_ioctx, RBD_MIRROR_PEER_DIRECTION_RX, token_b64));
1078 }
1079
1080 TEST_F(TestMirroring, PeerDirection) {
1081   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
1082
1083   std::string uuid;
1084   ASSERT_EQ(-EINVAL, m_rbd.mirror_peer_site_add(
1085                        m_ioctx, &uuid, RBD_MIRROR_PEER_DIRECTION_TX, "siteA",
1086                        "client.admin"));
1087   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &uuid,
1088                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1089                                           "siteA", "client.admin"));
1090
1091   std::vector<librbd::mirror_peer_site_t> peers;
1092   ASSERT_EQ(0, m_rbd.mirror_peer_site_list(m_ioctx, &peers));
1093   std::vector<librbd::mirror_peer_site_t> expected_peers = {
1094     {uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, "siteA", "", "client.admin", 0}};
1095   ASSERT_EQ(expected_peers, peers);
1096
1097   ASSERT_EQ(0, m_rbd.mirror_peer_site_set_direction(
1098                  m_ioctx, uuid, RBD_MIRROR_PEER_DIRECTION_RX));
1099   ASSERT_EQ(0, m_rbd.mirror_peer_site_list(m_ioctx, &peers));
1100   expected_peers = {
1101     {uuid, RBD_MIRROR_PEER_DIRECTION_RX, "siteA", "", "client.admin", 0}};
1102   ASSERT_EQ(expected_peers, peers);
1103
1104   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, uuid));
1105 }
1106
1107 TEST_F(TestMirroring, Snapshot)
1108 {
1109   REQUIRE_FORMAT_V2();
1110
1111   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1112
1113   uint64_t features;
1114   ASSERT_TRUE(get_features(&features));
1115   int order = 20;
1116   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1117                              &order));
1118
1119   librbd::Image image;
1120   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1121
1122   ASSERT_EQ(0, image.metadata_set(
1123               "conf_rbd_mirroring_max_mirroring_snapshots", "5"));
1124
1125   uint64_t snap_id;
1126
1127   ASSERT_EQ(-EINVAL, image.mirror_image_create_snapshot(&snap_id));
1128   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1129   ASSERT_EQ(-EINVAL, image.mirror_image_create_snapshot(&snap_id));
1130   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1131   librbd::mirror_image_mode_t mode;
1132   ASSERT_EQ(0, image.mirror_image_get_mode(&mode));
1133   ASSERT_EQ(RBD_MIRROR_IMAGE_MODE_SNAPSHOT, mode);
1134   ASSERT_EQ(-EINVAL, image.mirror_image_create_snapshot(&snap_id));
1135   std::string peer_uuid;
1136   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid,
1137                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1138                                           "cluster", "client"));
1139   // The mirroring was enabled when no peer was configured. Therefore, the
1140   // initial snapshot has no peers linked and will be removed after the
1141   // creation of a new mirror snapshot.
1142   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1143   vector<librbd::snap_info_t> snaps;
1144   ASSERT_EQ(0, image.snap_list(snaps));
1145   ASSERT_EQ(1U, snaps.size());
1146   ASSERT_EQ(snaps[0].id, snap_id);
1147
1148   for (int i = 0; i < 5; i++) {
1149     ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1150   }
1151   snaps.clear();
1152   ASSERT_EQ(0, image.snap_list(snaps));
1153   ASSERT_EQ(5U, snaps.size());
1154   ASSERT_EQ(snaps[4].id, snap_id);
1155
1156   // automatic peer unlink on max_mirroring_snapshots reached
1157   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1158   vector<librbd::snap_info_t> snaps1;
1159   ASSERT_EQ(0, image.snap_list(snaps1));
1160   ASSERT_EQ(5U, snaps1.size());
1161   ASSERT_EQ(snaps1[0].id, snaps[0].id);
1162   ASSERT_EQ(snaps1[1].id, snaps[1].id);
1163   ASSERT_EQ(snaps1[2].id, snaps[2].id);
1164   ASSERT_EQ(snaps1[3].id, snaps[3].id);
1165   ASSERT_EQ(snaps1[4].id, snap_id);
1166
1167   librbd::snap_namespace_type_t snap_ns_type;
1168   ASSERT_EQ(0, image.snap_get_namespace_type(snap_id, &snap_ns_type));
1169   ASSERT_EQ(RBD_SNAP_NAMESPACE_TYPE_MIRROR, snap_ns_type);
1170   librbd::snap_mirror_namespace_t mirror_snap;
1171   ASSERT_EQ(0, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1172                                                sizeof(mirror_snap)));
1173   ASSERT_EQ(1U, mirror_snap.mirror_peer_uuids.size());
1174   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer_uuid));
1175
1176   for (auto &snap : snaps1) {
1177     ASSERT_EQ(0, image.snap_remove_by_id(snap.id));
1178   }
1179
1180   ASSERT_EQ(0, image.close());
1181   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1182   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid));
1183   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1184 }
1185
1186 TEST_F(TestMirroring, SnapshotRemoveOnDisable)
1187 {
1188   REQUIRE_FORMAT_V2();
1189
1190   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1191   std::string peer_uuid;
1192   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid,
1193                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1194                                           "cluster", "client"));
1195
1196   uint64_t features;
1197   ASSERT_TRUE(get_features(&features));
1198   int order = 20;
1199   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1200                              &order));
1201   librbd::Image image;
1202   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1203   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1204   uint64_t snap_id;
1205   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1206
1207   vector<librbd::snap_info_t> snaps;
1208   ASSERT_EQ(0, image.snap_list(snaps));
1209   ASSERT_EQ(2U, snaps.size());
1210   ASSERT_EQ(snaps[1].id, snap_id);
1211
1212   ASSERT_EQ(0, image.mirror_image_disable(false));
1213
1214   snaps.clear();
1215   ASSERT_EQ(0, image.snap_list(snaps));
1216   ASSERT_TRUE(snaps.empty());
1217
1218   ASSERT_EQ(0, image.close());
1219   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1220   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid));
1221   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1222 }
1223
1224 TEST_F(TestMirroring, SnapshotUnlinkPeer)
1225 {
1226   REQUIRE_FORMAT_V2();
1227
1228   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1229   std::string peer1_uuid;
1230   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer1_uuid,
1231                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1232                                           "cluster1", "client"));
1233   std::string peer2_uuid;
1234   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer2_uuid,
1235                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1236                                           "cluster2", "client"));
1237   std::string peer3_uuid;
1238   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer3_uuid,
1239                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1240                                           "cluster3", "client"));
1241   uint64_t features;
1242   ASSERT_TRUE(get_features(&features));
1243   features &= ~RBD_FEATURE_JOURNALING;
1244   int order = 20;
1245   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1246                              &order));
1247   librbd::Image image;
1248   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1249   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1250   uint64_t snap_id;
1251   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1252   uint64_t snap_id2;
1253   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id2));
1254   librbd::snap_mirror_namespace_t mirror_snap;
1255   ASSERT_EQ(0, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1256                                                sizeof(mirror_snap)));
1257   ASSERT_EQ(3U, mirror_snap.mirror_peer_uuids.size());
1258   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer1_uuid));
1259   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1260   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer3_uuid));
1261
1262   auto ictx = new librbd::ImageCtx(image_name, "", nullptr, m_ioctx, false);
1263   ASSERT_EQ(0, ictx->state->open(0));
1264   BOOST_SCOPE_EXIT(&ictx) {
1265     if (ictx != nullptr) {
1266       ictx->state->close();
1267     }
1268   } BOOST_SCOPE_EXIT_END;
1269
1270   C_SaferCond cond1;
1271   auto req = librbd::mirror::snapshot::UnlinkPeerRequest<>::create(
1272     ictx, snap_id, peer1_uuid, true, &cond1);
1273   req->send();
1274   ASSERT_EQ(0, cond1.wait());
1275
1276   ASSERT_EQ(0, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1277                                                sizeof(mirror_snap)));
1278   ASSERT_EQ(2U, mirror_snap.mirror_peer_uuids.size());
1279   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1280   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer3_uuid));
1281
1282   ASSERT_EQ(0, librbd::api::Namespace<>::create(m_ioctx, "ns1"));
1283   librados::IoCtx ns_ioctx;
1284   ns_ioctx.dup(m_ioctx);
1285   ns_ioctx.set_namespace("ns1");
1286   ASSERT_EQ(0, m_rbd.mirror_mode_set(ns_ioctx, RBD_MIRROR_MODE_IMAGE));
1287   ASSERT_EQ(0, m_rbd.create2(ns_ioctx, image_name.c_str(), 4096, features,
1288                              &order));
1289
1290   librbd::Image ns_image;
1291   ASSERT_EQ(0, m_rbd.open(ns_ioctx, ns_image, image_name.c_str()));
1292   ASSERT_EQ(0, ns_image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1293   uint64_t ns_snap_id;
1294   ASSERT_EQ(0, ns_image.mirror_image_create_snapshot(&ns_snap_id));
1295   ASSERT_EQ(0, ns_image.snap_get_mirror_namespace(ns_snap_id, &mirror_snap,
1296                                                   sizeof(mirror_snap)));
1297   ASSERT_EQ(3U, mirror_snap.mirror_peer_uuids.size());
1298   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer1_uuid));
1299   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1300   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer3_uuid));
1301
1302   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer3_uuid));
1303
1304   ASSERT_EQ(0, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1305                                                sizeof(mirror_snap)));
1306   ASSERT_EQ(1U, mirror_snap.mirror_peer_uuids.size());
1307   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1308
1309   ASSERT_EQ(0, ns_image.snap_get_mirror_namespace(ns_snap_id, &mirror_snap,
1310                                                   sizeof(mirror_snap)));
1311   ASSERT_EQ(2U, mirror_snap.mirror_peer_uuids.size());
1312   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer1_uuid));
1313   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1314
1315   C_SaferCond cond2;
1316   req = librbd::mirror::snapshot::UnlinkPeerRequest<>::create(
1317     ictx, snap_id, peer2_uuid, true, &cond2);
1318   req->send();
1319   ASSERT_EQ(0, cond2.wait());
1320
1321   ASSERT_EQ(-ENOENT, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1322                                                      sizeof(mirror_snap)));
1323   ictx->state->close();
1324   ictx = nullptr;
1325   ASSERT_EQ(0, image.close());
1326   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1327
1328   ASSERT_EQ(0, ns_image.close());
1329   ASSERT_EQ(0, m_rbd.remove(ns_ioctx, image_name.c_str()));
1330   ASSERT_EQ(0, librbd::api::Namespace<>::remove(m_ioctx, "ns1"));
1331
1332   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer1_uuid));
1333   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer2_uuid));
1334   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1335 }
1336
1337 TEST_F(TestMirroring, SnapshotImageState)
1338 {
1339   REQUIRE_FORMAT_V2();
1340
1341   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1342
1343   uint64_t features;
1344   ASSERT_TRUE(get_features(&features));
1345   int order = 20;
1346   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1347                              &order));
1348
1349   librbd::Image image;
1350   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1351   ASSERT_EQ(0, image.snap_create("snap"));
1352   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1353   std::vector<librbd::snap_info_t> snaps;
1354   ASSERT_EQ(0, image.snap_list(snaps));
1355   ASSERT_EQ(2U, snaps.size());
1356   auto snap_id = snaps[1].id;
1357
1358   auto ictx = new librbd::ImageCtx(image_name, "", nullptr, m_ioctx, false);
1359   ASSERT_EQ(0, ictx->state->open(0));
1360   BOOST_SCOPE_EXIT(&ictx) {
1361     if (ictx != nullptr) {
1362       ictx->state->close();
1363     }
1364   } BOOST_SCOPE_EXIT_END;
1365
1366   {
1367     C_SaferCond cond;
1368     auto req = librbd::mirror::snapshot::SetImageStateRequest<>::create(
1369       ictx, snap_id, &cond);
1370     req->send();
1371     ASSERT_EQ(0, cond.wait());
1372   }
1373
1374   librbd::mirror::snapshot::ImageState image_state;
1375   {
1376     C_SaferCond cond;
1377     auto req = librbd::mirror::snapshot::GetImageStateRequest<>::create(
1378       ictx, snap_id, &image_state, &cond);
1379     req->send();
1380     ASSERT_EQ(0, cond.wait());
1381   }
1382
1383   ASSERT_EQ(image_name, image_state.name);
1384   ASSERT_EQ(0, image.features(&features));
1385   ASSERT_EQ(features & ~RBD_FEATURES_IMPLICIT_ENABLE, image_state.features);
1386   ASSERT_EQ(1U, image_state.snapshots.size());
1387   ASSERT_EQ("snap", image_state.snapshots.begin()->second.name);
1388   uint8_t original_pairs_num = image_state.metadata.size();
1389
1390   {
1391     C_SaferCond cond;
1392     auto req = librbd::mirror::snapshot::RemoveImageStateRequest<>::create(
1393       ictx, snap_id, &cond);
1394     req->send();
1395     ASSERT_EQ(0, cond.wait());
1396   }
1397
1398   // test storing "large" image state in multiple objects
1399
1400   ASSERT_EQ(0, ictx->config.set_val("rbd_default_order", "8"));
1401
1402   for (int i = 0; i < 10; i++) {
1403     ASSERT_EQ(0, image.metadata_set(stringify(i), std::string(1024, 'A' + i)));
1404   }
1405
1406   {
1407     C_SaferCond cond;
1408     auto req = librbd::mirror::snapshot::SetImageStateRequest<>::create(
1409       ictx, snap_id, &cond);
1410     req->send();
1411     ASSERT_EQ(0, cond.wait());
1412   }
1413
1414   {
1415     C_SaferCond cond;
1416     auto req = librbd::mirror::snapshot::GetImageStateRequest<>::create(
1417       ictx, snap_id, &image_state, &cond);
1418     req->send();
1419     ASSERT_EQ(0, cond.wait());
1420   }
1421
1422   ASSERT_EQ(image_name, image_state.name);
1423   ASSERT_EQ(features & ~RBD_FEATURES_IMPLICIT_ENABLE, image_state.features);
1424   ASSERT_EQ(original_pairs_num + 10, image_state.metadata.size());
1425   for (int i = 0; i < 10; i++) {
1426     auto &bl = image_state.metadata[stringify(i)];
1427     ASSERT_EQ(0, strncmp(std::string(1024, 'A' + i).c_str(), bl.c_str(),
1428                          bl.length()));
1429   }
1430
1431   {
1432     C_SaferCond cond;
1433     auto req = librbd::mirror::snapshot::RemoveImageStateRequest<>::create(
1434       ictx, snap_id, &cond);
1435     req->send();
1436     ASSERT_EQ(0, cond.wait());
1437   }
1438
1439   ASSERT_EQ(0, ictx->state->close());
1440   ictx = nullptr;
1441
1442   ASSERT_EQ(0, image.snap_remove("snap"));
1443   ASSERT_EQ(0, image.close());
1444   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1445 }
1446
1447 TEST_F(TestMirroring, SnapshotPromoteDemote)
1448 {
1449   REQUIRE_FORMAT_V2();
1450
1451   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1452   std::string peer_uuid;
1453   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid,
1454                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1455                                           "cluster", "client"));
1456
1457   uint64_t features;
1458   ASSERT_TRUE(get_features(&features));
1459   int order = 20;
1460   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1461                              &order));
1462
1463   librbd::Image image;
1464   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1465   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1466   librbd::mirror_image_mode_t mode;
1467   ASSERT_EQ(0, image.mirror_image_get_mode(&mode));
1468   ASSERT_EQ(RBD_MIRROR_IMAGE_MODE_SNAPSHOT, mode);
1469
1470   ASSERT_EQ(-EINVAL, image.mirror_image_promote(false));
1471   ASSERT_EQ(0, image.mirror_image_demote());
1472   ASSERT_EQ(0, image.mirror_image_promote(false));
1473   ASSERT_EQ(0, image.mirror_image_demote());
1474   ASSERT_EQ(0, image.mirror_image_promote(false));
1475
1476   ASSERT_EQ(0, image.close());
1477   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1478   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid));
1479   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1480 }
1481
1482 TEST_F(TestMirroring, AioSnapshotCreate)
1483 {
1484   REQUIRE_FORMAT_V2();
1485
1486   std::list<std::string> image_names;
1487   for (size_t idx = 0; idx < 10; ++idx) {
1488     image_names.push_back(get_temp_image_name());
1489   }
1490
1491   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1492   std::string peer_uuid;
1493   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid,
1494                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1495                                           "cluster", "client"));
1496   // create mirror images
1497   uint64_t features;
1498   ASSERT_TRUE(get_features(&features));
1499   int order = 20;
1500   std::list<librbd::Image> images;
1501   for (auto &image_name : image_names) {
1502     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048, features,
1503                                &order));
1504     images.emplace_back();
1505     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
1506     ASSERT_EQ(0, images.back().mirror_image_enable2(
1507                 RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1508   }
1509
1510   // create snapshots
1511   std::list<uint64_t> snap_ids;
1512   std::list<librbd::RBD::AioCompletion *> aio_comps;
1513   for (auto &image : images) {
1514     snap_ids.emplace_back();
1515     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
1516     ASSERT_EQ(0, image.aio_mirror_image_create_snapshot(0, &snap_ids.back(),
1517                                                         aio_comps.back()));
1518   }
1519   for (auto aio_comp : aio_comps) {
1520     ASSERT_EQ(0, aio_comp->wait_for_complete());
1521     ASSERT_EQ(1, aio_comp->is_complete());
1522     ASSERT_EQ(0, aio_comp->get_return_value());
1523     aio_comp->release();
1524   }
1525   aio_comps.clear();
1526
1527   // verify
1528   for (auto &image : images) {
1529     vector<librbd::snap_info_t> snaps;
1530     ASSERT_EQ(0, image.snap_list(snaps));
1531     ASSERT_EQ(2U, snaps.size());
1532     ASSERT_EQ(snaps[1].id, snap_ids.front());
1533
1534     std::string image_name;
1535     ASSERT_EQ(0, image.get_name(&image_name));
1536     ASSERT_EQ(0, image.close());
1537     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1538     snap_ids.pop_front();
1539   }
1540
1541   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid));
1542   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1543 }