ceph/src/test/librbd/test_mirroring.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 SUSE LINUX GmbH
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14 #include "test/librbd/test_fixture.h"
  15 #include "test/librbd/test_support.h"
  16 #include "librbd/ExclusiveLock.h"
  17 #include "librbd/ImageState.h"
  18 #include "librbd/ImageWatcher.h"
  19 #include "librbd/internal.h"
  20 #include "librbd/ObjectMap.h"
  21 #include "librbd/Operations.h"
  22 #include "librbd/api/Image.h"
  23 #include "librbd/api/Namespace.h"
  24 #include "librbd/io/AioCompletion.h"
  25 #include "librbd/io/ImageRequest.h"
  26 #include "librbd/io/ImageRequestWQ.h"
  27 #include "librbd/journal/Types.h"
  28 #include "librbd/mirror/snapshot/GetImageStateRequest.h"
  29 #include "librbd/mirror/snapshot/RemoveImageStateRequest.h"
  30 #include "librbd/mirror/snapshot/SetImageStateRequest.h"
  31 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
  32 #include "journal/Journaler.h"
  33 #include "journal/Settings.h"
  34 #include "common/Cond.h"
  35 #include <boost/scope_exit.hpp>
  36 #include <boost/assign/list_of.hpp>
  37 #include <utility>
  38 #include <vector>
  39
  40 void register_test_mirroring() {
  41 }
  42
  43 namespace librbd {
  44
  45 static bool operator==(const mirror_peer_site_t& lhs,
  46                        const mirror_peer_site_t& rhs) {
  47   return (lhs.uuid == rhs.uuid &&
  48           lhs.direction == rhs.direction &&
  49           lhs.site_name == rhs.site_name &&
  50           lhs.client_name == rhs.client_name &&
  51           lhs.last_seen == rhs.last_seen);
  52 }
  53
  54 static std::ostream& operator<<(std::ostream& os,
  55                                 const mirror_peer_site_t& rhs) {
  56   os << "uuid=" << rhs.uuid << ", "
  57      << "direction=" << rhs.direction << ", "
  58      << "site_name=" << rhs.site_name << ", "
  59      << "client_name=" << rhs.client_name << ", "
  60      << "last_seen=" << rhs.last_seen;
  61   return os;
  62 }
  63
  64 };
  65
  66 class TestMirroring : public TestFixture {
  67 public:
  68
  69   TestMirroring() {}
  70
  71
  72   void TearDown() override {
  73     unlock_image();
  74
  75     TestFixture::TearDown();
  76   }
  77
  78   void SetUp() override {
  79     ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), m_ioctx));
  80   }
  81
  82   std::string image_name = "mirrorimg1";
  83
  84   int get_local_mirror_image_site_status(
  85       const librbd::mirror_image_global_status_t& status,
  86       librbd::mirror_image_site_status_t* local_status) {
  87     auto it = std::find_if(status.site_statuses.begin(),
  88                            status.site_statuses.end(),
  89                            [](auto& site_status) {
  90         return (site_status.mirror_uuid ==
  91                   RBD_MIRROR_IMAGE_STATUS_LOCAL_MIRROR_UUID);
  92       });
  93     if (it == status.site_statuses.end()) {
  94       return -ENOENT;
  95     }
  96
  97     *local_status = *it;
  98     return 0;
  99   }
 100
 101   void check_mirror_image_enable(
 102       rbd_mirror_mode_t mirror_mode, uint64_t features, int expected_r,
 103       rbd_mirror_image_state_t mirror_state,
 104       rbd_mirror_image_mode_t mirror_image_mode = RBD_MIRROR_IMAGE_MODE_JOURNAL) {
 105
 106     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 107
 108     int order = 20;
 109     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
 110     librbd::Image image;
 111     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 112
 113     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 114
 115     ASSERT_EQ(expected_r, image.mirror_image_enable2(mirror_image_mode));
 116
 117     librbd::mirror_image_info_t mirror_image;
 118     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 119     ASSERT_EQ(mirror_state, mirror_image.state);
 120
 121     if (mirror_image.state == RBD_MIRROR_IMAGE_ENABLED) {
 122       librbd::mirror_image_mode_t mode;
 123       ASSERT_EQ(0, image.mirror_image_get_mode(&mode));
 124       ASSERT_EQ(mirror_image_mode, mode);
 125     }
 126
 127     librbd::mirror_image_global_status_t status;
 128     ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 129     librbd::mirror_image_site_status_t local_status;
 130     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 131     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 132
 133     std::string instance_id;
 134     ASSERT_EQ(mirror_state == RBD_MIRROR_IMAGE_ENABLED ? -ENOENT : -EINVAL,
 135               image.mirror_image_get_instance_id(&instance_id));
 136
 137     ASSERT_EQ(0, image.close());
 138     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 139     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 140   }
 141
 142   void check_mirror_image_disable(rbd_mirror_mode_t mirror_mode,
 143                                   uint64_t features,
 144                                   int expected_r,
 145                                   rbd_mirror_image_state_t mirror_state) {
 146
 147     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 148
 149     int order = 20;
 150     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
 151     librbd::Image image;
 152     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 153
 154     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 155
 156     ASSERT_EQ(expected_r, image.mirror_image_disable(false));
 157
 158     librbd::mirror_image_info_t mirror_image;
 159     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 160     ASSERT_EQ(mirror_state, mirror_image.state);
 161
 162     librbd::mirror_image_global_status_t status;
 163     ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 164     librbd::mirror_image_site_status_t local_status;
 165     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 166     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 167
 168     std::string instance_id;
 169     ASSERT_EQ(mirror_state == RBD_MIRROR_IMAGE_ENABLED ? -ENOENT : -EINVAL,
 170               image.mirror_image_get_instance_id(&instance_id));
 171
 172     ASSERT_EQ(0, image.close());
 173     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 174     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 175   }
 176
 177   void check_mirroring_status(size_t *images_count) {
 178     std::map<std::string, librbd::mirror_image_global_status_t> images;
 179     ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, "", 4096,
 180                                                        &images));
 181
 182     std::map<librbd::mirror_image_status_state_t, int> states;
 183     ASSERT_EQ(0, m_rbd.mirror_image_status_summary(m_ioctx, &states));
 184     size_t states_count = 0;
 185     for (auto &s : states) {
 186       states_count += s.second;
 187     }
 188     ASSERT_EQ(images.size(), states_count);
 189
 190     *images_count = images.size();
 191
 192     std::map<std::string, std::string> instance_ids;
 193     ASSERT_EQ(0, m_rbd.mirror_image_instance_id_list(m_ioctx, "", 4096,
 194                                                      &instance_ids));
 195     ASSERT_TRUE(instance_ids.empty());
 196   }
 197
 198   void check_mirroring_on_create(uint64_t features,
 199                                  rbd_mirror_mode_t mirror_mode,
 200                                  rbd_mirror_image_state_t mirror_state) {
 201
 202     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 203
 204     size_t mirror_images_count = 0;
 205     check_mirroring_status(&mirror_images_count);
 206
 207     int order = 20;
 208     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
 209     librbd::Image image;
 210     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 211
 212     librbd::mirror_image_info_t mirror_image;
 213     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 214     ASSERT_EQ(mirror_state, mirror_image.state);
 215
 216     librbd::mirror_image_global_status_t status;
 217     ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 218     librbd::mirror_image_site_status_t local_status;
 219     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 220     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 221
 222     size_t mirror_images_new_count = 0;
 223     check_mirroring_status(&mirror_images_new_count);
 224     if (mirror_mode == RBD_MIRROR_MODE_POOL &&
 225         mirror_state == RBD_MIRROR_IMAGE_ENABLED) {
 226       ASSERT_EQ(mirror_images_new_count, mirror_images_count + 1);
 227     } else {
 228       ASSERT_EQ(mirror_images_new_count, mirror_images_count);
 229     }
 230
 231     ASSERT_EQ(0, image.close());
 232     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 233     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 234
 235     check_mirroring_status(&mirror_images_new_count);
 236     ASSERT_EQ(mirror_images_new_count, mirror_images_count);
 237   }
 238
 239   void check_mirroring_on_update_features(
 240       uint64_t init_features, bool enable, bool enable_mirroring,
 241       uint64_t features, int expected_r, rbd_mirror_mode_t mirror_mode,
 242       rbd_mirror_image_state_t mirror_state,
 243       rbd_mirror_image_mode_t mirror_image_mode = RBD_MIRROR_IMAGE_MODE_JOURNAL) {
 244
 245     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 246
 247     int order = 20;
 248     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, init_features, &order));
 249     librbd::Image image;
 250     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 251
 252     if (enable_mirroring) {
 253       ASSERT_EQ(0, image.mirror_image_enable2(mirror_image_mode));
 254     }
 255
 256     ASSERT_EQ(expected_r, image.update_features(features, enable));
 257
 258     librbd::mirror_image_info_t mirror_image;
 259     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 260     ASSERT_EQ(mirror_state, mirror_image.state);
 261
 262     if (mirror_image.state == RBD_MIRROR_IMAGE_ENABLED) {
 263       librbd::mirror_image_mode_t mode;
 264       ASSERT_EQ(0, image.mirror_image_get_mode(&mode));
 265       ASSERT_EQ(mirror_image_mode, mode);
 266     }
 267
 268     librbd::mirror_image_global_status_t status;
 269     ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 270     librbd::mirror_image_site_status_t local_status;
 271     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 272     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 273
 274     ASSERT_EQ(0, image.close());
 275     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 276     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 277   }
 278
 279   void setup_images_with_mirror_mode(rbd_mirror_mode_t mirror_mode,
 280                                         std::vector<uint64_t>& features_vec) {
 281
 282     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 283
 284     int id = 1;
 285     int order = 20;
 286     for (const auto& features : features_vec) {
 287       std::stringstream img_name("img_");
 288       img_name << id++;
 289       std::string img_name_str = img_name.str();
 290       ASSERT_EQ(0, m_rbd.create2(m_ioctx, img_name_str.c_str(), 2048, features, &order));
 291     }
 292   }
 293
 294   void check_mirroring_on_mirror_mode_set(rbd_mirror_mode_t mirror_mode,
 295                             std::vector<rbd_mirror_image_state_t>& states_vec) {
 296
 297     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 298
 299     std::vector< std::tuple<std::string, rbd_mirror_image_state_t> > images;
 300     int id = 1;
 301     for (const auto& mirror_state : states_vec) {
 302       std::stringstream img_name("img_");
 303       img_name << id++;
 304       std::string img_name_str = img_name.str();
 305       librbd::Image image;
 306       ASSERT_EQ(0, m_rbd.open(m_ioctx, image, img_name_str.c_str()));
 307       images.push_back(std::make_tuple(img_name_str, mirror_state));
 308
 309       librbd::mirror_image_info_t mirror_image;
 310       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 311       ASSERT_EQ(mirror_state, mirror_image.state);
 312
 313       librbd::mirror_image_global_status_t status;
 314       ASSERT_EQ(0, image.mirror_image_get_global_status(&status,
 315                                                         sizeof(status)));
 316       librbd::mirror_image_site_status_t local_status;
 317       ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 318       ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 319
 320       ASSERT_EQ(0, image.close());
 321       ASSERT_EQ(0, m_rbd.remove(m_ioctx, img_name_str.c_str()));
 322     }
 323   }
 324
 325   void check_remove_image(rbd_mirror_mode_t mirror_mode, uint64_t features,
 326                           bool enable_mirroring, bool demote = false) {
 327
 328     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 329
 330     int order = 20;
 331     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 332               &order));
 333     librbd::Image image;
 334     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 335
 336     if (enable_mirroring) {
 337       ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL));
 338     }
 339
 340     if (demote) {
 341       ASSERT_EQ(0, image.mirror_image_demote());
 342       ASSERT_EQ(0, image.mirror_image_disable(true));
 343     }
 344
 345     image.close();
 346     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 347     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 348   }
 349
 350   void check_trash_move_restore(rbd_mirror_mode_t mirror_mode,
 351                                 bool enable_mirroring) {
 352
 353     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 354
 355     int order = 20;
 356     uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 357     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 358                                &order));
 359     librbd::Image image;
 360     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 361
 362     if (enable_mirroring) {
 363       ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL));
 364     }
 365
 366     std::string image_id;
 367     ASSERT_EQ(0, image.get_id(&image_id));
 368     image.close();
 369
 370     ASSERT_EQ(0, m_rbd.trash_move(m_ioctx, image_name.c_str(), 100));
 371
 372     ASSERT_EQ(0, m_rbd.open_by_id(m_ioctx, image, image_id.c_str(), NULL));
 373
 374     librbd::mirror_image_info_t mirror_image;
 375     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 376     ASSERT_EQ(mirror_image.state, RBD_MIRROR_IMAGE_DISABLED);
 377
 378     ASSERT_EQ(0, m_rbd.trash_restore(m_ioctx, image_id.c_str(), ""));
 379
 380     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 381     if (mirror_mode == RBD_MIRROR_MODE_POOL) {
 382       ASSERT_EQ(mirror_image.state, RBD_MIRROR_IMAGE_ENABLED);
 383     } else {
 384       ASSERT_EQ(mirror_image.state, RBD_MIRROR_IMAGE_DISABLED);
 385     }
 386
 387     image.close();
 388     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 389     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 390   }
 391
 392   void setup_mirror_peer(librados::IoCtx &io_ctx, librbd::Image &image) {
 393     ASSERT_EQ(0, image.snap_create("sync-point-snap"));
 394
 395     std::string image_id;
 396     ASSERT_EQ(0, get_image_id(image, &image_id));
 397
 398     librbd::journal::MirrorPeerClientMeta peer_client_meta(
 399       "remote-image-id", {{{}, "sync-point-snap", boost::none}}, {});
 400     librbd::journal::ClientData client_data(peer_client_meta);
 401
 402     journal::Journaler journaler(io_ctx, image_id, "peer-client", {}, nullptr);
 403     C_SaferCond init_ctx;
 404     journaler.init(&init_ctx);
 405     ASSERT_EQ(-ENOENT, init_ctx.wait());
 406
 407     bufferlist client_data_bl;
 408     encode(client_data, client_data_bl);
 409     ASSERT_EQ(0, journaler.register_client(client_data_bl));
 410
 411     C_SaferCond shut_down_ctx;
 412     journaler.shut_down(&shut_down_ctx);
 413     ASSERT_EQ(0, shut_down_ctx.wait());
 414   }
 415
 416 };
 417
 418 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeImage) {
 419   uint64_t features = 0;
 420   features |= RBD_FEATURE_OBJECT_MAP;
 421   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 422   features |= RBD_FEATURE_JOURNALING;
 423   check_mirror_image_enable(RBD_MIRROR_MODE_IMAGE, features, 0,
 424       RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_MODE_JOURNAL);
 425 }
 426
 427 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModePool) {
 428   uint64_t features = 0;
 429   features |= RBD_FEATURE_OBJECT_MAP;
 430   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 431   features |= RBD_FEATURE_JOURNALING;
 432   check_mirror_image_enable(RBD_MIRROR_MODE_POOL, features, -EINVAL,
 433       RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_MODE_JOURNAL);
 434 }
 435
 436 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeDisabled) {
 437   uint64_t features = 0;
 438   features |= RBD_FEATURE_OBJECT_MAP;
 439   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 440   features |= RBD_FEATURE_JOURNALING;
 441   check_mirror_image_enable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 442       RBD_MIRROR_IMAGE_DISABLED);
 443 }
 444
 445 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModeImage) {
 446   uint64_t features = 0;
 447   features |= RBD_FEATURE_OBJECT_MAP;
 448   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 449   features |= RBD_FEATURE_JOURNALING;
 450   check_mirror_image_disable(RBD_MIRROR_MODE_IMAGE, features, 0,
 451       RBD_MIRROR_IMAGE_DISABLED);
 452 }
 453
 454 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModeImage_NoObjectMap) {
 455   uint64_t features = 0;
 456   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 457   features |= RBD_FEATURE_JOURNALING;
 458   check_mirror_image_disable(RBD_MIRROR_MODE_IMAGE, features, 0,
 459       RBD_MIRROR_IMAGE_DISABLED);
 460 }
 461
 462 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModePool) {
 463   uint64_t features = 0;
 464   features |= RBD_FEATURE_OBJECT_MAP;
 465   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 466   features |= RBD_FEATURE_JOURNALING;
 467   check_mirror_image_disable(RBD_MIRROR_MODE_POOL, features, -EINVAL,
 468       RBD_MIRROR_IMAGE_ENABLED);
 469 }
 470
 471 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModeDisabled) {
 472   uint64_t features = 0;
 473   features |= RBD_FEATURE_OBJECT_MAP;
 474   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 475   features |= RBD_FEATURE_JOURNALING;
 476   check_mirror_image_disable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 477       RBD_MIRROR_IMAGE_DISABLED);
 478 }
 479
 480 TEST_F(TestMirroring, DisableImageMirrorWithPeer) {
 481   REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
 482
 483   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 484
 485   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 486   int order = 20;
 487   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 488                              &order));
 489
 490   librbd::Image image;
 491   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 492   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL));
 493
 494   setup_mirror_peer(m_ioctx, image);
 495
 496   ASSERT_EQ(0, image.mirror_image_disable(false));
 497
 498   std::vector<librbd::snap_info_t> snaps;
 499   ASSERT_EQ(0, image.snap_list(snaps));
 500   ASSERT_TRUE(snaps.empty());
 501
 502   librbd::mirror_image_info_t mirror_image;
 503   ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 504                                            sizeof(mirror_image)));
 505   ASSERT_EQ(RBD_MIRROR_IMAGE_DISABLED, mirror_image.state);
 506
 507   librbd::mirror_image_global_status_t status;
 508   ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 509   librbd::mirror_image_site_status_t local_status;
 510   ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 511   ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 512
 513   ASSERT_EQ(0, image.close());
 514   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 515   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 516 }
 517
 518 TEST_F(TestMirroring, DisableJournalingWithPeer) {
 519   REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
 520
 521   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 522
 523   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 524   int order = 20;
 525   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 526                              &order));
 527
 528   librbd::Image image;
 529   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 530
 531   setup_mirror_peer(m_ioctx, image);
 532
 533   ASSERT_EQ(0, image.update_features(RBD_FEATURE_JOURNALING, false));
 534
 535   std::vector<librbd::snap_info_t> snaps;
 536   ASSERT_EQ(0, image.snap_list(snaps));
 537   ASSERT_TRUE(snaps.empty());
 538
 539   librbd::mirror_image_info_t mirror_image;
 540   ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 541                                            sizeof(mirror_image)));
 542   ASSERT_EQ(RBD_MIRROR_IMAGE_DISABLED, mirror_image.state);
 543
 544   librbd::mirror_image_global_status_t status;
 545   ASSERT_EQ(0, image.mirror_image_get_global_status(&status, sizeof(status)));
 546   librbd::mirror_image_site_status_t local_status;
 547   ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
 548   ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
 549
 550   ASSERT_EQ(0, image.close());
 551   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 552   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 553 }
 554
 555 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling) {
 556   uint64_t features = 0;
 557   features |= RBD_FEATURE_OBJECT_MAP;
 558   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 559   check_mirror_image_enable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 560       RBD_MIRROR_IMAGE_DISABLED);
 561 }
 562
 563 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModePool_WithoutJournaling) {
 564   uint64_t features = 0;
 565   features |= RBD_FEATURE_OBJECT_MAP;
 566   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 567   check_mirror_image_enable(RBD_MIRROR_MODE_POOL, features, -EINVAL,
 568       RBD_MIRROR_IMAGE_DISABLED);
 569 }
 570
 571 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeImage_WithoutJournaling) {
 572   uint64_t features = 0;
 573   features |= RBD_FEATURE_OBJECT_MAP;
 574   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 575   check_mirror_image_enable(RBD_MIRROR_MODE_IMAGE, features, 0,
 576       RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_MODE_SNAPSHOT);
 577 }
 578
 579 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock) {
 580   uint64_t features = 0;
 581   check_mirror_image_enable(RBD_MIRROR_MODE_IMAGE, features, 0,
 582       RBD_MIRROR_IMAGE_ENABLED, RBD_MIRROR_IMAGE_MODE_SNAPSHOT);
 583 }
 584
 585 TEST_F(TestMirroring, CreateImage_In_MirrorModeDisabled) {
 586   uint64_t features = 0;
 587   features |= RBD_FEATURE_OBJECT_MAP;
 588   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 589   features |= RBD_FEATURE_JOURNALING;
 590   check_mirroring_on_create(features, RBD_MIRROR_MODE_DISABLED,
 591                             RBD_MIRROR_IMAGE_DISABLED);
 592 }
 593
 594 TEST_F(TestMirroring, CreateImage_In_MirrorModeImage) {
 595   uint64_t features = 0;
 596   features |= RBD_FEATURE_OBJECT_MAP;
 597   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 598   features |= RBD_FEATURE_JOURNALING;
 599   check_mirroring_on_create(features, RBD_MIRROR_MODE_IMAGE,
 600                             RBD_MIRROR_IMAGE_DISABLED);
 601 }
 602
 603 TEST_F(TestMirroring, CreateImage_In_MirrorModePool) {
 604   uint64_t features = 0;
 605   features |= RBD_FEATURE_OBJECT_MAP;
 606   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 607   features |= RBD_FEATURE_JOURNALING;
 608   check_mirroring_on_create(features, RBD_MIRROR_MODE_POOL,
 609                             RBD_MIRROR_IMAGE_ENABLED);
 610 }
 611
 612 TEST_F(TestMirroring, CreateImage_In_MirrorModePool_WithoutJournaling) {
 613   uint64_t features = 0;
 614   features |= RBD_FEATURE_OBJECT_MAP;
 615   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 616   check_mirroring_on_create(features, RBD_MIRROR_MODE_POOL,
 617                             RBD_MIRROR_IMAGE_DISABLED);
 618 }
 619
 620 TEST_F(TestMirroring, CreateImage_In_MirrorModeImage_WithoutJournaling) {
 621   uint64_t features = 0;
 622   features |= RBD_FEATURE_OBJECT_MAP;
 623   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 624   check_mirroring_on_create(features, RBD_MIRROR_MODE_IMAGE,
 625                             RBD_MIRROR_IMAGE_DISABLED);
 626 }
 627
 628 TEST_F(TestMirroring, EnableJournaling_In_MirrorModeDisabled) {
 629   uint64_t init_features = 0;
 630   init_features |= RBD_FEATURE_OBJECT_MAP;
 631   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 632   uint64_t features = RBD_FEATURE_JOURNALING;
 633   check_mirroring_on_update_features(init_features, true, false, features, 0,
 634                       RBD_MIRROR_MODE_DISABLED, RBD_MIRROR_IMAGE_DISABLED);
 635 }
 636
 637 TEST_F(TestMirroring, EnableJournaling_In_MirrorModeImage) {
 638   uint64_t init_features = 0;
 639   init_features |= RBD_FEATURE_OBJECT_MAP;
 640   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 641   uint64_t features = RBD_FEATURE_JOURNALING;
 642   check_mirroring_on_update_features(init_features, true, false, features, 0,
 643                       RBD_MIRROR_MODE_IMAGE, RBD_MIRROR_IMAGE_DISABLED);
 644 }
 645
 646 TEST_F(TestMirroring, EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled) {
 647   uint64_t init_features = 0;
 648   init_features |= RBD_FEATURE_OBJECT_MAP;
 649   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 650   uint64_t features = RBD_FEATURE_JOURNALING;
 651   check_mirroring_on_update_features(init_features, true, true, features,
 652                       0, RBD_MIRROR_MODE_IMAGE, RBD_MIRROR_IMAGE_ENABLED,
 653                       RBD_MIRROR_IMAGE_MODE_SNAPSHOT);
 654 }
 655
 656 TEST_F(TestMirroring, EnableJournaling_In_MirrorModePool) {
 657   uint64_t init_features = 0;
 658   init_features |= RBD_FEATURE_OBJECT_MAP;
 659   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 660   uint64_t features = RBD_FEATURE_JOURNALING;
 661   check_mirroring_on_update_features(init_features, true, false, features, 0,
 662                       RBD_MIRROR_MODE_POOL, RBD_MIRROR_IMAGE_ENABLED,
 663                       RBD_MIRROR_IMAGE_MODE_JOURNAL);
 664 }
 665
 666 TEST_F(TestMirroring, DisableJournaling_In_MirrorModePool) {
 667   uint64_t init_features = 0;
 668   init_features |= RBD_FEATURE_OBJECT_MAP;
 669   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 670   init_features |= RBD_FEATURE_JOURNALING;
 671   uint64_t features = RBD_FEATURE_JOURNALING;
 672   check_mirroring_on_update_features(init_features, false, false, features, 0,
 673                       RBD_MIRROR_MODE_POOL, RBD_MIRROR_IMAGE_DISABLED);
 674 }
 675
 676 TEST_F(TestMirroring, DisableJournaling_In_MirrorModeImage) {
 677   uint64_t init_features = 0;
 678   init_features |= RBD_FEATURE_OBJECT_MAP;
 679   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 680   init_features |= RBD_FEATURE_JOURNALING;
 681   uint64_t features = RBD_FEATURE_JOURNALING;
 682   check_mirroring_on_update_features(init_features, false, true, features,
 683                       -EINVAL, RBD_MIRROR_MODE_IMAGE, RBD_MIRROR_IMAGE_ENABLED,
 684                       RBD_MIRROR_IMAGE_MODE_JOURNAL);
 685 }
 686
 687 TEST_F(TestMirroring, MirrorModeSet_DisabledMode_To_PoolMode) {
 688   std::vector<uint64_t> features_vec;
 689   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 690   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 691
 692   setup_images_with_mirror_mode(RBD_MIRROR_MODE_DISABLED, features_vec);
 693
 694   std::vector<rbd_mirror_image_state_t> states_vec;
 695   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 696   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 697   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_POOL, states_vec);
 698 }
 699
 700 TEST_F(TestMirroring, MirrorModeSet_PoolMode_To_DisabledMode) {
 701   std::vector<uint64_t> features_vec;
 702   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 703   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 704
 705   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 706
 707   std::vector<rbd_mirror_image_state_t> states_vec;
 708   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 709   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 710   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_DISABLED, states_vec);
 711 }
 712
 713 TEST_F(TestMirroring, MirrorModeSet_DisabledMode_To_ImageMode) {
 714   std::vector<uint64_t> features_vec;
 715   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 716   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 717
 718   setup_images_with_mirror_mode(RBD_MIRROR_MODE_DISABLED, features_vec);
 719
 720   std::vector<rbd_mirror_image_state_t> states_vec;
 721   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 722   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 723   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_IMAGE, states_vec);
 724 }
 725
 726
 727 TEST_F(TestMirroring, MirrorModeSet_PoolMode_To_ImageMode) {
 728   std::vector<uint64_t> features_vec;
 729   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 730   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 731
 732   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 733
 734   std::vector<rbd_mirror_image_state_t> states_vec;
 735   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 736   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 737   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_IMAGE, states_vec);
 738 }
 739
 740 TEST_F(TestMirroring, MirrorModeSet_ImageMode_To_PoolMode) {
 741   std::vector<uint64_t> features_vec;
 742   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 743   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 744
 745   setup_images_with_mirror_mode(RBD_MIRROR_MODE_IMAGE, features_vec);
 746
 747   std::vector<rbd_mirror_image_state_t> states_vec;
 748   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 749   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 750   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_POOL, states_vec);
 751 }
 752
 753 TEST_F(TestMirroring, MirrorModeSet_ImageMode_To_DisabledMode) {
 754   std::vector<uint64_t> features_vec;
 755   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 756   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 757
 758   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 759
 760   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 761   ASSERT_EQ(-EINVAL, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 762   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 763
 764   std::vector<rbd_mirror_image_state_t> states_vec;
 765   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 766   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 767   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_DISABLED, states_vec);
 768 }
 769
 770 TEST_F(TestMirroring, RemoveImage_With_MirrorImageEnabled) {
 771   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 772                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 773                      true);
 774 }
 775
 776 TEST_F(TestMirroring, RemoveImage_With_MirrorImageDisabled) {
 777   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 778                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 779                      false);
 780 }
 781
 782 TEST_F(TestMirroring, RemoveImage_With_ImageWithoutJournal) {
 783   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 784                      RBD_FEATURE_EXCLUSIVE_LOCK,
 785                      false);
 786 }
 787
 788 TEST_F(TestMirroring, RemoveImage_With_MirrorImageDemoted) {
 789   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 790                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 791                      true, true);
 792 }
 793
 794 TEST_F(TestMirroring, TrashMoveRestore_PoolMode) {
 795   check_trash_move_restore(RBD_MIRROR_MODE_POOL, false);
 796 }
 797
 798 TEST_F(TestMirroring, TrashMoveRestore_ImageMode_MirroringDisabled) {
 799   check_trash_move_restore(RBD_MIRROR_MODE_IMAGE, false);
 800 }
 801
 802 TEST_F(TestMirroring, TrashMoveRestore_ImageMode_MirroringEnabled) {
 803   check_trash_move_restore(RBD_MIRROR_MODE_IMAGE, true);
 804 }
 805
 806 TEST_F(TestMirroring, MirrorStatusList) {
 807   std::vector<uint64_t>
 808       features_vec(5, RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 809   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 810
 811   std::string last_read = "";
 812   std::map<std::string, librbd::mirror_image_global_status_t> images;
 813   ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, last_read, 2,
 814                                                      &images));
 815   ASSERT_EQ(2U, images.size());
 816
 817   last_read = images.rbegin()->first;
 818   images.clear();
 819   ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, last_read, 2,
 820                                                      &images));
 821   ASSERT_EQ(2U, images.size());
 822
 823   last_read = images.rbegin()->first;
 824   images.clear();
 825   ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, last_read, 4096,
 826                                                      &images));
 827   ASSERT_EQ(1U, images.size());
 828
 829   last_read = images.rbegin()->first;
 830   images.clear();
 831   ASSERT_EQ(0, m_rbd.mirror_image_global_status_list(m_ioctx, last_read, 4096,
 832                                                      &images));
 833   ASSERT_EQ(0U, images.size());
 834 }
 835
 836 TEST_F(TestMirroring, RemoveBootstrapped)
 837 {
 838   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 839
 840   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 841   int order = 20;
 842   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 843                              &order));
 844   librbd::Image image;
 845   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 846
 847   librbd::NoOpProgressContext no_op;
 848   ASSERT_EQ(-EBUSY, librbd::api::Image<>::remove(m_ioctx, image_name, no_op));
 849
 850   // simulate the image is open by rbd-mirror bootstrap
 851   uint64_t handle;
 852   struct MirrorWatcher : public librados::WatchCtx2 {
 853     explicit MirrorWatcher(librados::IoCtx &ioctx) : m_ioctx(ioctx) {
 854     }
 855     void handle_notify(uint64_t notify_id, uint64_t cookie,
 856                                uint64_t notifier_id, bufferlist& bl) override {
 857       // received IMAGE_UPDATED notification from remove
 858       m_notified = true;
 859       m_ioctx.notify_ack(RBD_MIRRORING, notify_id, cookie, bl);
 860     }
 861     void handle_error(uint64_t cookie, int err) override {
 862     }
 863     librados::IoCtx &m_ioctx;
 864     bool m_notified = false;
 865   } watcher(m_ioctx);
 866   ASSERT_EQ(0, m_ioctx.create(RBD_MIRRORING, false));
 867   ASSERT_EQ(0, m_ioctx.watch2(RBD_MIRRORING, &handle, &watcher));
 868   // now remove should succeed
 869   ASSERT_EQ(0, librbd::api::Image<>::remove(m_ioctx, image_name, no_op));
 870   ASSERT_EQ(0, m_ioctx.unwatch2(handle));
 871   ASSERT_TRUE(watcher.m_notified);
 872   ASSERT_EQ(0, image.close());
 873 }
 874
 875 TEST_F(TestMirroring, AioPromoteDemote) {
 876   std::list<std::string> image_names;
 877   for (size_t idx = 0; idx < 10; ++idx) {
 878     image_names.push_back(get_temp_image_name());
 879   }
 880
 881   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 882
 883   // create mirror images
 884   int order = 20;
 885   std::list<librbd::Image> images;
 886   for (auto &image_name : image_names) {
 887     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
 888                                RBD_FEATURE_EXCLUSIVE_LOCK |
 889                                  RBD_FEATURE_JOURNALING,
 890                                &order));
 891
 892     images.emplace_back();
 893     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
 894     ASSERT_EQ(0, images.back().mirror_image_enable2(
 895                 RBD_MIRROR_IMAGE_MODE_JOURNAL));
 896   }
 897
 898   // demote all images
 899   std::list<librbd::RBD::AioCompletion *> aio_comps;
 900   for (auto &image : images) {
 901     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 902     ASSERT_EQ(0, image.aio_mirror_image_demote(aio_comps.back()));
 903   }
 904   for (auto aio_comp : aio_comps) {
 905     ASSERT_EQ(0, aio_comp->wait_for_complete());
 906     ASSERT_EQ(1, aio_comp->is_complete());
 907     ASSERT_EQ(0, aio_comp->get_return_value());
 908     aio_comp->release();
 909   }
 910   aio_comps.clear();
 911
 912   // verify demotions
 913   for (auto &image : images) {
 914     librbd::mirror_image_info_t mirror_image;
 915     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 916                                              sizeof(mirror_image)));
 917     ASSERT_FALSE(mirror_image.primary);
 918   }
 919
 920   // promote all images
 921   for (auto &image : images) {
 922     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 923     ASSERT_EQ(0, image.aio_mirror_image_promote(false, aio_comps.back()));
 924   }
 925   for (auto aio_comp : aio_comps) {
 926     ASSERT_EQ(0, aio_comp->wait_for_complete());
 927     ASSERT_EQ(1, aio_comp->is_complete());
 928     ASSERT_EQ(0, aio_comp->get_return_value());
 929     aio_comp->release();
 930   }
 931
 932   // verify promotions
 933   for (auto &image : images) {
 934     librbd::mirror_image_info_t mirror_image;
 935     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 936                                              sizeof(mirror_image)));
 937     ASSERT_TRUE(mirror_image.primary);
 938   }
 939 }
 940
 941 TEST_F(TestMirroring, AioGetInfo) {
 942   std::list<std::string> image_names;
 943   for (size_t idx = 0; idx < 10; ++idx) {
 944     image_names.push_back(get_temp_image_name());
 945   }
 946
 947   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 948
 949   // create mirror images
 950   int order = 20;
 951   std::list<librbd::Image> images;
 952   for (auto &image_name : image_names) {
 953     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
 954                                RBD_FEATURE_EXCLUSIVE_LOCK |
 955                                  RBD_FEATURE_JOURNALING,
 956                                &order));
 957
 958     images.emplace_back();
 959     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
 960   }
 961
 962   std::list<librbd::RBD::AioCompletion *> aio_comps;
 963   std::list<librbd::mirror_image_info_t> infos;
 964   for (auto &image : images) {
 965     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 966     infos.emplace_back();
 967     ASSERT_EQ(0, image.aio_mirror_image_get_info(&infos.back(),
 968                                                  sizeof(infos.back()),
 969                                                  aio_comps.back()));
 970   }
 971   for (auto aio_comp : aio_comps) {
 972     ASSERT_EQ(0, aio_comp->wait_for_complete());
 973     ASSERT_EQ(1, aio_comp->is_complete());
 974     ASSERT_EQ(0, aio_comp->get_return_value());
 975     aio_comp->release();
 976   }
 977   aio_comps.clear();
 978
 979   for (auto &info : infos) {
 980     ASSERT_NE("", info.global_id);
 981     ASSERT_EQ(RBD_MIRROR_IMAGE_ENABLED, info.state);
 982     ASSERT_TRUE(info.primary);
 983   }
 984 }
 985
 986 TEST_F(TestMirroring, AioGetStatus) {
 987   std::list<std::string> image_names;
 988   for (size_t idx = 0; idx < 10; ++idx) {
 989     image_names.push_back(get_temp_image_name());
 990   }
 991
 992   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 993
 994   // create mirror images
 995   int order = 20;
 996   std::list<librbd::Image> images;
 997   for (auto &image_name : image_names) {
 998     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
 999                                RBD_FEATURE_EXCLUSIVE_LOCK |
1000                                  RBD_FEATURE_JOURNALING,
1001                                &order));
1002
1003     images.emplace_back();
1004     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
1005   }
1006
1007   std::list<librbd::RBD::AioCompletion *> aio_comps;
1008   std::list<librbd::mirror_image_global_status_t> statuses;
1009   for (auto &image : images) {
1010     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
1011     statuses.emplace_back();
1012     ASSERT_EQ(0, image.aio_mirror_image_get_global_status(
1013                    &statuses.back(), sizeof(statuses.back()),
1014                    aio_comps.back()));
1015   }
1016   for (auto aio_comp : aio_comps) {
1017     ASSERT_EQ(0, aio_comp->wait_for_complete());
1018     ASSERT_EQ(1, aio_comp->is_complete());
1019     ASSERT_EQ(0, aio_comp->get_return_value());
1020     aio_comp->release();
1021   }
1022   aio_comps.clear();
1023
1024   for (auto &status : statuses) {
1025     ASSERT_NE("", status.name);
1026     ASSERT_NE("", status.info.global_id);
1027     ASSERT_EQ(RBD_MIRROR_IMAGE_ENABLED, status.info.state);
1028     ASSERT_TRUE(status.info.primary);
1029
1030     librbd::mirror_image_site_status_t local_status;
1031     ASSERT_EQ(0, get_local_mirror_image_site_status(status, &local_status));
1032     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, local_status.state);
1033     ASSERT_EQ("status not found", local_status.description);
1034     ASSERT_FALSE(local_status.up);
1035     ASSERT_EQ(0, local_status.last_update);
1036   }
1037 }
1038
1039 TEST_F(TestMirroring, SiteName) {
1040   REQUIRE(!is_librados_test_stub(_rados));
1041
1042   const std::string expected_site_name("us-east-1a");
1043   ASSERT_EQ(0, m_rbd.mirror_site_name_set(_rados, expected_site_name));
1044
1045   std::string site_name;
1046   ASSERT_EQ(0, m_rbd.mirror_site_name_get(_rados, &site_name));
1047   ASSERT_EQ(expected_site_name, site_name);
1048
1049   ASSERT_EQ(0, m_rbd.mirror_site_name_set(_rados, ""));
1050
1051   std::string fsid;
1052   ASSERT_EQ(0, _rados.cluster_fsid(&fsid));
1053   ASSERT_EQ(0, m_rbd.mirror_site_name_get(_rados, &site_name));
1054   ASSERT_EQ(fsid, site_name);
1055 }
1056
1057 TEST_F(TestMirroring, Bootstrap) {
1058   REQUIRE(!is_librados_test_stub(_rados));
1059
1060   std::string token_b64;
1061   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1062   ASSERT_EQ(-EINVAL, m_rbd.mirror_peer_bootstrap_create(m_ioctx, &token_b64));
1063
1064   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
1065   ASSERT_EQ(0, m_rbd.mirror_peer_bootstrap_create(m_ioctx, &token_b64));
1066
1067   bufferlist token_b64_bl;
1068   token_b64_bl.append(token_b64);
1069
1070   bufferlist token_bl;
1071   token_bl.decode_base64(token_b64_bl);
1072
1073   // cannot import token into same cluster
1074   ASSERT_EQ(-EINVAL,
1075             m_rbd.mirror_peer_bootstrap_import(
1076               m_ioctx, RBD_MIRROR_PEER_DIRECTION_RX, token_b64));
1077 }
1078
1079 TEST_F(TestMirroring, PeerDirection) {
1080   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
1081
1082   std::string uuid;
1083   ASSERT_EQ(-EINVAL, m_rbd.mirror_peer_site_add(
1084                        m_ioctx, &uuid, RBD_MIRROR_PEER_DIRECTION_TX, "siteA",
1085                        "client.admin"));
1086   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &uuid,
1087                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1088                                           "siteA", "client.admin"));
1089
1090   std::vector<librbd::mirror_peer_site_t> peers;
1091   ASSERT_EQ(0, m_rbd.mirror_peer_site_list(m_ioctx, &peers));
1092   std::vector<librbd::mirror_peer_site_t> expected_peers = {
1093     {uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, "siteA", "", "client.admin", 0}};
1094   ASSERT_EQ(expected_peers, peers);
1095
1096   ASSERT_EQ(0, m_rbd.mirror_peer_site_set_direction(
1097                  m_ioctx, uuid, RBD_MIRROR_PEER_DIRECTION_RX));
1098   ASSERT_EQ(0, m_rbd.mirror_peer_site_list(m_ioctx, &peers));
1099   expected_peers = {
1100     {uuid, RBD_MIRROR_PEER_DIRECTION_RX, "siteA", "", "client.admin", 0}};
1101   ASSERT_EQ(expected_peers, peers);
1102
1103   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, uuid));
1104 }
1105
1106 TEST_F(TestMirroring, Snapshot)
1107 {
1108   REQUIRE_FORMAT_V2();
1109
1110   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1111
1112   uint64_t features;
1113   ASSERT_TRUE(get_features(&features));
1114   int order = 20;
1115   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1116                              &order));
1117
1118   librbd::Image image;
1119   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1120
1121   ASSERT_EQ(0, image.metadata_set(
1122               "conf_rbd_mirroring_max_mirroring_snapshots", "3"));
1123
1124   uint64_t snap_id;
1125
1126   ASSERT_EQ(-EINVAL, image.mirror_image_create_snapshot(&snap_id));
1127   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1128   ASSERT_EQ(-EINVAL, image.mirror_image_create_snapshot(&snap_id));
1129   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1130   librbd::mirror_image_mode_t mode;
1131   ASSERT_EQ(0, image.mirror_image_get_mode(&mode));
1132   ASSERT_EQ(RBD_MIRROR_IMAGE_MODE_SNAPSHOT, mode);
1133   ASSERT_EQ(-EINVAL, image.mirror_image_create_snapshot(&snap_id));
1134   std::string peer_uuid;
1135   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid,
1136                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1137                                           "cluster", "client"));
1138   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1139   vector<librbd::snap_info_t> snaps;
1140   ASSERT_EQ(0, image.snap_list(snaps));
1141   ASSERT_EQ(2U, snaps.size());
1142   ASSERT_EQ(snaps[1].id, snap_id);
1143
1144   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1145   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1146   snaps.clear();
1147   ASSERT_EQ(0, image.snap_list(snaps));
1148   ASSERT_EQ(3U, snaps.size());
1149   ASSERT_EQ(snaps[2].id, snap_id);
1150
1151   // automatic peer unlink on max_mirroring_snapshots reached
1152   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1153   vector<librbd::snap_info_t> snaps1;
1154   ASSERT_EQ(0, image.snap_list(snaps1));
1155   ASSERT_EQ(3U, snaps1.size());
1156   ASSERT_EQ(snaps1[0].id, snaps[0].id);
1157   ASSERT_EQ(snaps1[1].id, snaps[2].id);
1158   ASSERT_EQ(snaps1[2].id, snap_id);
1159
1160   librbd::snap_namespace_type_t snap_ns_type;
1161   ASSERT_EQ(0, image.snap_get_namespace_type(snap_id, &snap_ns_type));
1162   ASSERT_EQ(RBD_SNAP_NAMESPACE_TYPE_MIRROR, snap_ns_type);
1163   librbd::snap_mirror_namespace_t mirror_snap;
1164   ASSERT_EQ(0, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1165                                                sizeof(mirror_snap)));
1166   ASSERT_EQ(1U, mirror_snap.mirror_peer_uuids.size());
1167   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer_uuid));
1168
1169   for (auto &snap : snaps1) {
1170     ASSERT_EQ(0, image.snap_remove_by_id(snap.id));
1171   }
1172
1173   ASSERT_EQ(0, image.close());
1174   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1175   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid));
1176   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1177 }
1178
1179 TEST_F(TestMirroring, SnapshotRemoveOnDisable)
1180 {
1181   REQUIRE_FORMAT_V2();
1182
1183   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1184   std::string peer_uuid;
1185   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid,
1186                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1187                                           "cluster", "client"));
1188
1189   uint64_t features;
1190   ASSERT_TRUE(get_features(&features));
1191   int order = 20;
1192   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1193                              &order));
1194   librbd::Image image;
1195   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1196   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1197   uint64_t snap_id;
1198   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1199
1200   vector<librbd::snap_info_t> snaps;
1201   ASSERT_EQ(0, image.snap_list(snaps));
1202   ASSERT_EQ(2U, snaps.size());
1203   ASSERT_EQ(snaps[1].id, snap_id);
1204
1205   ASSERT_EQ(0, image.mirror_image_disable(false));
1206
1207   snaps.clear();
1208   ASSERT_EQ(0, image.snap_list(snaps));
1209   ASSERT_TRUE(snaps.empty());
1210
1211   ASSERT_EQ(0, image.close());
1212   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1213   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid));
1214   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1215 }
1216
1217 TEST_F(TestMirroring, SnapshotUnlinkPeer)
1218 {
1219   REQUIRE_FORMAT_V2();
1220
1221   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1222   std::string peer1_uuid;
1223   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer1_uuid,
1224                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1225                                           "cluster1", "client"));
1226   std::string peer2_uuid;
1227   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer2_uuid,
1228                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1229                                           "cluster2", "client"));
1230   std::string peer3_uuid;
1231   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer3_uuid,
1232                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1233                                           "cluster3", "client"));
1234   uint64_t features;
1235   ASSERT_TRUE(get_features(&features));
1236   features &= ~RBD_FEATURE_JOURNALING;
1237   int order = 20;
1238   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1239                              &order));
1240   librbd::Image image;
1241   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1242   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1243   uint64_t snap_id;
1244   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id));
1245   uint64_t snap_id2;
1246   ASSERT_EQ(0, image.mirror_image_create_snapshot(&snap_id2));
1247   librbd::snap_mirror_namespace_t mirror_snap;
1248   ASSERT_EQ(0, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1249                                                sizeof(mirror_snap)));
1250   ASSERT_EQ(3U, mirror_snap.mirror_peer_uuids.size());
1251   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer1_uuid));
1252   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1253   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer3_uuid));
1254
1255   auto ictx = new librbd::ImageCtx(image_name, "", nullptr, m_ioctx, false);
1256   ASSERT_EQ(0, ictx->state->open(0));
1257   BOOST_SCOPE_EXIT(&ictx) {
1258     if (ictx != nullptr) {
1259       ictx->state->close();
1260     }
1261   } BOOST_SCOPE_EXIT_END;
1262
1263   C_SaferCond cond1;
1264   auto req = librbd::mirror::snapshot::UnlinkPeerRequest<>::create(
1265     ictx, snap_id, peer1_uuid, &cond1);
1266   req->send();
1267   ASSERT_EQ(0, cond1.wait());
1268
1269   ASSERT_EQ(0, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1270                                                sizeof(mirror_snap)));
1271   ASSERT_EQ(2U, mirror_snap.mirror_peer_uuids.size());
1272   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1273   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer3_uuid));
1274
1275   ASSERT_EQ(0, librbd::api::Namespace<>::create(m_ioctx, "ns1"));
1276   librados::IoCtx ns_ioctx;
1277   ns_ioctx.dup(m_ioctx);
1278   ns_ioctx.set_namespace("ns1");
1279   ASSERT_EQ(0, m_rbd.mirror_mode_set(ns_ioctx, RBD_MIRROR_MODE_IMAGE));
1280   ASSERT_EQ(0, m_rbd.create2(ns_ioctx, image_name.c_str(), 4096, features,
1281                              &order));
1282
1283   librbd::Image ns_image;
1284   ASSERT_EQ(0, m_rbd.open(ns_ioctx, ns_image, image_name.c_str()));
1285   ASSERT_EQ(0, ns_image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1286   uint64_t ns_snap_id;
1287   ASSERT_EQ(0, ns_image.mirror_image_create_snapshot(&ns_snap_id));
1288   ASSERT_EQ(0, ns_image.snap_get_mirror_namespace(ns_snap_id, &mirror_snap,
1289                                                   sizeof(mirror_snap)));
1290   ASSERT_EQ(3U, mirror_snap.mirror_peer_uuids.size());
1291   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer1_uuid));
1292   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1293   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer3_uuid));
1294
1295   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer3_uuid));
1296
1297   ASSERT_EQ(0, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1298                                                sizeof(mirror_snap)));
1299   ASSERT_EQ(1U, mirror_snap.mirror_peer_uuids.size());
1300   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1301
1302   ASSERT_EQ(0, ns_image.snap_get_mirror_namespace(ns_snap_id, &mirror_snap,
1303                                                   sizeof(mirror_snap)));
1304   ASSERT_EQ(2U, mirror_snap.mirror_peer_uuids.size());
1305   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer1_uuid));
1306   ASSERT_EQ(1, mirror_snap.mirror_peer_uuids.count(peer2_uuid));
1307
1308   C_SaferCond cond2;
1309   req = librbd::mirror::snapshot::UnlinkPeerRequest<>::create(
1310     ictx, snap_id, peer2_uuid, &cond2);
1311   req->send();
1312   ASSERT_EQ(0, cond2.wait());
1313
1314   ASSERT_EQ(-ENOENT, image.snap_get_mirror_namespace(snap_id, &mirror_snap,
1315                                                      sizeof(mirror_snap)));
1316   ictx->state->close();
1317   ictx = nullptr;
1318   ASSERT_EQ(0, image.close());
1319   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1320
1321   ASSERT_EQ(0, ns_image.close());
1322   ASSERT_EQ(0, m_rbd.remove(ns_ioctx, image_name.c_str()));
1323   ASSERT_EQ(0, librbd::api::Namespace<>::remove(m_ioctx, "ns1"));
1324
1325   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer1_uuid));
1326   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer2_uuid));
1327   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1328 }
1329
1330 TEST_F(TestMirroring, SnapshotImageState)
1331 {
1332   REQUIRE_FORMAT_V2();
1333
1334   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1335
1336   uint64_t features;
1337   ASSERT_TRUE(get_features(&features));
1338   int order = 20;
1339   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1340                              &order));
1341
1342   librbd::Image image;
1343   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1344   ASSERT_EQ(0, image.snap_create("snap"));
1345   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1346   std::vector<librbd::snap_info_t> snaps;
1347   ASSERT_EQ(0, image.snap_list(snaps));
1348   ASSERT_EQ(2U, snaps.size());
1349   auto snap_id = snaps[1].id;
1350
1351   auto ictx = new librbd::ImageCtx(image_name, "", nullptr, m_ioctx, false);
1352   ASSERT_EQ(0, ictx->state->open(0));
1353   BOOST_SCOPE_EXIT(&ictx) {
1354     if (ictx != nullptr) {
1355       ictx->state->close();
1356     }
1357   } BOOST_SCOPE_EXIT_END;
1358
1359   {
1360     C_SaferCond cond;
1361     auto req = librbd::mirror::snapshot::SetImageStateRequest<>::create(
1362       ictx, snap_id, &cond);
1363     req->send();
1364     ASSERT_EQ(0, cond.wait());
1365   }
1366
1367   librbd::mirror::snapshot::ImageState image_state;
1368   {
1369     C_SaferCond cond;
1370     auto req = librbd::mirror::snapshot::GetImageStateRequest<>::create(
1371       ictx, snap_id, &image_state, &cond);
1372     req->send();
1373     ASSERT_EQ(0, cond.wait());
1374   }
1375
1376   ASSERT_EQ(image_name, image_state.name);
1377   ASSERT_EQ(0, image.features(&features));
1378   ASSERT_EQ(features & ~RBD_FEATURES_IMPLICIT_ENABLE, image_state.features);
1379   ASSERT_EQ(1U, image_state.snapshots.size());
1380   ASSERT_EQ("snap", image_state.snapshots.begin()->second.name);
1381   ASSERT_TRUE(image_state.metadata.empty());
1382
1383   {
1384     C_SaferCond cond;
1385     auto req = librbd::mirror::snapshot::RemoveImageStateRequest<>::create(
1386       ictx, snap_id, &cond);
1387     req->send();
1388     ASSERT_EQ(0, cond.wait());
1389   }
1390
1391   // test storing "large" image state in multiple objects
1392
1393   ASSERT_EQ(0, ictx->config.set_val("rbd_default_order", "8"));
1394
1395   for (int i = 0; i < 10; i++) {
1396     ASSERT_EQ(0, image.metadata_set(stringify(i), std::string(1024, 'A' + i)));
1397   }
1398
1399   {
1400     C_SaferCond cond;
1401     auto req = librbd::mirror::snapshot::SetImageStateRequest<>::create(
1402       ictx, snap_id, &cond);
1403     req->send();
1404     ASSERT_EQ(0, cond.wait());
1405   }
1406
1407   {
1408     C_SaferCond cond;
1409     auto req = librbd::mirror::snapshot::GetImageStateRequest<>::create(
1410       ictx, snap_id, &image_state, &cond);
1411     req->send();
1412     ASSERT_EQ(0, cond.wait());
1413   }
1414
1415   ASSERT_EQ(image_name, image_state.name);
1416   ASSERT_EQ(features & ~RBD_FEATURES_IMPLICIT_ENABLE, image_state.features);
1417   ASSERT_EQ(10U, image_state.metadata.size());
1418   for (int i = 0; i < 10; i++) {
1419     auto &bl = image_state.metadata[stringify(i)];
1420     ASSERT_EQ(0, strncmp(std::string(1024, 'A' + i).c_str(), bl.c_str(),
1421                          bl.length()));
1422   }
1423
1424   {
1425     C_SaferCond cond;
1426     auto req = librbd::mirror::snapshot::RemoveImageStateRequest<>::create(
1427       ictx, snap_id, &cond);
1428     req->send();
1429     ASSERT_EQ(0, cond.wait());
1430   }
1431
1432   ASSERT_EQ(0, ictx->state->close());
1433   ictx = nullptr;
1434
1435   ASSERT_EQ(0, image.snap_remove("snap"));
1436   ASSERT_EQ(0, image.close());
1437   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1438 }
1439
1440 TEST_F(TestMirroring, SnapshotPromoteDemote)
1441 {
1442   REQUIRE_FORMAT_V2();
1443
1444   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
1445   std::string peer_uuid;
1446   ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid,
1447                                           RBD_MIRROR_PEER_DIRECTION_RX_TX,
1448                                           "cluster", "client"));
1449
1450   uint64_t features;
1451   ASSERT_TRUE(get_features(&features));
1452   int order = 20;
1453   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
1454                              &order));
1455
1456   librbd::Image image;
1457   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
1458   ASSERT_EQ(0, image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_SNAPSHOT));
1459   librbd::mirror_image_mode_t mode;
1460   ASSERT_EQ(0, image.mirror_image_get_mode(&mode));
1461   ASSERT_EQ(RBD_MIRROR_IMAGE_MODE_SNAPSHOT, mode);
1462
1463   ASSERT_EQ(-EINVAL, image.mirror_image_promote(false));
1464   ASSERT_EQ(0, image.mirror_image_demote());
1465   ASSERT_EQ(0, image.mirror_image_promote(false));
1466   ASSERT_EQ(0, image.mirror_image_demote());
1467   ASSERT_EQ(0, image.mirror_image_promote(false));
1468
1469   ASSERT_EQ(0, image.close());
1470   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
1471   ASSERT_EQ(0, m_rbd.mirror_peer_site_remove(m_ioctx, peer_uuid));
1472   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
1473 }