ceph/src/test/librbd/test_ImageWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 #include "test/librbd/test_fixture.h"
   4 #include "test/librbd/test_support.h"
   5 #include "include/int_types.h"
   6 #include "include/stringify.h"
   7 #include "include/rados/librados.h"
   8 #include "include/rbd/librbd.hpp"
   9 #include "common/Cond.h"
  10 #include "common/ceph_mutex.h"
  11 #include "common/errno.h"
  12 #include "cls/lock/cls_lock_client.h"
  13 #include "cls/lock/cls_lock_types.h"
  14 #include "librbd/internal.h"
  15 #include "librbd/ImageCtx.h"
  16 #include "librbd/ImageWatcher.h"
  17 #include "librbd/WatchNotifyTypes.h"
  18 #include "librbd/io/AioCompletion.h"
  19 #include "test/librados/test.h"
  20 #include "gtest/gtest.h"
  21 #include <boost/assign/std/set.hpp>
  22 #include <boost/assign/std/map.hpp>
  23 #include <boost/scope_exit.hpp>
  24 #include <boost/thread/thread.hpp>
  25 #include <iostream>
  26 #include <map>
  27 #include <set>
  28 #include <sstream>
  29 #include <vector>
  30
  31 using namespace ceph;
  32 using namespace boost::assign;
  33 using namespace librbd::watch_notify;
  34
  35 void register_test_image_watcher() {
  36 }
  37
  38 class TestImageWatcher : public TestFixture {
  39 public:
  40
  41   TestImageWatcher() : m_watch_ctx(NULL)
  42   {
  43   }
  44
  45   class WatchCtx : public librados::WatchCtx2 {
  46   public:
  47     explicit WatchCtx(TestImageWatcher &parent) : m_parent(parent), m_handle(0) {}
  48
  49     int watch(const librbd::ImageCtx &ictx) {
  50       m_header_oid = ictx.header_oid;
  51       return m_parent.m_ioctx.watch2(m_header_oid, &m_handle, this);
  52     }
  53
  54     int unwatch() {
  55       return m_parent.m_ioctx.unwatch2(m_handle);
  56     }
  57
  58     void handle_notify(uint64_t notify_id,
  59                                uint64_t cookie,
  60                                uint64_t notifier_id,
  61                                bufferlist& bl) override {
  62       try {
  63         int op;
  64         bufferlist payload;
  65         auto iter = bl.cbegin();
  66         DECODE_START(1, iter);
  67         decode(op, iter);
  68         iter.copy_all(payload);
  69         DECODE_FINISH(iter);
  70
  71         NotifyOp notify_op = static_cast<NotifyOp>(op);
  72         /*
  73         std::cout << "NOTIFY: " << notify_op << ", " << notify_id
  74                   << ", " << cookie << ", " << notifier_id << std::endl;
  75         */
  76
  77         std::lock_guard l{m_parent.m_callback_lock};
  78         m_parent.m_notify_payloads[notify_op] = payload;
  79
  80         bufferlist reply;
  81         if (m_parent.m_notify_acks.count(notify_op) > 0) {
  82           reply = m_parent.m_notify_acks[notify_op];
  83           m_parent.m_notifies += notify_op;
  84           m_parent.m_callback_cond.notify_all();
  85         }
  86
  87         m_parent.m_ioctx.notify_ack(m_header_oid, notify_id, cookie, reply);
  88       } catch (...) {
  89         FAIL();
  90       }
  91     }
  92
  93     void handle_error(uint64_t cookie, int err) override {
  94       std::cerr << "ERROR: " << cookie << ", " << cpp_strerror(err)
  95                 << std::endl;
  96     }
  97
  98     uint64_t get_handle() const {
  99       return m_handle;
 100     }
 101
 102   private:
 103     TestImageWatcher &m_parent;
 104     std::string m_header_oid;
 105     uint64_t m_handle;
 106   };
 107
 108   void TearDown() override {
 109     deregister_image_watch();
 110     TestFixture::TearDown();
 111   }
 112
 113   int deregister_image_watch() {
 114     if (m_watch_ctx != NULL) {
 115       int r = m_watch_ctx->unwatch();
 116
 117       librados::Rados rados(m_ioctx);
 118       rados.watch_flush();
 119
 120       delete m_watch_ctx;
 121       m_watch_ctx = NULL;
 122       return r;
 123     }
 124     return 0;
 125   }
 126
 127   int register_image_watch(librbd::ImageCtx &ictx) {
 128     m_watch_ctx = new WatchCtx(*this);
 129     return m_watch_ctx->watch(ictx);
 130   }
 131
 132   bool wait_for_notifies(librbd::ImageCtx &ictx) {
 133     std::unique_lock l{m_callback_lock};
 134     while (m_notifies.size() < m_notify_acks.size()) {
 135       if (m_callback_cond.wait_for(l, 10s) == std::cv_status::timeout) {
 136         break;
 137       }
 138     }
 139     return (m_notifies.size() == m_notify_acks.size());
 140   }
 141
 142   bufferlist create_response_message(int r) {
 143     bufferlist bl;
 144     encode(ResponseMessage(r), bl);
 145     return bl;
 146   }
 147
 148   bool extract_async_request_id(NotifyOp op, AsyncRequestId *id) {
 149     if (m_notify_payloads.count(op) == 0) {
 150       return false;
 151     }
 152
 153     bufferlist payload = m_notify_payloads[op];
 154     auto iter = payload.cbegin();
 155
 156     switch (op) {
 157     case NOTIFY_OP_FLATTEN:
 158       {
 159         FlattenPayload payload;
 160         payload.decode(2, iter);
 161         *id = payload.async_request_id;
 162       }
 163       return true;
 164     case NOTIFY_OP_RESIZE:
 165       {
 166         ResizePayload payload;
 167         payload.decode(2, iter);
 168         *id = payload.async_request_id;
 169       }
 170       return true;
 171     case NOTIFY_OP_SNAP_CREATE:
 172       {
 173         SnapCreatePayload payload;
 174         payload.decode(7, iter);
 175         *id = payload.async_request_id;
 176       }
 177       return true;
 178     case NOTIFY_OP_SNAP_RENAME:
 179       {
 180         SnapRenamePayload payload;
 181         payload.decode(7, iter);
 182         *id = payload.async_request_id;
 183       }
 184       return true;
 185     case NOTIFY_OP_SNAP_REMOVE:
 186       {
 187         SnapRemovePayload payload;
 188         payload.decode(7, iter);
 189         *id = payload.async_request_id;
 190       }
 191       return true;
 192     case NOTIFY_OP_SNAP_PROTECT:
 193       {
 194         SnapProtectPayload payload;
 195         payload.decode(7, iter);
 196         *id = payload.async_request_id;
 197       }
 198       return true;
 199     case NOTIFY_OP_SNAP_UNPROTECT:
 200       {
 201         SnapUnprotectPayload payload;
 202         payload.decode(7, iter);
 203         *id = payload.async_request_id;
 204       }
 205       return true;
 206     case NOTIFY_OP_RENAME:
 207       {
 208         RenamePayload payload;
 209         payload.decode(7, iter);
 210         *id = payload.async_request_id;
 211       }
 212       return true;
 213     case NOTIFY_OP_REBUILD_OBJECT_MAP:
 214       {
 215         RebuildObjectMapPayload payload;
 216         payload.decode(2, iter);
 217         *id = payload.async_request_id;
 218       }
 219       return true;
 220     default:
 221       break;
 222     }
 223     return false;
 224   }
 225
 226   int notify_async_progress(librbd::ImageCtx *ictx, const AsyncRequestId &id,
 227                             uint64_t offset, uint64_t total) {
 228     bufferlist bl;
 229     encode(NotifyMessage(new AsyncProgressPayload(id, offset, total)), bl);
 230     return m_ioctx.notify2(ictx->header_oid, bl, 5000, NULL);
 231   }
 232
 233   int notify_async_complete(librbd::ImageCtx *ictx, const AsyncRequestId &id,
 234                             int r) {
 235     bufferlist bl;
 236     encode(NotifyMessage(new AsyncCompletePayload(id, r)), bl);
 237     return m_ioctx.notify2(ictx->header_oid, bl, 5000, NULL);
 238   }
 239
 240   typedef std::map<NotifyOp, bufferlist> NotifyOpPayloads;
 241   typedef std::set<NotifyOp> NotifyOps;
 242
 243   WatchCtx *m_watch_ctx;
 244
 245   NotifyOps m_notifies;
 246   NotifyOpPayloads m_notify_payloads;
 247   NotifyOpPayloads m_notify_acks;
 248
 249   AsyncRequestId m_async_request_id;
 250
 251   ceph::mutex m_callback_lock = ceph::make_mutex("m_callback_lock");
 252   ceph::condition_variable m_callback_cond;
 253
 254 };
 255
 256 struct ProgressContext : public librbd::ProgressContext {
 257   ceph::mutex mutex = ceph::make_mutex("ProgressContext::mutex");
 258   ceph::condition_variable cond;
 259   bool received;
 260   uint64_t offset;
 261   uint64_t total;
 262
 263   ProgressContext() : received(false),
 264                       offset(0), total(0) {}
 265
 266   int update_progress(uint64_t offset_, uint64_t total_) override {
 267     std::lock_guard l{mutex};
 268     offset = offset_;
 269     total = total_;
 270     received = true;
 271     cond.notify_all();
 272     return 0;
 273   }
 274
 275   bool wait(librbd::ImageCtx *ictx, uint64_t offset_, uint64_t total_) {
 276     std::unique_lock l{mutex};
 277     while (!received) {
 278       if (cond.wait_for(l, 10s) == std::cv_status::timeout) {
 279         break;
 280       }
 281     }
 282     return (received && offset == offset_ && total == total_);
 283   }
 284 };
 285
 286 struct FlattenTask {
 287   librbd::ImageCtx *ictx;
 288   ProgressContext *progress_context;
 289   int result;
 290
 291   FlattenTask(librbd::ImageCtx *ictx_, ProgressContext *ctx)
 292     : ictx(ictx_), progress_context(ctx), result(0) {}
 293
 294   void operator()() {
 295     std::shared_lock l{ictx->owner_lock};
 296     C_SaferCond ctx;
 297     ictx->image_watcher->notify_flatten(0, *progress_context, &ctx);
 298     result = ctx.wait();
 299   }
 300 };
 301
 302 struct ResizeTask {
 303   librbd::ImageCtx *ictx;
 304   ProgressContext *progress_context;
 305   int result;
 306
 307   ResizeTask(librbd::ImageCtx *ictx_, ProgressContext *ctx)
 308     : ictx(ictx_), progress_context(ctx), result(0) {}
 309
 310   void operator()() {
 311     std::shared_lock l{ictx->owner_lock};
 312     C_SaferCond ctx;
 313     ictx->image_watcher->notify_resize(0, 0, true, *progress_context, &ctx);
 314     result = ctx.wait();
 315   }
 316 };
 317
 318 struct SnapCreateTask {
 319   librbd::ImageCtx *ictx;
 320   ProgressContext *progress_context;
 321   int result;
 322
 323   SnapCreateTask(librbd::ImageCtx *ictx_, ProgressContext *ctx)
 324     : ictx(ictx_), progress_context(ctx), result(0) {}
 325
 326   void operator()() {
 327     std::shared_lock l{ictx->owner_lock};
 328     C_SaferCond ctx;
 329     ictx->image_watcher->notify_snap_create(0, cls::rbd::UserSnapshotNamespace(),
 330                                             "snap", 0, *progress_context, &ctx);
 331     result = ctx.wait();
 332   }
 333 };
 334
 335 struct SnapRenameTask {
 336   librbd::ImageCtx *ictx;
 337   int result = 0;
 338
 339   SnapRenameTask(librbd::ImageCtx *ictx)
 340     : ictx(ictx) {
 341   }
 342
 343   void operator()() {
 344     std::shared_lock l{ictx->owner_lock};
 345     C_SaferCond ctx;
 346     ictx->image_watcher->notify_snap_rename(0, 1, "snap-rename", &ctx);
 347     result = ctx.wait();
 348   }
 349 };
 350
 351 struct SnapRemoveTask {
 352   librbd::ImageCtx *ictx;
 353   int result = 0;
 354
 355   SnapRemoveTask(librbd::ImageCtx *ictx)
 356     : ictx(ictx) {
 357   }
 358
 359   void operator()() {
 360     std::shared_lock l{ictx->owner_lock};
 361     C_SaferCond ctx;
 362     ictx->image_watcher->notify_snap_remove(
 363         0, cls::rbd::UserSnapshotNamespace(), "snap", &ctx);
 364     result = ctx.wait();
 365   }
 366 };
 367
 368 struct SnapProtectTask {
 369   librbd::ImageCtx *ictx;
 370   int result = 0;
 371
 372   SnapProtectTask(librbd::ImageCtx *ictx)
 373     : ictx(ictx) {
 374   }
 375
 376   void operator()() {
 377     std::shared_lock l{ictx->owner_lock};
 378     C_SaferCond ctx;
 379     ictx->image_watcher->notify_snap_protect(
 380         0, cls::rbd::UserSnapshotNamespace(), "snap", &ctx);
 381     result = ctx.wait();
 382   }
 383 };
 384
 385 struct SnapUnprotectTask {
 386   librbd::ImageCtx *ictx;
 387   int result = 0;
 388
 389   SnapUnprotectTask(librbd::ImageCtx *ictx)
 390     : ictx(ictx) {
 391   }
 392
 393   void operator()() {
 394     std::shared_lock l{ictx->owner_lock};
 395     C_SaferCond ctx;
 396     ictx->image_watcher->notify_snap_unprotect(
 397         0, cls::rbd::UserSnapshotNamespace(), "snap", &ctx);
 398     result = ctx.wait();
 399   }
 400 };
 401
 402 struct RenameTask {
 403   librbd::ImageCtx *ictx;
 404   int result = 0;
 405
 406   RenameTask(librbd::ImageCtx *ictx)
 407     : ictx(ictx) {
 408   }
 409
 410   void operator()() {
 411     std::shared_lock l{ictx->owner_lock};
 412     C_SaferCond ctx;
 413     ictx->image_watcher->notify_rename(0, "new_name", &ctx);
 414     result = ctx.wait();
 415   }
 416 };
 417
 418 struct RebuildObjectMapTask {
 419   librbd::ImageCtx *ictx;
 420   ProgressContext *progress_context;
 421   int result;
 422
 423   RebuildObjectMapTask(librbd::ImageCtx *ictx_, ProgressContext *ctx)
 424     : ictx(ictx_), progress_context(ctx), result(0) {}
 425
 426   void operator()() {
 427     std::shared_lock l{ictx->owner_lock};
 428     C_SaferCond ctx;
 429     ictx->image_watcher->notify_rebuild_object_map(0, *progress_context, &ctx);
 430     result = ctx.wait();
 431   }
 432 };
 433
 434 TEST_F(TestImageWatcher, NotifyHeaderUpdate) {
 435   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 436
 437   librbd::ImageCtx *ictx;
 438   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 439
 440   ASSERT_EQ(0, register_image_watch(*ictx));
 441
 442   m_notify_acks = {{NOTIFY_OP_HEADER_UPDATE, {}}};
 443   ictx->notify_update();
 444
 445   ASSERT_TRUE(wait_for_notifies(*ictx));
 446
 447   NotifyOps expected_notify_ops;
 448   expected_notify_ops += NOTIFY_OP_HEADER_UPDATE;
 449   ASSERT_EQ(expected_notify_ops, m_notifies);
 450 }
 451
 452 TEST_F(TestImageWatcher, NotifyFlatten) {
 453   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 454
 455   librbd::ImageCtx *ictx;
 456   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 457
 458   ASSERT_EQ(0, register_image_watch(*ictx));
 459   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 460         "auto " + stringify(m_watch_ctx->get_handle())));
 461
 462   m_notify_acks = {{NOTIFY_OP_FLATTEN, create_response_message(0)}};
 463
 464   ProgressContext progress_context;
 465   FlattenTask flatten_task(ictx, &progress_context);
 466   boost::thread thread(boost::ref(flatten_task));
 467
 468   ASSERT_TRUE(wait_for_notifies(*ictx));
 469
 470   NotifyOps expected_notify_ops;
 471   expected_notify_ops += NOTIFY_OP_FLATTEN;
 472   ASSERT_EQ(expected_notify_ops, m_notifies);
 473
 474   AsyncRequestId async_request_id;
 475   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_FLATTEN, &async_request_id));
 476
 477   ASSERT_EQ(0, notify_async_progress(ictx, async_request_id, 10, 20));
 478   ASSERT_TRUE(progress_context.wait(ictx, 10, 20));
 479
 480   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 481
 482   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 483   ASSERT_EQ(0, flatten_task.result);
 484 }
 485
 486 TEST_F(TestImageWatcher, NotifyResize) {
 487   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 488
 489   librbd::ImageCtx *ictx;
 490   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 491
 492   ASSERT_EQ(0, register_image_watch(*ictx));
 493   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 494         "auto " + stringify(m_watch_ctx->get_handle())));
 495
 496   m_notify_acks = {{NOTIFY_OP_RESIZE, create_response_message(0)}};
 497
 498   ProgressContext progress_context;
 499   ResizeTask resize_task(ictx, &progress_context);
 500   boost::thread thread(boost::ref(resize_task));
 501
 502   ASSERT_TRUE(wait_for_notifies(*ictx));
 503
 504   NotifyOps expected_notify_ops;
 505   expected_notify_ops += NOTIFY_OP_RESIZE;
 506   ASSERT_EQ(expected_notify_ops, m_notifies);
 507
 508   AsyncRequestId async_request_id;
 509   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_RESIZE, &async_request_id));
 510
 511   ASSERT_EQ(0, notify_async_progress(ictx, async_request_id, 10, 20));
 512   ASSERT_TRUE(progress_context.wait(ictx, 10, 20));
 513
 514   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 515
 516   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 517   ASSERT_EQ(0, resize_task.result);
 518 }
 519
 520 TEST_F(TestImageWatcher, NotifyRebuildObjectMap) {
 521   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 522
 523   librbd::ImageCtx *ictx;
 524   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 525
 526   ASSERT_EQ(0, register_image_watch(*ictx));
 527   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 528         "auto " + stringify(m_watch_ctx->get_handle())));
 529
 530   m_notify_acks = {{NOTIFY_OP_REBUILD_OBJECT_MAP, create_response_message(0)}};
 531
 532   ProgressContext progress_context;
 533   RebuildObjectMapTask rebuild_task(ictx, &progress_context);
 534   boost::thread thread(boost::ref(rebuild_task));
 535
 536   ASSERT_TRUE(wait_for_notifies(*ictx));
 537
 538   NotifyOps expected_notify_ops;
 539   expected_notify_ops += NOTIFY_OP_REBUILD_OBJECT_MAP;
 540   ASSERT_EQ(expected_notify_ops, m_notifies);
 541
 542   AsyncRequestId async_request_id;
 543   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_REBUILD_OBJECT_MAP,
 544                                        &async_request_id));
 545
 546   ASSERT_EQ(0, notify_async_progress(ictx, async_request_id, 10, 20));
 547   ASSERT_TRUE(progress_context.wait(ictx, 10, 20));
 548
 549   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 550
 551   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 552   ASSERT_EQ(0, rebuild_task.result);
 553 }
 554
 555 TEST_F(TestImageWatcher, NotifySnapCreate) {
 556   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 557
 558   librbd::ImageCtx *ictx;
 559   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 560
 561   ASSERT_EQ(0, register_image_watch(*ictx));
 562   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 563         "auto " + stringify(m_watch_ctx->get_handle())));
 564
 565   m_notify_acks = {{NOTIFY_OP_SNAP_CREATE, create_response_message(0)}};
 566
 567   ProgressContext progress_context;
 568   SnapCreateTask snap_create_task(ictx, &progress_context);
 569   boost::thread thread(boost::ref(snap_create_task));
 570
 571   ASSERT_TRUE(wait_for_notifies(*ictx));
 572
 573   NotifyOps expected_notify_ops;
 574   expected_notify_ops += NOTIFY_OP_SNAP_CREATE;
 575   ASSERT_EQ(expected_notify_ops, m_notifies);
 576
 577   AsyncRequestId async_request_id;
 578   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_SNAP_CREATE,
 579                                        &async_request_id));
 580
 581   ASSERT_EQ(0, notify_async_progress(ictx, async_request_id, 1, 10));
 582   ASSERT_TRUE(progress_context.wait(ictx, 1, 10));
 583
 584   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 585
 586   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 587   ASSERT_EQ(0, snap_create_task.result);
 588 }
 589
 590 TEST_F(TestImageWatcher, NotifySnapCreateError) {
 591   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 592
 593   librbd::ImageCtx *ictx;
 594   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 595
 596   ASSERT_EQ(0, register_image_watch(*ictx));
 597   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 598         "auto " + stringify(m_watch_ctx->get_handle())));
 599
 600   m_notify_acks = {{NOTIFY_OP_SNAP_CREATE, create_response_message(-EEXIST)}};
 601
 602   std::shared_lock l{ictx->owner_lock};
 603   C_SaferCond notify_ctx;
 604   librbd::NoOpProgressContext prog_ctx;
 605   ictx->image_watcher->notify_snap_create(0, cls::rbd::UserSnapshotNamespace(),
 606                                           "snap", 0, prog_ctx, &notify_ctx);
 607   ASSERT_EQ(-EEXIST, notify_ctx.wait());
 608
 609   NotifyOps expected_notify_ops;
 610   expected_notify_ops += NOTIFY_OP_SNAP_CREATE;
 611   ASSERT_EQ(expected_notify_ops, m_notifies);
 612 }
 613
 614 TEST_F(TestImageWatcher, NotifySnapRename) {
 615   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 616
 617   librbd::ImageCtx *ictx;
 618   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 619
 620   ASSERT_EQ(0, register_image_watch(*ictx));
 621   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 622         "auto " + stringify(m_watch_ctx->get_handle())));
 623
 624   m_notify_acks = {{NOTIFY_OP_SNAP_RENAME, create_response_message(0)}};
 625
 626   SnapRenameTask snap_rename_task(ictx);
 627   boost::thread thread(boost::ref(snap_rename_task));
 628
 629   ASSERT_TRUE(wait_for_notifies(*ictx));
 630
 631   NotifyOps expected_notify_ops;
 632   expected_notify_ops += NOTIFY_OP_SNAP_RENAME;
 633   ASSERT_EQ(expected_notify_ops, m_notifies);
 634
 635   AsyncRequestId async_request_id;
 636   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_SNAP_RENAME,
 637                                        &async_request_id));
 638
 639   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 640
 641   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 642   ASSERT_EQ(0, snap_rename_task.result);
 643 }
 644
 645 TEST_F(TestImageWatcher, NotifySnapRenameError) {
 646   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 647
 648   librbd::ImageCtx *ictx;
 649   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 650
 651   ASSERT_EQ(0, register_image_watch(*ictx));
 652   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 653         "auto " + stringify(m_watch_ctx->get_handle())));
 654
 655   m_notify_acks = {{NOTIFY_OP_SNAP_RENAME, create_response_message(-EEXIST)}};
 656
 657   std::shared_lock l{ictx->owner_lock};
 658   C_SaferCond notify_ctx;
 659   ictx->image_watcher->notify_snap_rename(0, 1, "snap-rename", &notify_ctx);
 660   ASSERT_EQ(-EEXIST, notify_ctx.wait());
 661
 662   NotifyOps expected_notify_ops;
 663   expected_notify_ops += NOTIFY_OP_SNAP_RENAME;
 664   ASSERT_EQ(expected_notify_ops, m_notifies);
 665 }
 666
 667 TEST_F(TestImageWatcher, NotifySnapRemove) {
 668   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 669
 670   librbd::ImageCtx *ictx;
 671   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 672
 673   ASSERT_EQ(0, register_image_watch(*ictx));
 674   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 675         "auto " + stringify(m_watch_ctx->get_handle())));
 676
 677   m_notify_acks = {{NOTIFY_OP_SNAP_REMOVE, create_response_message(0)}};
 678
 679   SnapRemoveTask snap_remove_task(ictx);
 680   boost::thread thread(boost::ref(snap_remove_task));
 681
 682   ASSERT_TRUE(wait_for_notifies(*ictx));
 683
 684   NotifyOps expected_notify_ops;
 685   expected_notify_ops += NOTIFY_OP_SNAP_REMOVE;
 686   ASSERT_EQ(expected_notify_ops, m_notifies);
 687
 688   AsyncRequestId async_request_id;
 689   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_SNAP_REMOVE,
 690                                        &async_request_id));
 691
 692   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 693
 694   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 695   ASSERT_EQ(0, snap_remove_task.result);
 696 }
 697
 698 TEST_F(TestImageWatcher, NotifySnapProtect) {
 699   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 700
 701   librbd::ImageCtx *ictx;
 702   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 703
 704   ASSERT_EQ(0, register_image_watch(*ictx));
 705   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 706         "auto " + stringify(m_watch_ctx->get_handle())));
 707
 708   m_notify_acks = {{NOTIFY_OP_SNAP_PROTECT, create_response_message(0)}};
 709
 710   SnapProtectTask snap_protect_task(ictx);
 711   boost::thread thread(boost::ref(snap_protect_task));
 712
 713   ASSERT_TRUE(wait_for_notifies(*ictx));
 714
 715   NotifyOps expected_notify_ops;
 716   expected_notify_ops += NOTIFY_OP_SNAP_PROTECT;
 717   ASSERT_EQ(expected_notify_ops, m_notifies);
 718
 719   AsyncRequestId async_request_id;
 720   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_SNAP_PROTECT,
 721                                        &async_request_id));
 722
 723   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 724
 725   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 726   ASSERT_EQ(0, snap_protect_task.result);
 727 }
 728
 729 TEST_F(TestImageWatcher, NotifySnapUnprotect) {
 730   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 731
 732   librbd::ImageCtx *ictx;
 733   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 734
 735   ASSERT_EQ(0, register_image_watch(*ictx));
 736   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 737         "auto " + stringify(m_watch_ctx->get_handle())));
 738
 739   m_notify_acks = {{NOTIFY_OP_SNAP_UNPROTECT, create_response_message(0)}};
 740
 741   SnapUnprotectTask snap_unprotect_task(ictx);
 742   boost::thread thread(boost::ref(snap_unprotect_task));
 743
 744   ASSERT_TRUE(wait_for_notifies(*ictx));
 745
 746   NotifyOps expected_notify_ops;
 747   expected_notify_ops += NOTIFY_OP_SNAP_UNPROTECT;
 748   ASSERT_EQ(expected_notify_ops, m_notifies);
 749
 750   AsyncRequestId async_request_id;
 751   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_SNAP_UNPROTECT,
 752                                        &async_request_id));
 753
 754   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 755
 756   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 757   ASSERT_EQ(0, snap_unprotect_task.result);
 758 }
 759
 760 TEST_F(TestImageWatcher, NotifyRename) {
 761   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 762
 763   librbd::ImageCtx *ictx;
 764   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 765
 766   ASSERT_EQ(0, register_image_watch(*ictx));
 767   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 768         "auto " + stringify(m_watch_ctx->get_handle())));
 769
 770   m_notify_acks = {{NOTIFY_OP_RENAME, create_response_message(0)}};
 771
 772   RenameTask rename_task(ictx);
 773   boost::thread thread(boost::ref(rename_task));
 774
 775   ASSERT_TRUE(wait_for_notifies(*ictx));
 776
 777   NotifyOps expected_notify_ops;
 778   expected_notify_ops += NOTIFY_OP_RENAME;
 779   ASSERT_EQ(expected_notify_ops, m_notifies);
 780
 781   AsyncRequestId async_request_id;
 782   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_RENAME,
 783                                        &async_request_id));
 784
 785   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, 0));
 786
 787   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 788   ASSERT_EQ(0, rename_task.result);
 789 }
 790
 791 TEST_F(TestImageWatcher, NotifyAsyncTimedOut) {
 792   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 793
 794   librbd::ImageCtx *ictx;
 795   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 796
 797   ASSERT_EQ(0, register_image_watch(*ictx));
 798   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 799         "auto " + stringify(m_watch_ctx->get_handle())));
 800
 801   m_notify_acks = {{NOTIFY_OP_FLATTEN, {}}};
 802
 803   ProgressContext progress_context;
 804   FlattenTask flatten_task(ictx, &progress_context);
 805   boost::thread thread(boost::ref(flatten_task));
 806
 807   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 808   ASSERT_EQ(-ETIMEDOUT, flatten_task.result);
 809 }
 810
 811 TEST_F(TestImageWatcher, NotifyAsyncError) {
 812   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 813
 814   librbd::ImageCtx *ictx;
 815   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 816
 817   ASSERT_EQ(0, register_image_watch(*ictx));
 818   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 819         "auto " + stringify(m_watch_ctx->get_handle())));
 820
 821   m_notify_acks = {{NOTIFY_OP_FLATTEN, create_response_message(-EIO)}};
 822
 823   ProgressContext progress_context;
 824   FlattenTask flatten_task(ictx, &progress_context);
 825   boost::thread thread(boost::ref(flatten_task));
 826
 827   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 828   ASSERT_EQ(-EIO, flatten_task.result);
 829 }
 830
 831 TEST_F(TestImageWatcher, NotifyAsyncCompleteError) {
 832   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 833
 834   librbd::ImageCtx *ictx;
 835   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 836
 837   ASSERT_EQ(0, register_image_watch(*ictx));
 838   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 839         "auto " + stringify(m_watch_ctx->get_handle())));
 840
 841   m_notify_acks = {{NOTIFY_OP_FLATTEN, create_response_message(0)}};
 842
 843   ProgressContext progress_context;
 844   FlattenTask flatten_task(ictx, &progress_context);
 845   boost::thread thread(boost::ref(flatten_task));
 846
 847   ASSERT_TRUE(wait_for_notifies(*ictx));
 848
 849   NotifyOps expected_notify_ops;
 850   expected_notify_ops += NOTIFY_OP_FLATTEN;
 851   ASSERT_EQ(expected_notify_ops, m_notifies);
 852
 853   AsyncRequestId async_request_id;
 854   ASSERT_TRUE(extract_async_request_id(NOTIFY_OP_FLATTEN, &async_request_id));
 855
 856   ASSERT_EQ(0, notify_async_complete(ictx, async_request_id, -ESHUTDOWN));
 857
 858   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 859   ASSERT_EQ(-ESHUTDOWN, flatten_task.result);
 860 }
 861
 862 TEST_F(TestImageWatcher, NotifyAsyncRequestTimedOut) {
 863   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 864
 865   librbd::ImageCtx *ictx;
 866   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 867
 868   ictx->config.set_val("rbd_request_timed_out_seconds", "0");
 869
 870   ASSERT_EQ(0, register_image_watch(*ictx));
 871   ASSERT_EQ(0, lock_image(*ictx, ClsLockType::EXCLUSIVE,
 872                           "auto " + stringify(m_watch_ctx->get_handle())));
 873
 874   m_notify_acks = {{NOTIFY_OP_FLATTEN, create_response_message(0)}};
 875
 876   ProgressContext progress_context;
 877   FlattenTask flatten_task(ictx, &progress_context);
 878   boost::thread thread(boost::ref(flatten_task));
 879
 880   ASSERT_TRUE(wait_for_notifies(*ictx));
 881
 882   ASSERT_TRUE(thread.timed_join(boost::posix_time::seconds(10)));
 883   ASSERT_EQ(-ETIMEDOUT, flatten_task.result);
 884 }
 885