import 15.2.0 Octopus source

[ceph.git] / ceph / src / test / librados / tier_cxx.cc

diff --git a/ceph/src/test/librados/tier_cxx.cc b/ceph/src/test/librados/tier_cxx.cc
index a9727c08e0b66330de754d5c5367837bb1339648..42c03c278e92ea875285c5059e56da66dcaa888f 100644 (file)
--- a/ceph/src/test/librados/tier_cxx.cc
+++ b/ceph/src/test/librados/tier_cxx.cc
@@ -45,7 +45,7 @@ void flush_evict_all(librados::Rados& cluster, librados::IoCtx& cache_ioctx)
       cache_ioctx.aio_operate(
         it->get_oid(), completion, &op,
        librados::OPERATION_IGNORE_OVERLAY, NULL);
-      completion->wait_for_safe();
+      completion->wait_for_complete();
       completion->get_return_value();
       completion->release();
     }
@@ -56,13 +56,39 @@ void flush_evict_all(librados::Rados& cluster, librados::IoCtx& cache_ioctx)
       cache_ioctx.aio_operate(
         it->get_oid(), completion, &op,
        librados::OPERATION_IGNORE_OVERLAY, NULL);
-      completion->wait_for_safe();
+      completion->wait_for_complete();
       completion->get_return_value();
       completion->release();
     }
   }
 }
 
+static string _get_required_osd_release(Rados& cluster)
+{
+  bufferlist inbl;
+  string cmd = string("{\"prefix\": \"osd dump\",\"format\":\"json\"}");
+  bufferlist outbl;
+  int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
+  ceph_assert(r >= 0);
+  string outstr(outbl.c_str(), outbl.length());
+  json_spirit::Value v;
+  if (!json_spirit::read(outstr, v)) {
+    cerr <<" unable to parse json " << outstr << std::endl;
+    return "";
+  }
+
+  json_spirit::Object& o = v.get_obj();
+  for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
+    json_spirit::Pair& p = o[i];
+    if (p.name_ == "require_osd_release") {
+      cout << "require_osd_release = " << p.value_.get_str() << std::endl;
+      return p.value_.get_str();
+    }
+  }
+  cerr << "didn't find require_osd_release in " << outstr << std::endl;
+  return "";
+}
+
 class LibRadosTwoPoolsPP : public RadosTestPP
 {
 public:
@@ -248,7 +274,7 @@ TEST_F(LibRadosTwoPoolsPP, Overlay) {
     ASSERT_EQ(0, ioctx.aio_operate(
        "foo", completion, &op,
        librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
     ASSERT_EQ('b', bl[0]);
@@ -607,10 +633,12 @@ TEST_F(LibRadosTwoPoolsPP, PromoteSnapTrimRace) {
 
   ioctx.snap_set_read(my_snaps[0]);
 
-  // read foo snap
+  // read foo snap.  the OSD may or may not realize that this snap has
+  // been logically deleted; either response is valid.
   {
     bufferlist bl;
-    ASSERT_EQ(-ENOENT, ioctx.read("foo", bl, 1, 0));
+    int r = ioctx.read("foo", bl, 1, 0);
+    ASSERT_TRUE(r == 1 || r == -ENOENT);
   }
 
   // cleanup
@@ -686,7 +714,7 @@ TEST_F(LibRadosTwoPoolsPP, Whiteout) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
                                   librados::OPERATION_IGNORE_CACHE));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
 
@@ -817,7 +845,7 @@ TEST_F(LibRadosTwoPoolsPP, Evict) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -830,7 +858,7 @@ TEST_F(LibRadosTwoPoolsPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
                                         librados::OPERATION_IGNORE_CACHE,
                                         NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EPERM, completion->get_return_value());
     completion->release();
   }
@@ -841,7 +869,7 @@ TEST_F(LibRadosTwoPoolsPP, Evict) {
     op.cache_unpin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -854,7 +882,7 @@ TEST_F(LibRadosTwoPoolsPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -878,7 +906,7 @@ TEST_F(LibRadosTwoPoolsPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
                                         librados::OPERATION_IGNORE_CACHE,
                                         NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -889,7 +917,7 @@ TEST_F(LibRadosTwoPoolsPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -900,7 +928,7 @@ TEST_F(LibRadosTwoPoolsPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EBUSY, completion->get_return_value());
     completion->release();
   }
@@ -1008,7 +1036,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "bam", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1020,7 +1048,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "bam", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-ENOENT, completion->get_return_value());
     completion->release();
   }
@@ -1041,7 +1069,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1054,7 +1082,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-ENOENT, completion->get_return_value());
     completion->release();
   }
@@ -1068,7 +1096,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1096,7 +1124,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EBUSY, completion->get_return_value());
     completion->release();
   }
@@ -1110,7 +1138,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1124,7 +1152,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1135,7 +1163,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1200,7 +1228,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap2) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1214,7 +1242,7 @@ TEST_F(LibRadosTwoPoolsPP, EvictSnap2) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op,
                                   librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-ENOENT, completion->get_return_value());
     completion->release();
   }
@@ -1326,7 +1354,7 @@ TEST_F(LibRadosTwoPoolsPP, ListSnap){
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1339,7 +1367,7 @@ TEST_F(LibRadosTwoPoolsPP, ListSnap){
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-ENOENT, completion->get_return_value());
     completion->release();
   }
@@ -1355,7 +1383,7 @@ TEST_F(LibRadosTwoPoolsPP, ListSnap){
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       0, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, snap_ret);
     ASSERT_LT(0u, snap_set.clones.size());
     for (vector<librados::clone_info_t>::const_iterator r = snap_set.clones.begin();
@@ -1432,7 +1460,7 @@ TEST_F(LibRadosTwoPoolsPP, TryFlush) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1446,7 +1474,7 @@ TEST_F(LibRadosTwoPoolsPP, TryFlush) {
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EPERM, completion->get_return_value());
     completion->release();
   }
@@ -1457,7 +1485,7 @@ TEST_F(LibRadosTwoPoolsPP, TryFlush) {
     op.cache_unpin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1471,7 +1499,7 @@ TEST_F(LibRadosTwoPoolsPP, TryFlush) {
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1503,7 +1531,7 @@ TEST_F(LibRadosTwoPoolsPP, TryFlush) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1579,7 +1607,7 @@ TEST_F(LibRadosTwoPoolsPP, Flush) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1593,7 +1621,7 @@ TEST_F(LibRadosTwoPoolsPP, Flush) {
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EPERM, completion->get_return_value());
     completion->release();
   }
@@ -1604,7 +1632,7 @@ TEST_F(LibRadosTwoPoolsPP, Flush) {
     op.cache_unpin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1617,7 +1645,7 @@ TEST_F(LibRadosTwoPoolsPP, Flush) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1649,7 +1677,7 @@ TEST_F(LibRadosTwoPoolsPP, Flush) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1682,7 +1710,7 @@ TEST_F(LibRadosTwoPoolsPP, Flush) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1694,7 +1722,7 @@ TEST_F(LibRadosTwoPoolsPP, Flush) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1791,7 +1819,7 @@ TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EBUSY, completion->get_return_value());
     completion->release();
   }
@@ -1804,7 +1832,7 @@ TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EBUSY, completion->get_return_value());
     completion->release();
   }
@@ -1817,7 +1845,7 @@ TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1830,7 +1858,7 @@ TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1843,7 +1871,7 @@ TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -1960,8 +1988,8 @@ TEST_F(LibRadosTierPP, FlushWriteRaces) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion2, &op2, 0));
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     ASSERT_EQ(0, completion2->get_return_value());
     completion->release();
@@ -1994,8 +2022,8 @@ TEST_F(LibRadosTierPP, FlushWriteRaces) {
       librados::AioCompletion *completion2 = cluster.aio_create_completion();
       ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
 
-      completion->wait_for_safe();
-      completion2->wait_for_safe();
+      completion->wait_for_complete();
+      completion2->wait_for_complete();
       int r = completion->get_return_value();
       ASSERT_TRUE(r == -EBUSY || r == 0);
       ASSERT_EQ(0, completion2->get_return_value());
@@ -2070,8 +2098,8 @@ TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
       "foo", completion2, &op2,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     ASSERT_EQ(0, completion2->get_return_value());
     completion->release();
@@ -2104,8 +2132,8 @@ TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     ASSERT_EQ(0, completion2->get_return_value());
     completion->release();
@@ -2141,8 +2169,8 @@ TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
         "foo", completion2, &op2,
        librados::OPERATION_IGNORE_OVERLAY, NULL));
 
-      completion->wait_for_safe();
-      completion2->wait_for_safe();
+      completion->wait_for_complete();
+      completion2->wait_for_complete();
       int r = completion->get_return_value();
       ASSERT_TRUE(r == -EBUSY || r == 0);
       ASSERT_EQ(0, completion2->get_return_value());
@@ -2182,8 +2210,8 @@ TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     ASSERT_EQ(0, completion2->get_return_value());
     completion->release();
@@ -2193,8 +2221,8 @@ TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
 
 
 IoCtx *read_ioctx = 0;
-Mutex test_lock("FlushReadRaces::lock");
-Cond cond;
+ceph::mutex test_lock = ceph::make_mutex("FlushReadRaces::lock");
+ceph::condition_variable cond;
 int max_reads = 100;
 int num_reads = 0; // in progress
 
@@ -2213,14 +2241,13 @@ void start_flush_read()
 void flush_read_race_cb(completion_t cb, void *arg)
 {
   //cout << " finished read" << std::endl;
-  test_lock.Lock();
+  std::lock_guard l{test_lock};
   if (num_reads > max_reads) {
     num_reads--;
-    cond.Signal();
+    cond.notify_all();
   } else {
     start_flush_read();
   }
-  test_lock.Unlock();
 }
 
 TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
@@ -2257,12 +2284,12 @@ TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
 
   // start a continuous stream of reads
   read_ioctx = &ioctx;
-  test_lock.Lock();
+  test_lock.lock();
   for (int i = 0; i < max_reads; ++i) {
     start_flush_read();
     num_reads++;
   }
-  test_lock.Unlock();
+  test_lock.unlock();
 
   // try-flush
   ObjectReadOperation op;
@@ -2273,16 +2300,14 @@ TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
 
-  completion->wait_for_safe();
+  completion->wait_for_complete();
   ASSERT_EQ(0, completion->get_return_value());
   completion->release();
 
   // stop reads
-  test_lock.Lock();
-  max_reads = 0;
-  while (num_reads > 0)
-    cond.Wait(test_lock);
-  test_lock.Unlock();
+  std::unique_lock locker{test_lock};
+  max_reads = 0;  
+  cond.wait(locker, [] { return num_reads == 0;});
 }
 
 TEST_F(LibRadosTierPP, HitSetNone) {
@@ -2410,7 +2435,6 @@ static int _get_pg_num(Rados& cluster, string pool_name)
   return -1;
 }
 
-
 TEST_F(LibRadosTwoPoolsPP, HitSetWrite) {
   int num_pg = _get_pg_num(cluster, pool_name);
   ceph_assert(num_pg > 0);
@@ -2827,7 +2851,7 @@ TEST_F(LibRadosTwoPoolsPP, CachePin) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -2836,7 +2860,7 @@ TEST_F(LibRadosTwoPoolsPP, CachePin) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -2932,7 +2956,7 @@ TEST_F(LibRadosTwoPoolsPP, SetRedirectRead) {
     op.set_redirect("bar", cache_ioctx, 0);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -2954,16 +2978,8 @@ TEST_F(LibRadosTwoPoolsPP, SetRedirectRead) {
 
 TEST_F(LibRadosTwoPoolsPP, SetChunkRead) {
   // skip test if not yet mimic
-  {
-    bufferlist inbl, outbl;
-    ASSERT_EQ(0, cluster.mon_command(
-               "{\"prefix\": \"osd dump\"}",
-               inbl, &outbl, NULL));
-    string s(outbl.c_str(), outbl.length());
-    if (s.find("mimic") == std::string::npos) {
-      cout << "cluster is not yet mimic, skipping test" << std::endl;
-      return;
-    }
+  if (_get_required_osd_release(cluster) < "mimic") {
+    GTEST_SKIP() << "cluster is not yet mimic, skipping test";
   }
 
   // create object
@@ -3000,7 +3016,7 @@ TEST_F(LibRadosTwoPoolsPP, SetChunkRead) {
     }
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3032,16 +3048,8 @@ TEST_F(LibRadosTwoPoolsPP, SetChunkRead) {
 
 TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
   // skip test if not yet mimic
-  {
-    bufferlist inbl, outbl;
-    ASSERT_EQ(0, cluster.mon_command(
-               "{\"prefix\": \"osd dump\"}",
-               inbl, &outbl, NULL));
-    string s(outbl.c_str(), outbl.length());
-    if (s.find("mimic") == std::string::npos) {
-      cout << "cluster is not yet mimic, skipping test" << std::endl;
-      return;
-    }
+  if (_get_required_osd_release(cluster) < "mimic") {
+    GTEST_SKIP() << "cluster is not yet mimic, skipping test";
   }
 
   // create object
@@ -3091,7 +3099,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
     op.set_redirect("bar", cache_ioctx, 0);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3101,7 +3109,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
     op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3111,7 +3119,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
     op.tier_promote();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3127,7 +3135,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
     op.tier_promote();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3189,7 +3197,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestRefRead) {
     op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3199,7 +3207,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestRefRead) {
     op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3236,16 +3244,8 @@ TEST_F(LibRadosTwoPoolsPP, ManifestRefRead) {
 
 TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
   // skip test if not yet nautilus
-  {
-    bufferlist inbl, outbl;
-    ASSERT_EQ(0, cluster.mon_command(
-               "{\"prefix\": \"osd dump\"}",
-               inbl, &outbl, NULL));
-    string s(outbl.c_str(), outbl.length());
-    if (s.find("nautilus") == std::string::npos) {
-      cout << "cluster is not yet nautilus, skipping test" << std::endl;
-      return;
-    }
+  if (_get_required_osd_release(cluster) < "nautilus") {
+    GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
   }
 
   // create object
@@ -3287,7 +3287,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
     op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3297,7 +3297,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
     op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3334,7 +3334,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
     op.unset_manifest();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3345,7 +3345,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
     op.unset_manifest();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3358,7 +3358,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
       op.unset_manifest();
       librados::AioCompletion *completion = cluster.aio_create_completion();
       ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-      completion->wait_for_safe();
+      completion->wait_for_complete();
       ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
       completion->release();
     }
@@ -3372,7 +3372,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
       op.unset_manifest();
       librados::AioCompletion *completion = cluster.aio_create_completion();
       ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
-      completion->wait_for_safe();
+      completion->wait_for_complete();
       ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
       completion->release();
     }
@@ -3387,17 +3387,10 @@ using ceph::crypto::SHA1;
 #include "rgw/rgw_common.h"
 TEST_F(LibRadosTwoPoolsPP, ManifestDedupRefRead) {
   // skip test if not yet nautilus
-  {
-    bufferlist inbl, outbl;
-    ASSERT_EQ(0, cluster.mon_command(
-               "{\"prefix\": \"osd dump\"}",
-               inbl, &outbl, NULL));
-    string s(outbl.c_str(), outbl.length());
-    if (s.find("nautilus") == std::string::npos) {
-      cout << "cluster is not yet nautilus, skipping test" << std::endl;
-      return;
-    }
+  if (_get_required_osd_release(cluster) < "nautilus") {
+    GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
   }
+
   bufferlist inbl;
   ASSERT_EQ(0, cluster.mon_command(
            set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
@@ -3445,7 +3438,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestDedupRefRead) {
                CEPH_OSD_OP_FLAG_WITH_REFERENCE);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo-dedup", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3457,7 +3450,7 @@ TEST_F(LibRadosTwoPoolsPP, ManifestDedupRefRead) {
                CEPH_OSD_OP_FLAG_WITH_REFERENCE);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -3519,6 +3512,93 @@ TEST_F(LibRadosTwoPoolsPP, ManifestDedupRefRead) {
   cluster.wait_for_latest_osdmap();
 }
 
+TEST_F(LibRadosTwoPoolsPP, ManifestFlushRead) {
+  // skip test if not yet octopus 
+  if (_get_required_osd_release(cluster) < "octopus") {
+    GTEST_SKIP() << "cluster is not yet octopus, skipping test";
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("base chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("CHUNKS");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
+  }
+
+  // configure tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // set-chunk
+  {
+    ObjectWriteOperation op;
+    op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // set-chunk
+  {
+    ObjectWriteOperation op;
+    op.set_chunk(2, 2, cache_ioctx, "bar-chunk", 2);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // make chunked object dirty
+  {
+    bufferlist bl;
+    bl.append("DD");
+    ObjectWriteOperation op;
+    op.write(0, bl);
+    ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
+  }
+  // flush
+  {
+    ObjectWriteOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // read and verify the chunked object
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("bar-chunk", bl, 1, 0));
+    ASSERT_EQ('D', bl[0]);
+  }
+
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
 class LibRadosTwoPoolsECPP : public RadosTestECPP
 {
 public:
@@ -3683,7 +3763,7 @@ TEST_F(LibRadosTwoPoolsECPP, Overlay) {
     ASSERT_EQ(0, ioctx.aio_operate(
        "foo", completion, &op,
        librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
     ASSERT_EQ('b', bl[0]);
@@ -3963,10 +4043,12 @@ TEST_F(LibRadosTwoPoolsECPP, PromoteSnapTrimRace) {
 
   ioctx.snap_set_read(my_snaps[0]);
 
-  // read foo snap
+  // read foo snap.  the OSD may or may not realize that this snap has
+  // been logically deleted; either response is valid.
   {
     bufferlist bl;
-    ASSERT_EQ(-ENOENT, ioctx.read("foo", bl, 1, 0));
+    int r = ioctx.read("foo", bl, 1, 0);
+    ASSERT_TRUE(r == 1 || r == -ENOENT);
   }
 
   // cleanup
@@ -4042,7 +4124,7 @@ TEST_F(LibRadosTwoPoolsECPP, Whiteout) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
                                   librados::OPERATION_IGNORE_CACHE));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
 
@@ -4128,7 +4210,7 @@ TEST_F(LibRadosTwoPoolsECPP, Evict) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4141,7 +4223,7 @@ TEST_F(LibRadosTwoPoolsECPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
                                         librados::OPERATION_IGNORE_CACHE,
                                         NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EPERM, completion->get_return_value());
     completion->release();
   }
@@ -4152,7 +4234,7 @@ TEST_F(LibRadosTwoPoolsECPP, Evict) {
     op.cache_unpin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4165,7 +4247,7 @@ TEST_F(LibRadosTwoPoolsECPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4189,7 +4271,7 @@ TEST_F(LibRadosTwoPoolsECPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
                                         librados::OPERATION_IGNORE_CACHE,
                                         NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4200,7 +4282,7 @@ TEST_F(LibRadosTwoPoolsECPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4211,7 +4293,7 @@ TEST_F(LibRadosTwoPoolsECPP, Evict) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EBUSY, completion->get_return_value());
     completion->release();
   }
@@ -4319,7 +4401,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "bam", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4331,7 +4413,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "bam", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-ENOENT, completion->get_return_value());
     completion->release();
   }
@@ -4352,7 +4434,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4365,7 +4447,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-ENOENT, completion->get_return_value());
     completion->release();
   }
@@ -4379,7 +4461,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4407,7 +4489,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EBUSY, completion->get_return_value());
     completion->release();
   }
@@ -4421,7 +4503,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4435,7 +4517,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4446,7 +4528,7 @@ TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "bar", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4516,7 +4598,7 @@ TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4530,7 +4612,7 @@ TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EPERM, completion->get_return_value());
     completion->release();
   }
@@ -4541,7 +4623,7 @@ TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
     op.cache_unpin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4555,7 +4637,7 @@ TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4587,7 +4669,7 @@ TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4651,7 +4733,7 @@ TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
     op.omap_set(omap);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4664,7 +4746,7 @@ TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_NE(0, completion->get_return_value());
     completion->release();
   }
@@ -4708,7 +4790,7 @@ TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4719,7 +4801,7 @@ TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4800,7 +4882,7 @@ TEST_F(LibRadosTwoPoolsECPP, Flush) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4814,7 +4896,7 @@ TEST_F(LibRadosTwoPoolsECPP, Flush) {
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EPERM, completion->get_return_value());
     completion->release();
   }
@@ -4825,7 +4907,7 @@ TEST_F(LibRadosTwoPoolsECPP, Flush) {
     op.cache_unpin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4838,7 +4920,7 @@ TEST_F(LibRadosTwoPoolsECPP, Flush) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4870,7 +4952,7 @@ TEST_F(LibRadosTwoPoolsECPP, Flush) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4903,7 +4985,7 @@ TEST_F(LibRadosTwoPoolsECPP, Flush) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -4915,7 +4997,7 @@ TEST_F(LibRadosTwoPoolsECPP, Flush) {
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate(
         "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -5012,7 +5094,7 @@ TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EBUSY, completion->get_return_value());
     completion->release();
   }
@@ -5025,7 +5107,7 @@ TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(-EBUSY, completion->get_return_value());
     completion->release();
   }
@@ -5038,7 +5120,7 @@ TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -5051,7 +5133,7 @@ TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -5064,7 +5146,7 @@ TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_CACHE, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -5182,8 +5264,8 @@ TEST_F(LibRadosTierECPP, FlushWriteRaces) {
     ASSERT_EQ(0, ioctx.aio_operate(
       "foo", completion2, &op2, 0));
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     ASSERT_EQ(0, completion2->get_return_value());
     completion->release();
@@ -5216,8 +5298,8 @@ TEST_F(LibRadosTierECPP, FlushWriteRaces) {
       librados::AioCompletion *completion2 = cluster.aio_create_completion();
       ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
 
-      completion->wait_for_safe();
-      completion2->wait_for_safe();
+      completion->wait_for_complete();
+      completion2->wait_for_complete();
       int r = completion->get_return_value();
       ASSERT_TRUE(r == -EBUSY || r == 0);
       ASSERT_EQ(0, completion2->get_return_value());
@@ -5292,8 +5374,8 @@ TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
       "foo", completion2, &op2,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     ASSERT_EQ(0, completion2->get_return_value());
     completion->release();
@@ -5326,8 +5408,8 @@ TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     ASSERT_EQ(0, completion2->get_return_value());
     completion->release();
@@ -5363,8 +5445,8 @@ TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
         "foo", completion2, &op2,
        librados::OPERATION_IGNORE_OVERLAY, NULL));
 
-      completion->wait_for_safe();
-      completion2->wait_for_safe();
+      completion->wait_for_complete();
+      completion2->wait_for_complete();
       int r = completion->get_return_value();
       ASSERT_TRUE(r == -EBUSY || r == 0);
       ASSERT_EQ(0, completion2->get_return_value());
@@ -5404,8 +5486,8 @@ TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
 
-    completion->wait_for_safe();
-    completion2->wait_for_safe();
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     ASSERT_EQ(0, completion2->get_return_value());
     completion->release();
@@ -5447,12 +5529,12 @@ TEST_F(LibRadosTwoPoolsECPP, TryFlushReadRace) {
 
   // start a continuous stream of reads
   read_ioctx = &ioctx;
-  test_lock.Lock();
+  test_lock.lock();
   for (int i = 0; i < max_reads; ++i) {
     start_flush_read();
     num_reads++;
   }
-  test_lock.Unlock();
+  test_lock.unlock();
 
   // try-flush
   ObjectReadOperation op;
@@ -5463,16 +5545,14 @@ TEST_F(LibRadosTwoPoolsECPP, TryFlushReadRace) {
       librados::OPERATION_IGNORE_OVERLAY |
       librados::OPERATION_SKIPRWLOCKS, NULL));
 
-  completion->wait_for_safe();
+  completion->wait_for_complete();
   ASSERT_EQ(0, completion->get_return_value());
   completion->release();
 
   // stop reads
-  test_lock.Lock();
-  max_reads = 0;
-  while (num_reads > 0)
-    cond.Wait(test_lock);
-  test_lock.Unlock();
+  std::unique_lock locker{test_lock};
+  max_reads = 0;  
+  cond.wait(locker, [] { return num_reads == 0;});
 }
 
 TEST_F(LibRadosTierECPP, CallForcesPromote) {
@@ -5538,7 +5618,7 @@ TEST_F(LibRadosTierECPP, CallForcesPromote) {
     ASSERT_EQ(0, cache_ioctx.aio_operate(
       "foo", completion, &op,
       librados::OPERATION_IGNORE_OVERLAY, NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -5551,7 +5631,7 @@ TEST_F(LibRadosTierECPP, CallForcesPromote) {
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
                                         librados::OPERATION_IGNORE_CACHE,
                                         NULL));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -5589,7 +5669,7 @@ TEST_F(LibRadosTierECPP, CallForcesPromote) {
   cluster.wait_for_latest_osdmap();
 
   ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
-  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+  ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, cluster));
 }
 
 TEST_F(LibRadosTierECPP, HitSetNone) {
@@ -6085,7 +6165,7 @@ TEST_F(LibRadosTwoPoolsECPP, CachePin) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -6094,7 +6174,7 @@ TEST_F(LibRadosTwoPoolsECPP, CachePin) {
     op.cache_pin();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -6189,7 +6269,7 @@ TEST_F(LibRadosTwoPoolsECPP, SetRedirectRead) {
     op.set_redirect("bar", cache_ioctx, 0);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -6243,7 +6323,7 @@ TEST_F(LibRadosTwoPoolsECPP, SetChunkRead) {
     op.set_chunk(0, 8, cache_ioctx, "bar", 0);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -6321,7 +6401,7 @@ TEST_F(LibRadosTwoPoolsECPP, ManifestPromoteRead) {
     op.set_redirect("bar", cache_ioctx, 0);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -6331,7 +6411,7 @@ TEST_F(LibRadosTwoPoolsECPP, ManifestPromoteRead) {
     op.set_chunk(0, 10, cache_ioctx, "bar-chunk", 0);
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -6341,7 +6421,7 @@ TEST_F(LibRadosTwoPoolsECPP, ManifestPromoteRead) {
     op.tier_promote();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -6357,7 +6437,7 @@ TEST_F(LibRadosTwoPoolsECPP, ManifestPromoteRead) {
     op.tier_promote();
     librados::AioCompletion *completion = cluster.aio_create_completion();
     ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
-    completion->wait_for_safe();
+    completion->wait_for_complete();
     ASSERT_EQ(0, completion->get_return_value());
     completion->release();
   }
@@ -6428,3 +6508,71 @@ TEST_F(LibRadosTwoPoolsPP, PropagateBaseTierError) {
 
   ASSERT_EQ(-ECANCELED, ioctx.operate("propagate-base-tier-error", &op2));
 }
+
+TEST_F(LibRadosTwoPoolsPP, HelloWriteReturn) {
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // set things up such that the op would normally be proxied
+  ASSERT_EQ(0, cluster.mon_command(
+             set_pool_str(cache_pool_name, "hit_set_count", 2),
+             inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+             set_pool_str(cache_pool_name, "hit_set_period", 600),
+             inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+             set_pool_str(cache_pool_name, "hit_set_type",
+                          "explicit_object"),
+             inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+             set_pool_str(cache_pool_name, "min_read_recency_for_promote",
+                          "10000"),
+             inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // this *will* return data due to the RETURNVEC flag
+  {
+    bufferlist in, out;
+    int rval;
+    ObjectWriteOperation o;
+    o.exec("hello", "write_return_data", in, &out, &rval);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
+                                  librados::OPERATION_RETURNVEC));
+    completion->wait_for_complete();
+    ASSERT_EQ(42, completion->get_return_value());
+    ASSERT_EQ(42, rval);
+    out.hexdump(std::cout);
+    ASSERT_EQ("you might see this", std::string(out.c_str(), out.length()));
+  }
+
+  // this will overflow because the return data is too big
+  {
+    bufferlist in, out;
+    int rval;
+    ObjectWriteOperation o;
+    o.exec("hello", "write_too_much_return_data", in, &out, &rval);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
+                                  librados::OPERATION_RETURNVEC));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EOVERFLOW, completion->get_return_value());
+    ASSERT_EQ(-EOVERFLOW, rval);
+    ASSERT_EQ("", std::string(out.c_str(), out.length()));
+  }
+}