1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
3 #include "gtest/gtest.h"
5 #include "mds/mdstypes.h"
6 #include "include/buffer.h"
7 #include "include/rbd_types.h"
8 #include "include/rados/librados.hpp"
9 #include "include/stringify.h"
10 #include "include/types.h"
11 #include "global/global_context.h"
12 #include "common/Cond.h"
13 #include "test/librados/test_cxx.h"
14 #include "test/librados/testcase_cxx.h"
15 #include "json_spirit/json_spirit.h"
16 #include "cls/cas/cls_cas_ops.h"
17 #include "cls/cas/cls_cas_internal.h"
19 #include "osd/HitSet.h"
26 #include "cls/cas/cls_cas_client.h"
27 #include "cls/cas/cls_cas_internal.h"
29 using namespace librados
;
31 using std::ostringstream
;
34 typedef RadosTestPP LibRadosTierPP
;
35 typedef RadosTestECPP LibRadosTierECPP
;
37 void flush_evict_all(librados::Rados
& cluster
, librados::IoCtx
& cache_ioctx
)
40 cache_ioctx
.set_namespace(all_nspaces
);
41 for (NObjectIterator it
= cache_ioctx
.nobjects_begin();
42 it
!= cache_ioctx
.nobjects_end(); ++it
) {
43 cache_ioctx
.locator_set_key(it
->get_locator());
44 cache_ioctx
.set_namespace(it
->get_nspace());
46 ObjectReadOperation op
;
48 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
49 cache_ioctx
.aio_operate(
50 it
->get_oid(), completion
, &op
,
51 librados::OPERATION_IGNORE_OVERLAY
, NULL
);
52 completion
->wait_for_complete();
53 completion
->get_return_value();
54 completion
->release();
57 ObjectReadOperation op
;
59 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
60 cache_ioctx
.aio_operate(
61 it
->get_oid(), completion
, &op
,
62 librados::OPERATION_IGNORE_OVERLAY
, NULL
);
63 completion
->wait_for_complete();
64 completion
->get_return_value();
65 completion
->release();
70 static string
_get_required_osd_release(Rados
& cluster
)
73 string cmd
= string("{\"prefix\": \"osd dump\",\"format\":\"json\"}");
75 int r
= cluster
.mon_command(cmd
, inbl
, &outbl
, NULL
);
77 string
outstr(outbl
.c_str(), outbl
.length());
79 if (!json_spirit::read(outstr
, v
)) {
80 cerr
<<" unable to parse json " << outstr
<< std::endl
;
84 json_spirit::Object
& o
= v
.get_obj();
85 for (json_spirit::Object::size_type i
=0; i
<o
.size(); i
++) {
86 json_spirit::Pair
& p
= o
[i
];
87 if (p
.name_
== "require_osd_release") {
88 cout
<< "require_osd_release = " << p
.value_
.get_str() << std::endl
;
89 return p
.value_
.get_str();
92 cerr
<< "didn't find require_osd_release in " << outstr
<< std::endl
;
96 void manifest_set_chunk(Rados
& cluster
, librados::IoCtx
& src_ioctx
,
97 librados::IoCtx
& tgt_ioctx
,
98 uint64_t src_offset
, uint64_t length
,
99 std::string src_oid
, std::string tgt_oid
)
101 ObjectReadOperation op
;
102 op
.set_chunk(src_offset
, length
, src_ioctx
, src_oid
, 0,
103 CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
104 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
105 ASSERT_EQ(0, tgt_ioctx
.aio_operate(tgt_oid
, completion
, &op
,
106 librados::OPERATION_IGNORE_CACHE
, NULL
));
107 completion
->wait_for_complete();
108 ASSERT_EQ(0, completion
->get_return_value());
109 completion
->release();
112 #include "common/ceph_crypto.h"
113 using ceph::crypto::SHA1
;
114 #include "rgw/rgw_common.h"
116 void check_fp_oid_refcount(librados::IoCtx
& ioctx
, std::string foid
, uint64_t count
,
117 std::string fp_algo
= NULL
)
120 int size
= foid
.length();
121 if (fp_algo
== "sha1") {
122 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
123 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
125 sha1_gen
.Update((const unsigned char *)foid
.c_str(), size
);
126 sha1_gen
.Final(fingerprint
);
127 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
128 ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
129 } else if (fp_algo
.empty()) {
130 ioctx
.getxattr(foid
, CHUNK_REFCOUNT_ATTR
, t
);
131 } else if (!fp_algo
.empty()) {
132 ceph_assert(0 == "unrecognized fingerprint algorithm");
137 auto iter
= t
.cbegin();
139 } catch (buffer::error
& err
) {
142 ASSERT_EQ(count
, refs
.count());
145 string
get_fp_oid(string oid
, std::string fp_algo
= NULL
)
147 if (fp_algo
== "sha1") {
148 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
149 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
151 int size
= oid
.length();
152 sha1_gen
.Update((const unsigned char *)oid
.c_str(), size
);
153 sha1_gen
.Final(fingerprint
);
154 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
155 return string(p_str
);
161 void is_intended_refcount_state(librados::IoCtx
& src_ioctx
,
163 librados::IoCtx
& dst_ioctx
,
165 int expected_refcount
)
167 int src_refcount
= 0, dst_refcount
= 0;
169 int r
= dst_ioctx
.getxattr(dst_oid
, CHUNK_REFCOUNT_ATTR
, t
);
175 auto iter
= t
.cbegin();
177 } catch (buffer::error
& err
) {
180 dst_refcount
= refs
.count();
182 for (int tries
= 0; tries
< 10; ++tries
) {
183 r
= cls_cas_references_chunk(src_ioctx
, src_oid
, dst_oid
);
184 if (r
== -ENOENT
|| r
== -ENOLINK
) {
186 } else if (r
== -EBUSY
) {
194 ASSERT_TRUE(src_refcount
>= 0);
195 ASSERT_TRUE(src_refcount
== expected_refcount
);
196 ASSERT_TRUE(src_refcount
<= dst_refcount
);
199 class LibRadosTwoPoolsPP
: public RadosTestPP
202 LibRadosTwoPoolsPP() {};
203 ~LibRadosTwoPoolsPP() override
{};
205 static void SetUpTestCase() {
206 pool_name
= get_temp_pool_name();
207 ASSERT_EQ("", create_one_pool_pp(pool_name
, s_cluster
));
209 static void TearDownTestCase() {
210 ASSERT_EQ(0, destroy_one_pool_pp(pool_name
, s_cluster
));
212 static std::string cache_pool_name
;
214 void SetUp() override
{
215 cache_pool_name
= get_temp_pool_name();
216 ASSERT_EQ(0, s_cluster
.pool_create(cache_pool_name
.c_str()));
217 RadosTestPP::SetUp();
219 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
220 cache_ioctx
.application_enable("rados", true);
221 cache_ioctx
.set_namespace(nspace
);
223 void TearDown() override
{
224 // flush + evict cache
225 flush_evict_all(cluster
, cache_ioctx
);
229 ASSERT_EQ(0, cluster
.mon_command(
230 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
233 ASSERT_EQ(0, cluster
.mon_command(
234 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
235 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
238 // wait for maps to settle before next test
239 cluster
.wait_for_latest_osdmap();
241 RadosTestPP::TearDown();
243 cleanup_default_namespace(cache_ioctx
);
244 cleanup_namespace(cache_ioctx
, nspace
);
247 ASSERT_EQ(0, s_cluster
.pool_delete(cache_pool_name
.c_str()));
249 librados::IoCtx cache_ioctx
;
255 Completions() = default;
256 librados::AioCompletion
* getCompletion() {
257 librados::AioCompletion
* comp
= librados::Rados::aio_create_completion();
258 m_completions
.push_back(comp
);
263 for (auto& comp
: m_completions
) {
269 vector
<librados::AioCompletion
*> m_completions
;
272 Completions completions
;
274 std::string
LibRadosTwoPoolsPP::cache_pool_name
;
276 TEST_F(LibRadosTierPP
, Dirty
) {
278 ObjectWriteOperation op
;
280 ASSERT_EQ(0, ioctx
.operate("foo", &op
)); // still get 0 if it dne
283 ObjectWriteOperation op
;
285 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
290 ObjectReadOperation op
;
291 op
.is_dirty(&dirty
, &r
);
292 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
297 ObjectWriteOperation op
;
299 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
302 ObjectWriteOperation op
;
304 ASSERT_EQ(0, ioctx
.operate("foo", &op
)); // still 0 if already clean
309 ObjectReadOperation op
;
310 op
.is_dirty(&dirty
, &r
);
311 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
316 ObjectWriteOperation op
;
317 op
.truncate(0); // still a write even tho it is a no-op
318 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
323 ObjectReadOperation op
;
324 op
.is_dirty(&dirty
, &r
);
325 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
331 TEST_F(LibRadosTwoPoolsPP
, Overlay
) {
336 ObjectWriteOperation op
;
338 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
343 ObjectWriteOperation op
;
345 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
350 ASSERT_EQ(0, cluster
.mon_command(
351 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
352 "\", \"tierpool\": \"" + cache_pool_name
+
353 "\", \"force_nonempty\": \"--force-nonempty\" }",
355 ASSERT_EQ(0, cluster
.mon_command(
356 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
357 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
360 // wait for maps to settle
361 cluster
.wait_for_latest_osdmap();
363 // by default, the overlay sends us to cache pool
366 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
367 ASSERT_EQ('c', bl
[0]);
371 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
372 ASSERT_EQ('c', bl
[0]);
375 // unless we say otherwise
378 ObjectReadOperation op
;
379 op
.read(0, 1, &bl
, NULL
);
380 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
381 ASSERT_EQ(0, ioctx
.aio_operate(
382 "foo", completion
, &op
,
383 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
384 completion
->wait_for_complete();
385 ASSERT_EQ(0, completion
->get_return_value());
386 completion
->release();
387 ASSERT_EQ('b', bl
[0]);
391 TEST_F(LibRadosTwoPoolsPP
, Promote
) {
395 bl
.append("hi there");
396 ObjectWriteOperation op
;
398 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
403 ASSERT_EQ(0, cluster
.mon_command(
404 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
405 "\", \"tierpool\": \"" + cache_pool_name
+
406 "\", \"force_nonempty\": \"--force-nonempty\" }",
408 ASSERT_EQ(0, cluster
.mon_command(
409 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
410 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
412 ASSERT_EQ(0, cluster
.mon_command(
413 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
414 "\", \"mode\": \"writeback\"}",
417 // wait for maps to settle
418 cluster
.wait_for_latest_osdmap();
420 // read, trigger a promote
423 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
426 // read, trigger a whiteout
429 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
430 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
433 // verify the object is present in the cache tier
435 NObjectIterator it
= cache_ioctx
.nobjects_begin();
436 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
437 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
439 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
441 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
445 TEST_F(LibRadosTwoPoolsPP
, PromoteSnap
) {
449 bl
.append("hi there");
450 ObjectWriteOperation op
;
452 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
456 bl
.append("hi there");
457 ObjectWriteOperation op
;
459 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
463 bl
.append("hi there");
464 ObjectWriteOperation op
;
466 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
470 bl
.append("hi there");
471 ObjectWriteOperation op
;
473 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
476 // create a snapshot, clone
477 vector
<uint64_t> my_snaps(1);
478 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
479 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
484 ObjectWriteOperation op
;
486 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
491 ObjectWriteOperation op
;
493 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
496 ObjectWriteOperation op
;
498 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
503 ObjectWriteOperation op
;
505 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
510 ASSERT_EQ(0, cluster
.mon_command(
511 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
512 "\", \"tierpool\": \"" + cache_pool_name
+
513 "\", \"force_nonempty\": \"--force-nonempty\" }",
515 ASSERT_EQ(0, cluster
.mon_command(
516 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
517 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
519 ASSERT_EQ(0, cluster
.mon_command(
520 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
521 "\", \"mode\": \"writeback\"}",
524 // wait for maps to settle
525 cluster
.wait_for_latest_osdmap();
527 // read, trigger a promote on the head
530 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
531 ASSERT_EQ('c', bl
[0]);
535 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
536 ASSERT_EQ('c', bl
[0]);
539 ioctx
.snap_set_read(my_snaps
[0]);
544 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
545 ASSERT_EQ('h', bl
[0]);
551 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
552 ASSERT_EQ('h', bl
[0]);
558 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
559 ASSERT_EQ('h', bl
[0]);
562 ioctx
.snap_set_read(librados::SNAP_HEAD
);
567 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
568 ASSERT_EQ('c', bl
[0]);
574 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
575 ASSERT_EQ('c', bl
[0]);
581 ASSERT_EQ(-ENOENT
, ioctx
.read("baz", bl
, 1, 0));
585 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
588 TEST_F(LibRadosTwoPoolsPP
, PromoteSnapScrub
) {
592 for (int i
=0; i
<num
; ++i
) {
594 bl
.append("hi there");
595 ObjectWriteOperation op
;
597 ASSERT_EQ(0, ioctx
.operate(string("foo") + stringify(i
), &op
));
600 vector
<uint64_t> my_snaps
;
601 for (int snap
=0; snap
<4; ++snap
) {
602 // create a snapshot, clone
603 vector
<uint64_t> ns(1);
604 ns
.insert(ns
.end(), my_snaps
.begin(), my_snaps
.end());
606 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
607 cout
<< "my_snaps " << my_snaps
<< std::endl
;
608 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
610 for (int i
=0; i
<num
; ++i
) {
612 bl
.append(string("ciao! snap") + stringify(snap
));
613 ObjectWriteOperation op
;
615 ASSERT_EQ(0, ioctx
.operate(string("foo") + stringify(i
), &op
));
621 ASSERT_EQ(0, cluster
.mon_command(
622 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
623 "\", \"tierpool\": \"" + cache_pool_name
+
624 "\", \"force_nonempty\": \"--force-nonempty\" }",
626 ASSERT_EQ(0, cluster
.mon_command(
627 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
628 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
630 ASSERT_EQ(0, cluster
.mon_command(
631 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
632 "\", \"mode\": \"writeback\"}",
635 // wait for maps to settle
636 cluster
.wait_for_latest_osdmap();
638 // read, trigger a promote on _some_ heads to make sure we handle cases
639 // where snaps are present and where they are not.
640 cout
<< "promoting some heads" << std::endl
;
641 for (int i
=0; i
<num
; ++i
) {
642 if (i
% 5 == 0 || i
> num
- 3) {
644 ASSERT_EQ(1, ioctx
.read(string("foo") + stringify(i
), bl
, 1, 0));
645 ASSERT_EQ('c', bl
[0]);
649 for (unsigned snap
= 0; snap
< my_snaps
.size(); ++snap
) {
650 cout
<< "promoting from clones for snap " << my_snaps
[snap
] << std::endl
;
651 ioctx
.snap_set_read(my_snaps
[snap
]);
653 // read some snaps, semi-randomly
654 for (int i
=0; i
<50; ++i
) {
656 string o
= string("foo") + stringify((snap
* i
* 137) % 80);
657 //cout << o << std::endl;
658 ASSERT_EQ(1, ioctx
.read(o
, bl
, 1, 0));
662 // ok, stop and scrub this pool (to make sure scrub can handle
663 // missing clones in the cache tier).
666 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
667 for (int i
=0; i
<10; ++i
) {
670 ss
<< "{\"prefix\": \"pg scrub\", \"pgid\": \""
671 << cache_ioctx
.get_id() << "." << i
673 int r
= cluster
.mon_command(ss
.str(), inbl
, NULL
, NULL
);
674 if (r
== -ENOENT
|| // in case mgr osdmap is stale
682 // give it a few seconds to go. this is sloppy but is usually enough time
683 cout
<< "waiting for scrubs..." << std::endl
;
685 cout
<< "done waiting" << std::endl
;
688 ioctx
.snap_set_read(librados::SNAP_HEAD
);
691 for (unsigned snap
= 0; snap
< my_snaps
.size(); ++snap
) {
692 ioctx
.selfmanaged_snap_remove(my_snaps
[snap
]);
696 TEST_F(LibRadosTwoPoolsPP
, PromoteSnapTrimRace
) {
700 bl
.append("hi there");
701 ObjectWriteOperation op
;
703 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
706 // create a snapshot, clone
707 vector
<uint64_t> my_snaps(1);
708 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
709 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
714 ObjectWriteOperation op
;
716 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
721 ASSERT_EQ(0, cluster
.mon_command(
722 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
723 "\", \"tierpool\": \"" + cache_pool_name
+
724 "\", \"force_nonempty\": \"--force-nonempty\" }",
726 ASSERT_EQ(0, cluster
.mon_command(
727 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
728 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
730 ASSERT_EQ(0, cluster
.mon_command(
731 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
732 "\", \"mode\": \"writeback\"}",
735 // wait for maps to settle
736 cluster
.wait_for_latest_osdmap();
739 ASSERT_EQ(0, ioctx
.selfmanaged_snap_remove(my_snaps
[0]));
741 ioctx
.snap_set_read(my_snaps
[0]);
743 // read foo snap. the OSD may or may not realize that this snap has
744 // been logically deleted; either response is valid.
747 int r
= ioctx
.read("foo", bl
, 1, 0);
748 ASSERT_TRUE(r
== 1 || r
== -ENOENT
);
752 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
755 TEST_F(LibRadosTwoPoolsPP
, Whiteout
) {
759 bl
.append("hi there");
760 ObjectWriteOperation op
;
762 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
767 ASSERT_EQ(0, cluster
.mon_command(
768 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
769 "\", \"tierpool\": \"" + cache_pool_name
+
770 "\", \"force_nonempty\": \"--force-nonempty\" }",
772 ASSERT_EQ(0, cluster
.mon_command(
773 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
774 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
776 ASSERT_EQ(0, cluster
.mon_command(
777 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
778 "\", \"mode\": \"writeback\"}",
781 // wait for maps to settle
782 cluster
.wait_for_latest_osdmap();
784 // create some whiteouts, verify they behave
786 ObjectWriteOperation op
;
789 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
793 ObjectWriteOperation op
;
796 ASSERT_EQ(-ENOENT
, ioctx
.operate("bar", &op
));
799 ObjectWriteOperation op
;
802 ASSERT_EQ(-ENOENT
, ioctx
.operate("bar", &op
));
805 // verify the whiteouts are there in the cache tier
807 NObjectIterator it
= cache_ioctx
.nobjects_begin();
808 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
809 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
811 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
813 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
816 // delete a whiteout and verify it goes away
817 ASSERT_EQ(-ENOENT
, ioctx
.remove("foo"));
819 ObjectWriteOperation op
;
821 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
822 ASSERT_EQ(0, ioctx
.aio_operate("bar", completion
, &op
,
823 librados::OPERATION_IGNORE_CACHE
));
824 completion
->wait_for_complete();
825 ASSERT_EQ(0, completion
->get_return_value());
826 completion
->release();
828 NObjectIterator it
= cache_ioctx
.nobjects_begin();
829 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
830 ASSERT_TRUE(it
->get_oid() == string("foo"));
832 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
835 // recreate an object and verify we can read it
838 bl
.append("hi there");
839 ObjectWriteOperation op
;
841 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
845 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
846 ASSERT_EQ('h', bl
[0]);
850 TEST_F(LibRadosTwoPoolsPP
, WhiteoutDeleteCreate
) {
853 ASSERT_EQ(0, cluster
.mon_command(
854 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
855 "\", \"tierpool\": \"" + cache_pool_name
+
856 "\", \"force_nonempty\": \"--force-nonempty\" }",
858 ASSERT_EQ(0, cluster
.mon_command(
859 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
860 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
862 ASSERT_EQ(0, cluster
.mon_command(
863 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
864 "\", \"mode\": \"writeback\"}",
867 // wait for maps to settle
868 cluster
.wait_for_latest_osdmap();
874 ASSERT_EQ(0, ioctx
.write_full("foo", bl
));
877 // do delete + create operation
879 ObjectWriteOperation op
;
884 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
887 // verify it still "exists" (w/ new content)
890 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
891 ASSERT_EQ('b', bl
[0]);
895 TEST_F(LibRadosTwoPoolsPP
, Evict
) {
899 bl
.append("hi there");
900 ObjectWriteOperation op
;
902 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
907 ASSERT_EQ(0, cluster
.mon_command(
908 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
909 "\", \"tierpool\": \"" + cache_pool_name
+
910 "\", \"force_nonempty\": \"--force-nonempty\" }",
912 ASSERT_EQ(0, cluster
.mon_command(
913 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
914 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
916 ASSERT_EQ(0, cluster
.mon_command(
917 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
918 "\", \"mode\": \"writeback\"}",
921 // wait for maps to settle
922 cluster
.wait_for_latest_osdmap();
924 // read, trigger a promote
927 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
930 // read, trigger a whiteout, and a dirty object
933 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
934 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
935 ASSERT_EQ(0, ioctx
.write("bar", bl
, bl
.length(), 0));
938 // verify the object is present in the cache tier
940 NObjectIterator it
= cache_ioctx
.nobjects_begin();
941 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
942 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
944 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
946 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
951 ObjectWriteOperation op
;
953 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
954 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
955 completion
->wait_for_complete();
956 ASSERT_EQ(0, completion
->get_return_value());
957 completion
->release();
960 // evict the pinned object with -EPERM
962 ObjectReadOperation op
;
964 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
965 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
966 librados::OPERATION_IGNORE_CACHE
,
968 completion
->wait_for_complete();
969 ASSERT_EQ(-EPERM
, completion
->get_return_value());
970 completion
->release();
975 ObjectWriteOperation op
;
977 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
978 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
979 completion
->wait_for_complete();
980 ASSERT_EQ(0, completion
->get_return_value());
981 completion
->release();
986 ObjectReadOperation op
;
988 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
989 ASSERT_EQ(0, cache_ioctx
.aio_operate(
990 "foo", completion
, &op
,
991 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
992 completion
->wait_for_complete();
993 ASSERT_EQ(0, completion
->get_return_value());
994 completion
->release();
1001 ObjectReadOperation op
;
1002 op
.is_dirty(&dirty
, &r
);
1003 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1004 ASSERT_FALSE(dirty
);
1010 ObjectReadOperation op
;
1012 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1013 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
1014 librados::OPERATION_IGNORE_CACHE
,
1016 completion
->wait_for_complete();
1017 ASSERT_EQ(0, completion
->get_return_value());
1018 completion
->release();
1021 ObjectReadOperation op
;
1023 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1024 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1025 "foo", completion
, &op
,
1026 librados::OPERATION_IGNORE_CACHE
, NULL
));
1027 completion
->wait_for_complete();
1028 ASSERT_EQ(0, completion
->get_return_value());
1029 completion
->release();
1032 ObjectReadOperation op
;
1034 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1035 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1036 "bar", completion
, &op
,
1037 librados::OPERATION_IGNORE_CACHE
, NULL
));
1038 completion
->wait_for_complete();
1039 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
1040 completion
->release();
1044 TEST_F(LibRadosTwoPoolsPP
, EvictSnap
) {
1048 bl
.append("hi there");
1049 ObjectWriteOperation op
;
1051 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1055 bl
.append("hi there");
1056 ObjectWriteOperation op
;
1058 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
1062 bl
.append("hi there");
1063 ObjectWriteOperation op
;
1065 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
1069 bl
.append("hi there");
1070 ObjectWriteOperation op
;
1072 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
1075 // create a snapshot, clone
1076 vector
<uint64_t> my_snaps(1);
1077 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
1078 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
1083 ObjectWriteOperation op
;
1085 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1090 ObjectWriteOperation op
;
1092 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
1095 ObjectWriteOperation op
;
1097 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
1102 ObjectWriteOperation op
;
1104 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
1109 ASSERT_EQ(0, cluster
.mon_command(
1110 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1111 "\", \"tierpool\": \"" + cache_pool_name
+
1112 "\", \"force_nonempty\": \"--force-nonempty\" }",
1114 ASSERT_EQ(0, cluster
.mon_command(
1115 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1116 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1118 ASSERT_EQ(0, cluster
.mon_command(
1119 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1120 "\", \"mode\": \"writeback\"}",
1123 // wait for maps to settle
1124 cluster
.wait_for_latest_osdmap();
1126 // read, trigger a promote on the head
1129 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1130 ASSERT_EQ('c', bl
[0]);
1134 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
1135 ASSERT_EQ('c', bl
[0]);
1140 ObjectReadOperation op
;
1142 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1143 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1144 "bam", completion
, &op
,
1145 librados::OPERATION_IGNORE_CACHE
, NULL
));
1146 completion
->wait_for_complete();
1147 ASSERT_EQ(0, completion
->get_return_value());
1148 completion
->release();
1152 ObjectReadOperation op
;
1153 op
.read(1, 0, &bl
, NULL
);
1154 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1155 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1156 "bam", completion
, &op
,
1157 librados::OPERATION_IGNORE_CACHE
, NULL
));
1158 completion
->wait_for_complete();
1159 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
1160 completion
->release();
1164 ioctx
.snap_set_read(my_snaps
[0]);
1167 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1168 ASSERT_EQ('h', bl
[0]);
1173 ObjectReadOperation op
;
1175 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1176 ASSERT_EQ(0, ioctx
.aio_operate(
1177 "foo", completion
, &op
,
1178 librados::OPERATION_IGNORE_CACHE
, NULL
));
1179 completion
->wait_for_complete();
1180 ASSERT_EQ(0, completion
->get_return_value());
1181 completion
->release();
1186 ObjectReadOperation op
;
1187 op
.read(1, 0, &bl
, NULL
);
1188 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1189 ASSERT_EQ(0, ioctx
.aio_operate(
1190 "foo", completion
, &op
,
1191 librados::OPERATION_IGNORE_CACHE
, NULL
));
1192 completion
->wait_for_complete();
1193 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
1194 completion
->release();
1196 // head is still there...
1197 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1200 ObjectReadOperation op
;
1201 op
.read(1, 0, &bl
, NULL
);
1202 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1203 ASSERT_EQ(0, ioctx
.aio_operate(
1204 "foo", completion
, &op
,
1205 librados::OPERATION_IGNORE_CACHE
, NULL
));
1206 completion
->wait_for_complete();
1207 ASSERT_EQ(0, completion
->get_return_value());
1208 completion
->release();
1211 // promote head + snap of bar
1212 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1215 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
1216 ASSERT_EQ('c', bl
[0]);
1218 ioctx
.snap_set_read(my_snaps
[0]);
1221 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
1222 ASSERT_EQ('h', bl
[0]);
1225 // evict bar head (fail)
1226 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1228 ObjectReadOperation op
;
1230 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1231 ASSERT_EQ(0, ioctx
.aio_operate(
1232 "bar", completion
, &op
,
1233 librados::OPERATION_IGNORE_CACHE
, NULL
));
1234 completion
->wait_for_complete();
1235 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
1236 completion
->release();
1240 ioctx
.snap_set_read(my_snaps
[0]);
1242 ObjectReadOperation op
;
1244 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1245 ASSERT_EQ(0, ioctx
.aio_operate(
1246 "bar", completion
, &op
,
1247 librados::OPERATION_IGNORE_CACHE
, NULL
));
1248 completion
->wait_for_complete();
1249 ASSERT_EQ(0, completion
->get_return_value());
1250 completion
->release();
1253 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1256 ObjectReadOperation op
;
1257 op
.read(1, 0, &bl
, NULL
);
1258 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1259 ASSERT_EQ(0, ioctx
.aio_operate(
1260 "bar", completion
, &op
,
1261 librados::OPERATION_IGNORE_CACHE
, NULL
));
1262 completion
->wait_for_complete();
1263 ASSERT_EQ(0, completion
->get_return_value());
1264 completion
->release();
1267 ObjectReadOperation op
;
1269 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1270 ASSERT_EQ(0, ioctx
.aio_operate(
1271 "bar", completion
, &op
,
1272 librados::OPERATION_IGNORE_CACHE
, NULL
));
1273 completion
->wait_for_complete();
1274 ASSERT_EQ(0, completion
->get_return_value());
1275 completion
->release();
1279 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
1282 // this test case reproduces http://tracker.ceph.com/issues/8629
1283 TEST_F(LibRadosTwoPoolsPP
, EvictSnap2
) {
1287 bl
.append("hi there");
1288 ObjectWriteOperation op
;
1290 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1292 // create a snapshot, clone
1293 vector
<uint64_t> my_snaps(1);
1294 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
1295 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
1300 ObjectWriteOperation op
;
1302 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1306 ASSERT_EQ(0, cluster
.mon_command(
1307 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1308 "\", \"tierpool\": \"" + cache_pool_name
+
1309 "\", \"force_nonempty\": \"--force-nonempty\" }",
1311 ASSERT_EQ(0, cluster
.mon_command(
1312 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1313 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1315 ASSERT_EQ(0, cluster
.mon_command(
1316 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1317 "\", \"mode\": \"writeback\"}",
1320 // wait for maps to settle
1321 cluster
.wait_for_latest_osdmap();
1323 // read, trigger a promote on the head
1326 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1327 ASSERT_EQ('c', bl
[0]);
1332 ObjectReadOperation op
;
1334 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1335 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1336 "foo", completion
, &op
,
1337 librados::OPERATION_IGNORE_CACHE
, NULL
));
1338 completion
->wait_for_complete();
1339 ASSERT_EQ(0, completion
->get_return_value());
1340 completion
->release();
1343 // verify the snapdir is not present in the cache pool
1345 ObjectReadOperation op
;
1346 librados::snap_set_t snapset
;
1347 op
.list_snaps(&snapset
, NULL
);
1348 ioctx
.snap_set_read(librados::SNAP_DIR
);
1349 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1350 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
,
1351 librados::OPERATION_IGNORE_CACHE
, NULL
));
1352 completion
->wait_for_complete();
1353 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
1354 completion
->release();
1358 //This test case reproduces http://tracker.ceph.com/issues/17445
1359 TEST_F(LibRadosTwoPoolsPP
, ListSnap
){
1363 bl
.append("hi there");
1364 ObjectWriteOperation op
;
1366 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1370 bl
.append("hi there");
1371 ObjectWriteOperation op
;
1373 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
1377 bl
.append("hi there");
1378 ObjectWriteOperation op
;
1380 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
1384 bl
.append("hi there");
1385 ObjectWriteOperation op
;
1387 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
1390 // Create a snapshot, clone
1391 vector
<uint64_t> my_snaps(1);
1392 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
1393 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
1398 ObjectWriteOperation op
;
1400 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1405 ObjectWriteOperation op
;
1407 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
1410 ObjectWriteOperation op
;
1412 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
1417 ObjectWriteOperation op
;
1419 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
1424 ASSERT_EQ(0, cluster
.mon_command(
1425 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1426 "\", \"tierpool\": \"" + cache_pool_name
+
1427 "\", \"force_nonempty\": \"--force-nonempty\" }",
1429 ASSERT_EQ(0, cluster
.mon_command(
1430 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1431 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1433 ASSERT_EQ(0, cluster
.mon_command(
1434 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1435 "\", \"mode\": \"writeback\"}",
1438 // Wait for maps to settle
1439 cluster
.wait_for_latest_osdmap();
1441 // Read, trigger a promote on the head
1444 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1445 ASSERT_EQ('c', bl
[0]);
1449 ioctx
.snap_set_read(my_snaps
[0]);
1452 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1453 ASSERT_EQ('h', bl
[0]);
1458 ObjectReadOperation op
;
1460 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1461 ASSERT_EQ(0, ioctx
.aio_operate(
1462 "foo", completion
, &op
,
1463 librados::OPERATION_IGNORE_CACHE
, NULL
));
1464 completion
->wait_for_complete();
1465 ASSERT_EQ(0, completion
->get_return_value());
1466 completion
->release();
1471 ObjectReadOperation op
;
1472 op
.read(1, 0, &bl
, NULL
);
1473 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1474 ASSERT_EQ(0, ioctx
.aio_operate(
1475 "foo", completion
, &op
,
1476 librados::OPERATION_IGNORE_CACHE
, NULL
));
1477 completion
->wait_for_complete();
1478 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
1479 completion
->release();
1483 ioctx
.snap_set_read(CEPH_SNAPDIR
);
1485 snap_set_t snap_set
;
1487 ObjectReadOperation op
;
1488 op
.list_snaps(&snap_set
, &snap_ret
);
1489 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1490 ASSERT_EQ(0, ioctx
.aio_operate(
1491 "foo", completion
, &op
,
1493 completion
->wait_for_complete();
1494 ASSERT_EQ(0, snap_ret
);
1495 ASSERT_LT(0u, snap_set
.clones
.size());
1496 for (vector
<librados::clone_info_t
>::const_iterator r
= snap_set
.clones
.begin();
1497 r
!= snap_set
.clones
.end();
1499 if (r
->cloneid
!= librados::SNAP_HEAD
) {
1500 ASSERT_LT(0u, r
->snaps
.size());
1506 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
1509 TEST_F(LibRadosTwoPoolsPP
, TryFlush
) {
1512 ASSERT_EQ(0, cluster
.mon_command(
1513 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1514 "\", \"tierpool\": \"" + cache_pool_name
+
1515 "\", \"force_nonempty\": \"--force-nonempty\" }",
1517 ASSERT_EQ(0, cluster
.mon_command(
1518 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1519 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1521 ASSERT_EQ(0, cluster
.mon_command(
1522 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1523 "\", \"mode\": \"writeback\"}",
1526 // wait for maps to settle
1527 cluster
.wait_for_latest_osdmap();
1532 bl
.append("hi there");
1533 ObjectWriteOperation op
;
1535 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1538 // verify the object is present in the cache tier
1540 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1541 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
1542 ASSERT_TRUE(it
->get_oid() == string("foo"));
1544 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1547 // verify the object is NOT present in the base tier
1549 NObjectIterator it
= ioctx
.nobjects_begin();
1550 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1557 ObjectReadOperation op
;
1558 op
.is_dirty(&dirty
, &r
);
1559 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1566 ObjectWriteOperation op
;
1568 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1569 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
1570 completion
->wait_for_complete();
1571 ASSERT_EQ(0, completion
->get_return_value());
1572 completion
->release();
1575 // flush the pinned object with -EPERM
1577 ObjectReadOperation op
;
1578 op
.cache_try_flush();
1579 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1580 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1581 "foo", completion
, &op
,
1582 librados::OPERATION_IGNORE_OVERLAY
|
1583 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1584 completion
->wait_for_complete();
1585 ASSERT_EQ(-EPERM
, completion
->get_return_value());
1586 completion
->release();
1591 ObjectWriteOperation op
;
1593 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1594 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
1595 completion
->wait_for_complete();
1596 ASSERT_EQ(0, completion
->get_return_value());
1597 completion
->release();
1602 ObjectReadOperation op
;
1603 op
.cache_try_flush();
1604 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1605 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1606 "foo", completion
, &op
,
1607 librados::OPERATION_IGNORE_OVERLAY
|
1608 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1609 completion
->wait_for_complete();
1610 ASSERT_EQ(0, completion
->get_return_value());
1611 completion
->release();
1618 ObjectReadOperation op
;
1619 op
.is_dirty(&dirty
, &r
);
1620 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1621 ASSERT_FALSE(dirty
);
1625 // verify in base tier
1627 NObjectIterator it
= ioctx
.nobjects_begin();
1628 ASSERT_TRUE(it
!= ioctx
.nobjects_end());
1629 ASSERT_TRUE(it
->get_oid() == string("foo"));
1631 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1636 ObjectReadOperation op
;
1638 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1639 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1640 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
1641 completion
->wait_for_complete();
1642 ASSERT_EQ(0, completion
->get_return_value());
1643 completion
->release();
1646 // verify no longer in cache tier
1648 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1649 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1653 TEST_F(LibRadosTwoPoolsPP
, Flush
) {
1656 ASSERT_EQ(0, cluster
.mon_command(
1657 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1658 "\", \"tierpool\": \"" + cache_pool_name
+
1659 "\", \"force_nonempty\": \"--force-nonempty\" }",
1661 ASSERT_EQ(0, cluster
.mon_command(
1662 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1663 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1665 ASSERT_EQ(0, cluster
.mon_command(
1666 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1667 "\", \"mode\": \"writeback\"}",
1670 // wait for maps to settle
1671 cluster
.wait_for_latest_osdmap();
1673 uint64_t user_version
= 0;
1678 bl
.append("hi there");
1679 ObjectWriteOperation op
;
1681 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1684 // verify the object is present in the cache tier
1686 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1687 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
1688 ASSERT_TRUE(it
->get_oid() == string("foo"));
1690 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1693 // verify the object is NOT present in the base tier
1695 NObjectIterator it
= ioctx
.nobjects_begin();
1696 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1703 ObjectReadOperation op
;
1704 op
.is_dirty(&dirty
, &r
);
1705 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1708 user_version
= cache_ioctx
.get_last_version();
1713 ObjectWriteOperation op
;
1715 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1716 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
1717 completion
->wait_for_complete();
1718 ASSERT_EQ(0, completion
->get_return_value());
1719 completion
->release();
1722 // flush the pinned object with -EPERM
1724 ObjectReadOperation op
;
1725 op
.cache_try_flush();
1726 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1727 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1728 "foo", completion
, &op
,
1729 librados::OPERATION_IGNORE_OVERLAY
|
1730 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1731 completion
->wait_for_complete();
1732 ASSERT_EQ(-EPERM
, completion
->get_return_value());
1733 completion
->release();
1738 ObjectWriteOperation op
;
1740 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1741 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
1742 completion
->wait_for_complete();
1743 ASSERT_EQ(0, completion
->get_return_value());
1744 completion
->release();
1749 ObjectReadOperation op
;
1751 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1752 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1753 "foo", completion
, &op
,
1754 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1755 completion
->wait_for_complete();
1756 ASSERT_EQ(0, completion
->get_return_value());
1757 completion
->release();
1764 ObjectReadOperation op
;
1765 op
.is_dirty(&dirty
, &r
);
1766 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1767 ASSERT_FALSE(dirty
);
1771 // verify in base tier
1773 NObjectIterator it
= ioctx
.nobjects_begin();
1774 ASSERT_TRUE(it
!= ioctx
.nobjects_end());
1775 ASSERT_TRUE(it
->get_oid() == string("foo"));
1777 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1782 ObjectReadOperation op
;
1784 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1785 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1786 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
1787 completion
->wait_for_complete();
1788 ASSERT_EQ(0, completion
->get_return_value());
1789 completion
->release();
1792 // verify no longer in cache tier
1794 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1795 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1798 // read it again and verify the version is consistent
1801 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
1802 ASSERT_EQ(user_version
, cache_ioctx
.get_last_version());
1807 ObjectWriteOperation op
;
1809 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1814 ObjectReadOperation op
;
1816 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1817 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1818 "foo", completion
, &op
,
1819 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1820 completion
->wait_for_complete();
1821 ASSERT_EQ(0, completion
->get_return_value());
1822 completion
->release();
1827 ObjectReadOperation op
;
1829 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1830 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1831 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
1832 completion
->wait_for_complete();
1833 ASSERT_EQ(0, completion
->get_return_value());
1834 completion
->release();
1837 // verify no longer in cache tier
1839 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1840 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1844 NObjectIterator it
= ioctx
.nobjects_begin();
1845 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1849 TEST_F(LibRadosTwoPoolsPP
, FlushSnap
) {
1852 ASSERT_EQ(0, cluster
.mon_command(
1853 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1854 "\", \"tierpool\": \"" + cache_pool_name
+
1855 "\", \"force_nonempty\": \"--force-nonempty\" }",
1857 ASSERT_EQ(0, cluster
.mon_command(
1858 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1859 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1861 ASSERT_EQ(0, cluster
.mon_command(
1862 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1863 "\", \"mode\": \"writeback\"}",
1866 // wait for maps to settle
1867 cluster
.wait_for_latest_osdmap();
1873 ObjectWriteOperation op
;
1875 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1878 // create a snapshot, clone
1879 vector
<uint64_t> my_snaps(1);
1880 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
1881 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
1886 ObjectWriteOperation op
;
1888 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1893 my_snaps
[1] = my_snaps
[0];
1894 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
1895 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
1900 ObjectWriteOperation op
;
1902 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1905 // verify the object is present in the cache tier
1907 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1908 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
1909 ASSERT_TRUE(it
->get_oid() == string("foo"));
1911 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1914 // verify the object is NOT present in the base tier
1916 NObjectIterator it
= ioctx
.nobjects_begin();
1917 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1920 // flush on head (should fail)
1921 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1923 ObjectReadOperation op
;
1925 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1926 ASSERT_EQ(0, ioctx
.aio_operate(
1927 "foo", completion
, &op
,
1928 librados::OPERATION_IGNORE_CACHE
, NULL
));
1929 completion
->wait_for_complete();
1930 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
1931 completion
->release();
1933 // flush on recent snap (should fail)
1934 ioctx
.snap_set_read(my_snaps
[0]);
1936 ObjectReadOperation op
;
1938 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1939 ASSERT_EQ(0, ioctx
.aio_operate(
1940 "foo", completion
, &op
,
1941 librados::OPERATION_IGNORE_CACHE
, NULL
));
1942 completion
->wait_for_complete();
1943 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
1944 completion
->release();
1946 // flush on oldest snap
1947 ioctx
.snap_set_read(my_snaps
[1]);
1949 ObjectReadOperation op
;
1951 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1952 ASSERT_EQ(0, ioctx
.aio_operate(
1953 "foo", completion
, &op
,
1954 librados::OPERATION_IGNORE_CACHE
, NULL
));
1955 completion
->wait_for_complete();
1956 ASSERT_EQ(0, completion
->get_return_value());
1957 completion
->release();
1959 // flush on next oldest snap
1960 ioctx
.snap_set_read(my_snaps
[0]);
1962 ObjectReadOperation op
;
1964 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1965 ASSERT_EQ(0, ioctx
.aio_operate(
1966 "foo", completion
, &op
,
1967 librados::OPERATION_IGNORE_CACHE
, NULL
));
1968 completion
->wait_for_complete();
1969 ASSERT_EQ(0, completion
->get_return_value());
1970 completion
->release();
1973 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1975 ObjectReadOperation op
;
1977 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1978 ASSERT_EQ(0, ioctx
.aio_operate(
1979 "foo", completion
, &op
,
1980 librados::OPERATION_IGNORE_CACHE
, NULL
));
1981 completion
->wait_for_complete();
1982 ASSERT_EQ(0, completion
->get_return_value());
1983 completion
->release();
1986 // verify i can read the snaps from the cache pool
1987 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1990 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1991 ASSERT_EQ('c', bl
[0]);
1993 ioctx
.snap_set_read(my_snaps
[0]);
1996 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1997 ASSERT_EQ('b', bl
[0]);
1999 ioctx
.snap_set_read(my_snaps
[1]);
2002 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2003 ASSERT_EQ('a', bl
[0]);
2007 ASSERT_EQ(0, cluster
.mon_command(
2008 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
2012 // wait for maps to settle
2013 cluster
.wait_for_latest_osdmap();
2015 // verify i can read the snaps from the base pool
2016 ioctx
.snap_set_read(librados::SNAP_HEAD
);
2019 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2020 ASSERT_EQ('c', bl
[0]);
2022 ioctx
.snap_set_read(my_snaps
[0]);
2025 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2026 ASSERT_EQ('b', bl
[0]);
2028 ioctx
.snap_set_read(my_snaps
[1]);
2031 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2032 ASSERT_EQ('a', bl
[0]);
2035 ASSERT_EQ(0, cluster
.mon_command(
2036 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2037 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2041 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
2044 TEST_F(LibRadosTierPP
, FlushWriteRaces
) {
2046 std::string pool_name
= get_temp_pool_name();
2047 std::string cache_pool_name
= pool_name
+ "-cache";
2048 ASSERT_EQ("", create_one_pool_pp(pool_name
, cluster
));
2049 ASSERT_EQ(0, cluster
.pool_create(cache_pool_name
.c_str()));
2051 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
2052 cache_ioctx
.application_enable("rados", true);
2054 ASSERT_EQ(0, cluster
.ioctx_create(pool_name
.c_str(), ioctx
));
2058 ASSERT_EQ(0, cluster
.mon_command(
2059 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2060 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2062 ASSERT_EQ(0, cluster
.mon_command(
2063 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2064 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2066 ASSERT_EQ(0, cluster
.mon_command(
2067 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2068 "\", \"mode\": \"writeback\"}",
2071 // wait for maps to settle
2072 cluster
.wait_for_latest_osdmap();
2074 // create/dirty object
2076 bl
.append("hi there");
2078 ObjectWriteOperation op
;
2080 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2085 ObjectReadOperation op
;
2087 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2088 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2089 "foo", completion
, &op
,
2090 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
2092 ObjectWriteOperation op2
;
2094 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
2095 ASSERT_EQ(0, ioctx
.aio_operate(
2096 "foo", completion2
, &op2
, 0));
2098 completion
->wait_for_complete();
2099 completion2
->wait_for_complete();
2100 ASSERT_EQ(0, completion
->get_return_value());
2101 ASSERT_EQ(0, completion2
->get_return_value());
2102 completion
->release();
2103 completion2
->release();
2108 // create/dirty object
2111 bl
.append("hi there");
2112 ObjectWriteOperation op
;
2114 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2117 // try-flush + write
2119 ObjectReadOperation op
;
2120 op
.cache_try_flush();
2121 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2122 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2123 "foo", completion
, &op
,
2124 librados::OPERATION_IGNORE_OVERLAY
|
2125 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2127 ObjectWriteOperation op2
;
2129 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
2130 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion2
, &op2
, 0));
2132 completion
->wait_for_complete();
2133 completion2
->wait_for_complete();
2134 int r
= completion
->get_return_value();
2135 ASSERT_TRUE(r
== -EBUSY
|| r
== 0);
2136 ASSERT_EQ(0, completion2
->get_return_value());
2137 completion
->release();
2138 completion2
->release();
2141 cout
<< "didn't get EBUSY, trying again" << std::endl
;
2143 ASSERT_TRUE(--tries
);
2147 ASSERT_EQ(0, cluster
.mon_command(
2148 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
2151 ASSERT_EQ(0, cluster
.mon_command(
2152 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
2153 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2156 // wait for maps to settle before next test
2157 cluster
.wait_for_latest_osdmap();
2159 ASSERT_EQ(0, cluster
.pool_delete(cache_pool_name
.c_str()));
2160 ASSERT_EQ(0, destroy_one_pool_pp(pool_name
, cluster
));
2163 TEST_F(LibRadosTwoPoolsPP
, FlushTryFlushRaces
) {
2166 ASSERT_EQ(0, cluster
.mon_command(
2167 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2168 "\", \"tierpool\": \"" + cache_pool_name
+
2169 "\", \"force_nonempty\": \"--force-nonempty\" }",
2171 ASSERT_EQ(0, cluster
.mon_command(
2172 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2173 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2175 ASSERT_EQ(0, cluster
.mon_command(
2176 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2177 "\", \"mode\": \"writeback\"}",
2180 // wait for maps to settle
2181 cluster
.wait_for_latest_osdmap();
2183 // create/dirty object
2186 bl
.append("hi there");
2187 ObjectWriteOperation op
;
2189 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2194 ObjectReadOperation op
;
2196 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2197 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2198 "foo", completion
, &op
,
2199 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
2201 ObjectReadOperation op2
;
2203 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
2204 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2205 "foo", completion2
, &op2
,
2206 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
2208 completion
->wait_for_complete();
2209 completion2
->wait_for_complete();
2210 ASSERT_EQ(0, completion
->get_return_value());
2211 ASSERT_EQ(0, completion2
->get_return_value());
2212 completion
->release();
2213 completion2
->release();
2216 // create/dirty object
2219 bl
.append("hi there");
2220 ObjectWriteOperation op
;
2222 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2225 // flush + try-flush
2227 ObjectReadOperation op
;
2229 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2230 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2231 "foo", completion
, &op
,
2232 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
2234 ObjectReadOperation op2
;
2235 op2
.cache_try_flush();
2236 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
2237 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2238 "foo", completion2
, &op2
,
2239 librados::OPERATION_IGNORE_OVERLAY
|
2240 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2242 completion
->wait_for_complete();
2243 completion2
->wait_for_complete();
2244 ASSERT_EQ(0, completion
->get_return_value());
2245 ASSERT_EQ(0, completion2
->get_return_value());
2246 completion
->release();
2247 completion2
->release();
2250 // create/dirty object
2255 bl
.append("hi there");
2256 ObjectWriteOperation op
;
2258 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2261 // try-flush + flush
2262 // (flush will not piggyback on try-flush)
2264 ObjectReadOperation op
;
2265 op
.cache_try_flush();
2266 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2267 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2268 "foo", completion
, &op
,
2269 librados::OPERATION_IGNORE_OVERLAY
|
2270 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2272 ObjectReadOperation op2
;
2274 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
2275 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2276 "foo", completion2
, &op2
,
2277 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
2279 completion
->wait_for_complete();
2280 completion2
->wait_for_complete();
2281 int r
= completion
->get_return_value();
2282 ASSERT_TRUE(r
== -EBUSY
|| r
== 0);
2283 ASSERT_EQ(0, completion2
->get_return_value());
2284 completion
->release();
2285 completion2
->release();
2288 cout
<< "didn't get EBUSY, trying again" << std::endl
;
2290 ASSERT_TRUE(--tries
);
2293 // create/dirty object
2296 bl
.append("hi there");
2297 ObjectWriteOperation op
;
2299 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2302 // try-flush + try-flush
2304 ObjectReadOperation op
;
2305 op
.cache_try_flush();
2306 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2307 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2308 "foo", completion
, &op
,
2309 librados::OPERATION_IGNORE_OVERLAY
|
2310 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2312 ObjectReadOperation op2
;
2313 op2
.cache_try_flush();
2314 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
2315 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2316 "foo", completion2
, &op2
,
2317 librados::OPERATION_IGNORE_OVERLAY
|
2318 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2320 completion
->wait_for_complete();
2321 completion2
->wait_for_complete();
2322 ASSERT_EQ(0, completion
->get_return_value());
2323 ASSERT_EQ(0, completion2
->get_return_value());
2324 completion
->release();
2325 completion2
->release();
2330 IoCtx
*read_ioctx
= 0;
2331 ceph::mutex test_lock
= ceph::make_mutex("FlushReadRaces::lock");
2332 ceph::condition_variable cond
;
2333 int max_reads
= 100;
2334 int num_reads
= 0; // in progress
2336 void flush_read_race_cb(completion_t cb
, void *arg
);
2338 void start_flush_read()
2340 //cout << " starting read" << std::endl;
2341 ObjectReadOperation op
;
2342 op
.stat(NULL
, NULL
, NULL
);
2343 librados::AioCompletion
*completion
= completions
.getCompletion();
2344 completion
->set_complete_callback(0, flush_read_race_cb
);
2345 read_ioctx
->aio_operate("foo", completion
, &op
, NULL
);
2348 void flush_read_race_cb(completion_t cb
, void *arg
)
2350 //cout << " finished read" << std::endl;
2351 std::lock_guard l
{test_lock
};
2352 if (num_reads
> max_reads
) {
2360 TEST_F(LibRadosTwoPoolsPP
, TryFlushReadRace
) {
2363 ASSERT_EQ(0, cluster
.mon_command(
2364 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2365 "\", \"tierpool\": \"" + cache_pool_name
+
2366 "\", \"force_nonempty\": \"--force-nonempty\" }",
2368 ASSERT_EQ(0, cluster
.mon_command(
2369 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2370 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2372 ASSERT_EQ(0, cluster
.mon_command(
2373 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2374 "\", \"mode\": \"writeback\"}",
2377 // wait for maps to settle
2378 cluster
.wait_for_latest_osdmap();
2380 // create/dirty object
2383 bl
.append("hi there");
2384 bufferptr
bp(4000000); // make it big!
2387 ObjectWriteOperation op
;
2389 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2392 // start a continuous stream of reads
2393 read_ioctx
= &ioctx
;
2395 for (int i
= 0; i
< max_reads
; ++i
) {
2402 ObjectReadOperation op
;
2403 op
.cache_try_flush();
2404 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2405 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2406 "foo", completion
, &op
,
2407 librados::OPERATION_IGNORE_OVERLAY
|
2408 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2410 completion
->wait_for_complete();
2411 ASSERT_EQ(0, completion
->get_return_value());
2412 completion
->release();
2415 std::unique_lock locker
{test_lock
};
2417 cond
.wait(locker
, [] { return num_reads
== 0;});
2420 TEST_F(LibRadosTierPP
, HitSetNone
) {
2422 list
< pair
<time_t,time_t> > ls
;
2423 AioCompletion
*c
= librados::Rados::aio_create_completion();
2424 ASSERT_EQ(0, ioctx
.hit_set_list(123, c
, &ls
));
2425 c
->wait_for_complete();
2426 ASSERT_EQ(0, c
->get_return_value());
2427 ASSERT_TRUE(ls
.empty());
2432 AioCompletion
*c
= librados::Rados::aio_create_completion();
2433 ASSERT_EQ(0, ioctx
.hit_set_get(123, c
, 12345, &bl
));
2434 c
->wait_for_complete();
2435 ASSERT_EQ(-ENOENT
, c
->get_return_value());
2440 string
set_pool_str(string pool
, string var
, string val
)
2442 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2443 + string("\",\"var\": \"") + var
+ string("\",\"val\": \"")
2444 + val
+ string("\"}");
2447 string
set_pool_str(string pool
, string var
, int val
)
2449 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2450 + string("\",\"var\": \"") + var
+ string("\",\"val\": \"")
2451 + stringify(val
) + string("\"}");
2454 TEST_F(LibRadosTwoPoolsPP
, HitSetRead
) {
2457 ASSERT_EQ(0, cluster
.mon_command(
2458 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2459 "\", \"tierpool\": \"" + cache_pool_name
+
2460 "\", \"force_nonempty\": \"--force-nonempty\" }",
2463 // enable hitset tracking for this pool
2464 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", 2),
2466 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", 600),
2468 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type",
2472 // wait for maps to settle
2473 cluster
.wait_for_latest_osdmap();
2475 cache_ioctx
.set_namespace("");
2477 // keep reading until we see our object appear in the HitSet
2478 utime_t start
= ceph_clock_now();
2479 utime_t hard_stop
= start
+ utime_t(600, 0);
2482 utime_t now
= ceph_clock_now();
2483 ASSERT_TRUE(now
< hard_stop
);
2485 string name
= "foo";
2487 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(name
, &hash
));
2488 hobject_t
oid(sobject_t(name
, CEPH_NOSNAP
), "", hash
,
2489 cluster
.pool_lookup(cache_pool_name
.c_str()), "");
2492 ASSERT_EQ(-ENOENT
, cache_ioctx
.read("foo", bl
, 1, 0));
2495 AioCompletion
*c
= librados::Rados::aio_create_completion();
2496 ASSERT_EQ(0, cache_ioctx
.hit_set_get(hash
, c
, now
.sec(), &hbl
));
2497 c
->wait_for_complete();
2501 auto p
= hbl
.cbegin();
2504 if (hs
.contains(oid
)) {
2505 cout
<< "ok, hit_set contains " << oid
<< std::endl
;
2508 cout
<< "hmm, not in HitSet yet" << std::endl
;
2510 cout
<< "hmm, no HitSet yet" << std::endl
;
2517 static int _get_pg_num(Rados
& cluster
, string pool_name
)
2520 string cmd
= string("{\"prefix\": \"osd pool get\",\"pool\":\"")
2522 + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
2524 int r
= cluster
.mon_command(cmd
, inbl
, &outbl
, NULL
);
2525 ceph_assert(r
>= 0);
2526 string
outstr(outbl
.c_str(), outbl
.length());
2527 json_spirit::Value v
;
2528 if (!json_spirit::read(outstr
, v
)) {
2529 cerr
<<" unable to parse json " << outstr
<< std::endl
;
2533 json_spirit::Object
& o
= v
.get_obj();
2534 for (json_spirit::Object::size_type i
=0; i
<o
.size(); i
++) {
2535 json_spirit::Pair
& p
= o
[i
];
2536 if (p
.name_
== "pg_num") {
2537 cout
<< "pg_num = " << p
.value_
.get_int() << std::endl
;
2538 return p
.value_
.get_int();
2541 cerr
<< "didn't find pg_num in " << outstr
<< std::endl
;
2545 TEST_F(LibRadosTwoPoolsPP
, HitSetWrite
) {
2546 int num_pg
= _get_pg_num(cluster
, pool_name
);
2547 ceph_assert(num_pg
> 0);
2551 ASSERT_EQ(0, cluster
.mon_command(
2552 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2553 "\", \"tierpool\": \"" + cache_pool_name
+
2554 "\", \"force_nonempty\": \"--force-nonempty\" }",
2557 // enable hitset tracking for this pool
2558 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", 8),
2560 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", 600),
2562 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type",
2566 // wait for maps to settle
2567 cluster
.wait_for_latest_osdmap();
2569 cache_ioctx
.set_namespace("");
2573 // do a bunch of writes
2574 for (int i
=0; i
<num
; ++i
) {
2577 ASSERT_EQ(0, cache_ioctx
.write(stringify(i
), bl
, 1, 0));
2581 std::map
<int,HitSet
> hitsets
;
2582 for (int i
=0; i
<num_pg
; ++i
) {
2583 list
< pair
<time_t,time_t> > ls
;
2584 AioCompletion
*c
= librados::Rados::aio_create_completion();
2585 ASSERT_EQ(0, cache_ioctx
.hit_set_list(i
, c
, &ls
));
2586 c
->wait_for_complete();
2588 std::cout
<< "pg " << i
<< " ls " << ls
<< std::endl
;
2589 ASSERT_FALSE(ls
.empty());
2592 c
= librados::Rados::aio_create_completion();
2594 ASSERT_EQ(0, cache_ioctx
.hit_set_get(i
, c
, ls
.back().first
, &bl
));
2595 c
->wait_for_complete();
2599 auto p
= bl
.cbegin();
2600 decode(hitsets
[i
], p
);
2602 catch (buffer::error
& e
) {
2603 std::cout
<< "failed to decode hit set; bl len is " << bl
.length() << "\n";
2604 bl
.hexdump(std::cout
);
2605 std::cout
<< std::endl
;
2609 // cope with racing splits by refreshing pg_num
2610 if (i
== num_pg
- 1)
2611 num_pg
= _get_pg_num(cluster
, cache_pool_name
);
2614 for (int i
=0; i
<num
; ++i
) {
2615 string n
= stringify(i
);
2617 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(n
, &hash
));
2618 hobject_t
oid(sobject_t(n
, CEPH_NOSNAP
), "", hash
,
2619 cluster
.pool_lookup(cache_pool_name
.c_str()), "");
2620 std::cout
<< "checking for " << oid
<< std::endl
;
2622 for (int p
=0; p
<num_pg
; ++p
) {
2623 if (hitsets
[p
].contains(oid
)) {
2632 TEST_F(LibRadosTwoPoolsPP
, HitSetTrim
) {
2634 unsigned period
= 3;
2638 ASSERT_EQ(0, cluster
.mon_command(
2639 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2640 "\", \"tierpool\": \"" + cache_pool_name
+
2641 "\", \"force_nonempty\": \"--force-nonempty\" }",
2644 // enable hitset tracking for this pool
2645 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", count
),
2647 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", period
),
2649 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
2651 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_fpp", ".01"),
2654 // wait for maps to settle
2655 cluster
.wait_for_latest_osdmap();
2657 cache_ioctx
.set_namespace("");
2659 // do a bunch of writes and make sure the hitsets rotate
2660 utime_t start
= ceph_clock_now();
2661 utime_t hard_stop
= start
+ utime_t(count
* period
* 50, 0);
2665 string name
= "foo";
2667 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(name
, &hash
));
2668 hobject_t
oid(sobject_t(name
, CEPH_NOSNAP
), "", hash
, -1, "");
2672 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, 1, 0));
2674 list
<pair
<time_t, time_t> > ls
;
2675 AioCompletion
*c
= librados::Rados::aio_create_completion();
2676 ASSERT_EQ(0, cache_ioctx
.hit_set_list(hash
, c
, &ls
));
2677 c
->wait_for_complete();
2680 cout
<< " got ls " << ls
<< std::endl
;
2683 first
= ls
.front().first
;
2684 cout
<< "first is " << first
<< std::endl
;
2686 if (ls
.front().first
!= first
) {
2687 cout
<< "first now " << ls
.front().first
<< ", trimmed" << std::endl
;
2693 utime_t now
= ceph_clock_now();
2694 ASSERT_TRUE(now
< hard_stop
);
2700 TEST_F(LibRadosTwoPoolsPP
, PromoteOn2ndRead
) {
2702 for (int i
=0; i
<20; ++i
) {
2704 bl
.append("hi there");
2705 ObjectWriteOperation op
;
2707 ASSERT_EQ(0, ioctx
.operate("foo" + stringify(i
), &op
));
2712 ASSERT_EQ(0, cluster
.mon_command(
2713 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2714 "\", \"tierpool\": \"" + cache_pool_name
+
2715 "\", \"force_nonempty\": \"--force-nonempty\" }",
2717 ASSERT_EQ(0, cluster
.mon_command(
2718 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2719 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2721 ASSERT_EQ(0, cluster
.mon_command(
2722 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2723 "\", \"mode\": \"writeback\"}",
2726 // enable hitset tracking for this pool
2727 ASSERT_EQ(0, cluster
.mon_command(
2728 set_pool_str(cache_pool_name
, "hit_set_count", 2),
2730 ASSERT_EQ(0, cluster
.mon_command(
2731 set_pool_str(cache_pool_name
, "hit_set_period", 600),
2733 ASSERT_EQ(0, cluster
.mon_command(
2734 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
2736 ASSERT_EQ(0, cluster
.mon_command(
2737 set_pool_str(cache_pool_name
, "min_read_recency_for_promote", 1),
2739 ASSERT_EQ(0, cluster
.mon_command(
2740 set_pool_str(cache_pool_name
, "hit_set_grade_decay_rate", 20),
2742 ASSERT_EQ(0, cluster
.mon_command(
2743 set_pool_str(cache_pool_name
, "hit_set_search_last_n", 1),
2746 // wait for maps to settle
2747 cluster
.wait_for_latest_osdmap();
2749 int fake
= 0; // set this to non-zero to test spurious promotion,
2750 // e.g. from thrashing
2754 // 1st read, don't trigger a promote
2755 obj
= "foo" + stringify(attempt
);
2756 cout
<< obj
<< std::endl
;
2759 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
2762 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
2767 // verify the object is NOT present in the cache tier
2770 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2771 while (it
!= cache_ioctx
.nobjects_end()) {
2772 cout
<< " see " << it
->get_oid() << std::endl
;
2773 if (it
->get_oid() == string(obj
.c_str())) {
2784 ASSERT_LE(attempt
, 20);
2785 cout
<< "hrm, object is present in cache on attempt " << attempt
2786 << ", retrying" << std::endl
;
2789 // Read until the object is present in the cache tier
2790 cout
<< "verifying " << obj
<< " is eventually promoted" << std::endl
;
2793 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
2796 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2797 while (it
!= cache_ioctx
.nobjects_end()) {
2798 if (it
->get_oid() == string(obj
.c_str())) {
2811 ASSERT_EQ(0, cluster
.mon_command(
2812 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
2815 ASSERT_EQ(0, cluster
.mon_command(
2816 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
2817 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2820 // wait for maps to settle before next test
2821 cluster
.wait_for_latest_osdmap();
2824 TEST_F(LibRadosTwoPoolsPP
, ProxyRead
) {
2828 bl
.append("hi there");
2829 ObjectWriteOperation op
;
2831 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2836 ASSERT_EQ(0, cluster
.mon_command(
2837 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2838 "\", \"tierpool\": \"" + cache_pool_name
+
2839 "\", \"force_nonempty\": \"--force-nonempty\" }",
2841 ASSERT_EQ(0, cluster
.mon_command(
2842 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2843 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2845 ASSERT_EQ(0, cluster
.mon_command(
2846 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2847 "\", \"mode\": \"readproxy\"}",
2850 // wait for maps to settle
2851 cluster
.wait_for_latest_osdmap();
2853 // read and verify the object
2856 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2857 ASSERT_EQ('h', bl
[0]);
2860 // Verify 10 times the object is NOT present in the cache tier
2863 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2864 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
2869 ASSERT_EQ(0, cluster
.mon_command(
2870 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
2873 ASSERT_EQ(0, cluster
.mon_command(
2874 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
2875 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2878 // wait for maps to settle before next test
2879 cluster
.wait_for_latest_osdmap();
2882 TEST_F(LibRadosTwoPoolsPP
, CachePin
) {
2886 bl
.append("hi there");
2887 ObjectWriteOperation op
;
2889 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2893 bl
.append("hi there");
2894 ObjectWriteOperation op
;
2896 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
2900 bl
.append("hi there");
2901 ObjectWriteOperation op
;
2903 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
2907 bl
.append("hi there");
2908 ObjectWriteOperation op
;
2910 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
2915 ASSERT_EQ(0, cluster
.mon_command(
2916 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2917 "\", \"tierpool\": \"" + cache_pool_name
+
2918 "\", \"force_nonempty\": \"--force-nonempty\" }",
2920 ASSERT_EQ(0, cluster
.mon_command(
2921 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2922 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2924 ASSERT_EQ(0, cluster
.mon_command(
2925 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2926 "\", \"mode\": \"writeback\"}",
2929 // wait for maps to settle
2930 cluster
.wait_for_latest_osdmap();
2932 // read, trigger promote
2935 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2936 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
2937 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
2938 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
2941 // verify the objects are present in the cache tier
2943 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2944 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
2945 for (uint32_t i
= 0; i
< 4; i
++) {
2946 ASSERT_TRUE(it
->get_oid() == string("foo") ||
2947 it
->get_oid() == string("bar") ||
2948 it
->get_oid() == string("baz") ||
2949 it
->get_oid() == string("bam"));
2952 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
2957 ObjectWriteOperation op
;
2959 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2960 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
2961 completion
->wait_for_complete();
2962 ASSERT_EQ(0, completion
->get_return_value());
2963 completion
->release();
2966 ObjectWriteOperation op
;
2968 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2969 ASSERT_EQ(0, cache_ioctx
.aio_operate("baz", completion
, &op
));
2970 completion
->wait_for_complete();
2971 ASSERT_EQ(0, completion
->get_return_value());
2972 completion
->release();
2976 ASSERT_EQ(0, cluster
.mon_command(
2977 set_pool_str(cache_pool_name
, "hit_set_count", 2),
2979 ASSERT_EQ(0, cluster
.mon_command(
2980 set_pool_str(cache_pool_name
, "hit_set_period", 600),
2982 ASSERT_EQ(0, cluster
.mon_command(
2983 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
2985 ASSERT_EQ(0, cluster
.mon_command(
2986 set_pool_str(cache_pool_name
, "min_read_recency_for_promote", 1),
2988 ASSERT_EQ(0, cluster
.mon_command(
2989 set_pool_str(cache_pool_name
, "target_max_objects", 1),
2994 // Verify the pinned object 'foo' is not flushed/evicted
2998 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
3001 NObjectIterator it
= cache_ioctx
.nobjects_begin();
3002 while (it
!= cache_ioctx
.nobjects_end()) {
3003 ASSERT_TRUE(it
->get_oid() == string("foo") ||
3004 it
->get_oid() == string("bar") ||
3005 it
->get_oid() == string("baz") ||
3006 it
->get_oid() == string("bam"));
3011 ASSERT_TRUE(it
->get_oid() == string("foo") ||
3012 it
->get_oid() == string("baz"));
3020 ASSERT_EQ(0, cluster
.mon_command(
3021 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
3024 ASSERT_EQ(0, cluster
.mon_command(
3025 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
3026 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
3029 // wait for maps to settle before next test
3030 cluster
.wait_for_latest_osdmap();
3033 TEST_F(LibRadosTwoPoolsPP
, SetRedirectRead
) {
3037 bl
.append("hi there");
3038 ObjectWriteOperation op
;
3040 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3045 ObjectWriteOperation op
;
3047 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
3050 // wait for maps to settle
3051 cluster
.wait_for_latest_osdmap();
3054 ObjectWriteOperation op
;
3055 op
.set_redirect("bar", cache_ioctx
, 0);
3056 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3057 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
3058 completion
->wait_for_complete();
3059 ASSERT_EQ(0, completion
->get_return_value());
3060 completion
->release();
3062 // read and verify the object
3065 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3066 ASSERT_EQ('t', bl
[0]);
3069 // wait for maps to settle before next test
3070 cluster
.wait_for_latest_osdmap();
3073 TEST_F(LibRadosTwoPoolsPP
, ManifestPromoteRead
) {
3074 // skip test if not yet mimic
3075 if (_get_required_osd_release(cluster
) < "mimic") {
3076 GTEST_SKIP() << "cluster is not yet mimic, skipping test";
3082 bl
.append("hi there");
3083 ObjectWriteOperation op
;
3085 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3089 bl
.append("base chunk");
3090 ObjectWriteOperation op
;
3092 ASSERT_EQ(0, ioctx
.operate("foo-chunk", &op
));
3097 ObjectWriteOperation op
;
3099 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
3104 ObjectWriteOperation op
;
3106 ASSERT_EQ(0, cache_ioctx
.operate("bar-chunk", &op
));
3109 // wait for maps to settle
3110 cluster
.wait_for_latest_osdmap();
3114 ObjectWriteOperation op
;
3115 op
.set_redirect("bar", cache_ioctx
, 0);
3116 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3117 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
3118 completion
->wait_for_complete();
3119 ASSERT_EQ(0, completion
->get_return_value());
3120 completion
->release();
3123 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 0, 2, "bar-chunk", "foo-chunk");
3127 ObjectWriteOperation op
;
3129 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3130 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
3131 completion
->wait_for_complete();
3132 ASSERT_EQ(0, completion
->get_return_value());
3133 completion
->release();
3135 // read and verify the object (redirect)
3138 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3139 ASSERT_EQ('t', bl
[0]);
3143 ObjectWriteOperation op
;
3145 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3146 ASSERT_EQ(0, ioctx
.aio_operate("foo-chunk", completion
, &op
));
3147 completion
->wait_for_complete();
3148 ASSERT_EQ(0, completion
->get_return_value());
3149 completion
->release();
3151 // read and verify the object
3154 ASSERT_EQ(1, ioctx
.read("foo-chunk", bl
, 1, 0));
3155 ASSERT_EQ('C', bl
[0]);
3158 // wait for maps to settle before next test
3159 cluster
.wait_for_latest_osdmap();
3162 TEST_F(LibRadosTwoPoolsPP
, ManifestRefRead
) {
3163 // note: require >= mimic
3168 bl
.append("hi there");
3169 ObjectWriteOperation op
;
3171 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3175 bl
.append("base chunk");
3176 ObjectWriteOperation op
;
3178 ASSERT_EQ(0, ioctx
.operate("foo-chunk", &op
));
3183 ObjectWriteOperation op
;
3185 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
3190 ObjectWriteOperation op
;
3192 ASSERT_EQ(0, cache_ioctx
.operate("bar-chunk", &op
));
3195 // wait for maps to settle
3196 cluster
.wait_for_latest_osdmap();
3200 ObjectWriteOperation op
;
3201 op
.set_redirect("bar", cache_ioctx
, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
3202 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3203 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
3204 completion
->wait_for_complete();
3205 ASSERT_EQ(0, completion
->get_return_value());
3206 completion
->release();
3210 ObjectReadOperation op
;
3211 op
.set_chunk(0, 2, cache_ioctx
, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
3212 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3213 ASSERT_EQ(0, ioctx
.aio_operate("foo-chunk", completion
, &op
,
3214 librados::OPERATION_IGNORE_CACHE
, NULL
));
3215 completion
->wait_for_complete();
3216 ASSERT_EQ(0, completion
->get_return_value());
3217 completion
->release();
3219 // redirect's refcount
3222 cache_ioctx
.getxattr("bar", CHUNK_REFCOUNT_ATTR
, t
);
3225 auto iter
= t
.cbegin();
3227 } catch (buffer::error
& err
) {
3230 ASSERT_EQ(1U, refs
.count());
3235 cache_ioctx
.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR
, t
);
3238 auto iter
= t
.cbegin();
3240 } catch (buffer::error
& err
) {
3243 ASSERT_EQ(1u, refs
.count());
3246 // wait for maps to settle before next test
3247 cluster
.wait_for_latest_osdmap();
3250 TEST_F(LibRadosTwoPoolsPP
, ManifestUnset
) {
3251 // skip test if not yet nautilus
3252 if (_get_required_osd_release(cluster
) < "nautilus") {
3253 GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
3259 bl
.append("hi there");
3260 ObjectWriteOperation op
;
3262 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3266 bl
.append("base chunk");
3267 ObjectWriteOperation op
;
3269 ASSERT_EQ(0, ioctx
.operate("foo-chunk", &op
));
3274 ObjectWriteOperation op
;
3276 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
3281 ObjectWriteOperation op
;
3283 ASSERT_EQ(0, cache_ioctx
.operate("bar-chunk", &op
));
3286 // wait for maps to settle
3287 cluster
.wait_for_latest_osdmap();
3291 ObjectWriteOperation op
;
3292 op
.set_redirect("bar", cache_ioctx
, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
3293 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3294 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
3295 completion
->wait_for_complete();
3296 ASSERT_EQ(0, completion
->get_return_value());
3297 completion
->release();
3301 ObjectReadOperation op
;
3302 op
.set_chunk(0, 2, cache_ioctx
, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
3303 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3304 ASSERT_EQ(0, ioctx
.aio_operate("foo-chunk", completion
, &op
,
3305 librados::OPERATION_IGNORE_CACHE
, NULL
));
3306 completion
->wait_for_complete();
3307 ASSERT_EQ(0, completion
->get_return_value());
3308 completion
->release();
3310 // redirect's refcount
3313 cache_ioctx
.getxattr("bar", CHUNK_REFCOUNT_ATTR
, t
);
3316 auto iter
= t
.cbegin();
3318 } catch (buffer::error
& err
) {
3321 ASSERT_EQ(1u, refs
.count());
3326 cache_ioctx
.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR
, t
);
3329 auto iter
= t
.cbegin();
3331 } catch (buffer::error
& err
) {
3334 ASSERT_EQ(1u, refs
.count());
3337 // unset-manifest for set-redirect
3339 ObjectWriteOperation op
;
3340 op
.unset_manifest();
3341 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3342 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
3343 completion
->wait_for_complete();
3344 ASSERT_EQ(0, completion
->get_return_value());
3345 completion
->release();
3348 // unset-manifest for set-chunk
3350 ObjectWriteOperation op
;
3351 op
.unset_manifest();
3352 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3353 ASSERT_EQ(0, ioctx
.aio_operate("foo-chunk", completion
, &op
));
3354 completion
->wait_for_complete();
3355 ASSERT_EQ(0, completion
->get_return_value());
3356 completion
->release();
3358 // redirect's refcount
3361 cache_ioctx
.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR
, t
);
3362 if (t
.length() != 0U) {
3363 ObjectWriteOperation op
;
3364 op
.unset_manifest();
3365 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3366 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
3367 completion
->wait_for_complete();
3368 ASSERT_EQ(-EOPNOTSUPP
, completion
->get_return_value());
3369 completion
->release();
3375 cache_ioctx
.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR
, t
);
3376 if (t
.length() != 0U) {
3377 ObjectWriteOperation op
;
3378 op
.unset_manifest();
3379 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3380 ASSERT_EQ(0, ioctx
.aio_operate("foo-chunk", completion
, &op
));
3381 completion
->wait_for_complete();
3382 ASSERT_EQ(-EOPNOTSUPP
, completion
->get_return_value());
3383 completion
->release();
3387 // wait for maps to settle before next test
3388 cluster
.wait_for_latest_osdmap();
3391 #include "common/ceph_crypto.h"
3392 using ceph::crypto::SHA1
;
3393 #include "rgw/rgw_common.h"
3394 TEST_F(LibRadosTwoPoolsPP
, ManifestDedupRefRead
) {
3395 // skip test if not yet nautilus
3396 if (_get_required_osd_release(cluster
) < "nautilus") {
3397 GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
3401 ASSERT_EQ(0, cluster
.mon_command(
3402 set_pool_str(pool_name
, "fingerprint_algorithm", "sha1"),
3404 cluster
.wait_for_latest_osdmap();
3408 tgt_oid
= get_fp_oid("There hi", "sha1");
3413 bl
.append("There hi");
3414 ObjectWriteOperation op
;
3416 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3420 bl
.append("There hi");
3421 ObjectWriteOperation op
;
3423 ASSERT_EQ(0, ioctx
.operate("foo-dedup", &op
));
3428 ObjectWriteOperation op
;
3430 bl
.append("There hi");
3432 ASSERT_EQ(0, cache_ioctx
.operate(tgt_oid
, &op
));
3435 // set-chunk (dedup)
3436 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 0, 8, tgt_oid
, "foo-dedup");
3437 // set-chunk (dedup)
3438 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 0, 8, tgt_oid
, "foo");
3442 cache_ioctx
.getxattr(tgt_oid
, CHUNK_REFCOUNT_ATTR
, t
);
3445 auto iter
= t
.cbegin();
3447 } catch (buffer::error
& err
) {
3450 ASSERT_EQ(2u, refs
.count());
3453 // wait for maps to settle before next test
3454 cluster
.wait_for_latest_osdmap();
3457 TEST_F(LibRadosTwoPoolsPP
, ManifestSnapRefcount
) {
3458 // skip test if not yet octopus
3459 if (_get_required_osd_release(cluster
) < "octopus") {
3460 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
3465 ASSERT_EQ(0, cluster
.mon_command(
3466 set_pool_str(pool_name
, "fingerprint_algorithm", "sha1"),
3468 cluster
.wait_for_latest_osdmap();
3473 bl
.append("there hi");
3474 ObjectWriteOperation op
;
3476 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3480 bl
.append("there hi");
3481 ObjectWriteOperation op
;
3483 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
3486 // wait for maps to settle
3487 cluster
.wait_for_latest_osdmap();
3489 string er_fp_oid
, hi_fp_oid
, bb_fp_oid
;
3492 er_fp_oid
= get_fp_oid("er", "sha1");
3493 hi_fp_oid
= get_fp_oid("hi", "sha1");
3494 bb_fp_oid
= get_fp_oid("bb", "sha1");
3498 ObjectWriteOperation op
;
3502 ASSERT_EQ(0, cache_ioctx
.operate(er_fp_oid
, &op
));
3506 ObjectWriteOperation op
;
3510 ASSERT_EQ(0, cache_ioctx
.operate(hi_fp_oid
, &op
));
3514 ObjectWriteOperation op
;
3518 ASSERT_EQ(0, cache_ioctx
.operate(bb_fp_oid
, &op
));
3521 // set-chunk (dedup)
3522 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, er_fp_oid
, "foo");
3523 // set-chunk (dedup)
3524 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, hi_fp_oid
, "foo");
3526 // make all chunks dirty --> flush
3529 // check chunk's refcount
3533 int size
= strlen("er");
3534 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3535 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3536 sha1_gen
.Update((const unsigned char *)"er", size
);
3537 sha1_gen
.Final(fingerprint
);
3538 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3539 cache_ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
3542 auto iter
= t
.cbegin();
3544 } catch (buffer::error
& err
) {
3547 ASSERT_EQ(1u, refs
.count());
3550 // create a snapshot, clone
3551 vector
<uint64_t> my_snaps(1);
3552 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
3553 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
3561 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3567 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3569 // set-chunk (dedup)
3570 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, bb_fp_oid
, "foo");
3574 my_snaps
[1] = my_snaps
[0];
3575 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
3576 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
3584 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3590 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3592 // set-chunk (dedup)
3593 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, er_fp_oid
, "foo");
3595 // check chunk's refcount
3599 int size
= strlen("er");
3600 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3601 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3602 sha1_gen
.Update((const unsigned char *)"er", size
);
3603 sha1_gen
.Final(fingerprint
);
3604 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3605 cache_ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
3608 auto iter
= t
.cbegin();
3610 } catch (buffer::error
& err
) {
3613 ASSERT_EQ(2u, refs
.count());
3618 my_snaps
[2] = my_snaps
[1];
3619 my_snaps
[1] = my_snaps
[0];
3620 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
3621 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
3629 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3635 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3637 // set-chunk (dedup)
3638 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, bb_fp_oid
, "foo");
3641 * snap[2]: [er] [hi]
3642 * snap[1]: [bb] [hi]
3643 * snap[0]: [er] [hi]
3647 // check chunk's refcount
3651 int size
= strlen("hi");
3652 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3653 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3654 sha1_gen
.Update((const unsigned char *)"hi", size
);
3655 sha1_gen
.Final(fingerprint
);
3656 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3657 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, p_str
, 1);
3660 // check chunk's refcount
3664 int size
= strlen("er");
3665 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3666 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3667 sha1_gen
.Update((const unsigned char *)"er", size
);
3668 sha1_gen
.Final(fingerprint
);
3669 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3670 cache_ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
3673 auto iter
= t
.cbegin();
3675 } catch (buffer::error
& err
) {
3678 ASSERT_EQ(2u, refs
.count());
3682 ioctx
.selfmanaged_snap_remove(my_snaps
[2]);
3685 * snap[1]: [bb] [hi]
3686 * snap[0]: [er] [hi]
3692 // check chunk's refcount
3696 int size
= strlen("hi");
3697 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3698 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3699 sha1_gen
.Update((const unsigned char *)"hi", size
);
3700 sha1_gen
.Final(fingerprint
);
3701 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3702 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, p_str
, 1);
3706 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
3709 * snap[1]: [bb] [hi]
3715 // check chunk's refcount
3719 int size
= strlen("bb");
3720 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3721 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3722 sha1_gen
.Update((const unsigned char *)"bb", size
);
3723 sha1_gen
.Final(fingerprint
);
3724 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3725 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, p_str
, 1);
3729 ioctx
.selfmanaged_snap_remove(my_snaps
[1]);
3732 * snap[1]: [bb] [hi]
3737 // check chunk's refcount
3741 int size
= strlen("bb");
3742 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3743 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3744 sha1_gen
.Update((const unsigned char *)"bb", size
);
3745 sha1_gen
.Final(fingerprint
);
3746 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3747 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, p_str
, 1);
3750 // check chunk's refcount
3754 int size
= strlen("hi");
3755 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3756 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3757 sha1_gen
.Update((const unsigned char *)"hi", size
);
3758 sha1_gen
.Final(fingerprint
);
3759 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3760 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, p_str
, 1);
3764 TEST_F(LibRadosTwoPoolsPP
, ManifestSnapRefcount2
) {
3765 // skip test if not yet octopus
3766 if (_get_required_osd_release(cluster
) < "octopus") {
3767 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
3772 ASSERT_EQ(0, cluster
.mon_command(
3773 set_pool_str(pool_name
, "fingerprint_algorithm", "sha1"),
3775 cluster
.wait_for_latest_osdmap();
3780 bl
.append("Thabe cdHI");
3781 ObjectWriteOperation op
;
3783 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3787 bl
.append("there hiHI");
3788 ObjectWriteOperation op
;
3790 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
3793 string ab_fp_oid
, cd_fp_oid
, ef_fp_oid
, BB_fp_oid
;
3796 ab_fp_oid
= get_fp_oid("ab", "sha1");
3797 cd_fp_oid
= get_fp_oid("cd", "sha1");
3798 ef_fp_oid
= get_fp_oid("ef", "sha1");
3799 BB_fp_oid
= get_fp_oid("BB", "sha1");
3803 ObjectWriteOperation op
;
3807 ASSERT_EQ(0, cache_ioctx
.operate(ab_fp_oid
, &op
));
3811 ObjectWriteOperation op
;
3815 ASSERT_EQ(0, cache_ioctx
.operate(cd_fp_oid
, &op
));
3819 ObjectWriteOperation op
;
3823 ASSERT_EQ(0, cache_ioctx
.operate(ef_fp_oid
, &op
));
3827 ObjectWriteOperation op
;
3831 ASSERT_EQ(0, cache_ioctx
.operate(BB_fp_oid
, &op
));
3834 // set-chunk (dedup)
3835 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, ab_fp_oid
, "foo");
3836 // set-chunk (dedup)
3837 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, cd_fp_oid
, "foo");
3838 // set-chunk (dedup)
3839 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, ef_fp_oid
, "foo");
3842 // make all chunks dirty --> flush
3843 // foo: [ab] [cd] [ef]
3845 // create a snapshot, clone
3846 vector
<uint64_t> my_snaps(1);
3847 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
3848 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
3851 // foo: [BB] [BB] [ef]
3855 bl
.append("ThBBe BB");
3856 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3861 bl
.append("ThBBe BB");
3862 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3864 // set-chunk (dedup)
3865 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, BB_fp_oid
, "foo");
3866 // set-chunk (dedup)
3867 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, BB_fp_oid
, "foo");
3871 my_snaps
[1] = my_snaps
[0];
3872 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
3873 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
3876 // foo: [ab] [cd] [ef]
3880 bl
.append("Thabe cd");
3881 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3886 bl
.append("Thabe cd");
3887 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
3889 // set-chunk (dedup)
3890 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, ab_fp_oid
, "foo");
3891 // set-chunk (dedup)
3892 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, cd_fp_oid
, "foo");
3895 * snap[1]: [ab] [cd] [ef]
3896 * snap[0]: [BB] [BB] [ef]
3897 * head: [ab] [cd] [ef]
3900 // check chunk's refcount
3904 int size
= strlen("ab");
3905 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3906 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3907 sha1_gen
.Update((const unsigned char *)"ab", size
);
3908 sha1_gen
.Final(fingerprint
);
3909 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3910 cache_ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
3913 auto iter
= t
.cbegin();
3915 } catch (buffer::error
& err
) {
3918 ASSERT_EQ(2u, refs
.count());
3921 // check chunk's refcount
3925 int size
= strlen("cd");
3926 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3927 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3928 sha1_gen
.Update((const unsigned char *)"cd", size
);
3929 sha1_gen
.Final(fingerprint
);
3930 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3931 cache_ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
3934 auto iter
= t
.cbegin();
3936 } catch (buffer::error
& err
) {
3939 ASSERT_EQ(2u, refs
.count());
3942 // check chunk's refcount
3946 int size
= strlen("BB");
3947 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3948 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3949 sha1_gen
.Update((const unsigned char *)"BB", size
);
3950 sha1_gen
.Final(fingerprint
);
3951 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3952 cache_ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
3955 auto iter
= t
.cbegin();
3957 } catch (buffer::error
& err
) {
3960 ASSERT_EQ(2u, refs
.count());
3964 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
3967 * snap[1]: [ab] [cd] [ef]
3968 * head: [ab] [cd] [ef]
3973 // check chunk's refcount
3977 int size
= strlen("BB");
3978 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
3979 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
3980 sha1_gen
.Update((const unsigned char *)"BB", size
);
3981 sha1_gen
.Final(fingerprint
);
3982 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
3983 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, p_str
, 0);
3987 TEST_F(LibRadosTwoPoolsPP
, ManifestTestSnapCreate
) {
3988 // skip test if not yet octopus
3989 if (_get_required_osd_release(cluster
) < "octopus") {
3990 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
3996 bl
.append("base chunk");
3997 ObjectWriteOperation op
;
3999 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4003 bl
.append("CHUNKS CHUNKS");
4004 ObjectWriteOperation op
;
4006 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
4009 string ba_fp_oid
, se_fp_oid
, ch_fp_oid
;
4012 ba_fp_oid
= get_fp_oid("ba", "sha1");
4013 se_fp_oid
= get_fp_oid("se", "sha1");
4014 ch_fp_oid
= get_fp_oid("ch", "sha1");
4018 ObjectWriteOperation op
;
4022 ASSERT_EQ(0, cache_ioctx
.operate(ba_fp_oid
, &op
));
4026 ObjectWriteOperation op
;
4030 ASSERT_EQ(0, cache_ioctx
.operate(se_fp_oid
, &op
));
4034 ObjectWriteOperation op
;
4038 ASSERT_EQ(0, cache_ioctx
.operate(ch_fp_oid
, &op
));
4041 // set-chunk (dedup)
4042 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 0, 2, ba_fp_oid
, "foo");
4044 // try to create a snapshot, clone
4045 vector
<uint64_t> my_snaps(1);
4046 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4047 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4050 // set-chunk (dedup)
4051 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, se_fp_oid
, "foo");
4053 // check whether clone is created
4054 ioctx
.snap_set_read(librados::SNAP_DIR
);
4056 snap_set_t snap_set
;
4058 ObjectReadOperation op
;
4059 op
.list_snaps(&snap_set
, &snap_ret
);
4060 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4061 ASSERT_EQ(0, ioctx
.aio_operate(
4062 "foo", completion
, &op
,
4064 completion
->wait_for_complete();
4065 ASSERT_EQ(0, snap_ret
);
4066 ASSERT_LT(0u, snap_set
.clones
.size());
4067 ASSERT_EQ(1, snap_set
.clones
.size());
4071 ioctx
.snap_set_read(librados::SNAP_HEAD
);
4075 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 0));
4078 ioctx
.snap_set_read(my_snaps
[0]);
4079 // set-chunk to clone
4080 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, ch_fp_oid
, "foo");
4083 TEST_F(LibRadosTwoPoolsPP
, ManifestRedirectAfterPromote
) {
4084 // skip test if not yet octopus
4085 if (_get_required_osd_release(cluster
) < "octopus") {
4086 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4092 bl
.append("base chunk");
4093 ObjectWriteOperation op
;
4095 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4099 bl
.append("BASE CHUNK");
4100 ObjectWriteOperation op
;
4102 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
4107 ObjectWriteOperation op
;
4108 op
.set_redirect("bar", cache_ioctx
, 0);
4109 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4110 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
4111 completion
->wait_for_complete();
4112 ASSERT_EQ(0, completion
->get_return_value());
4113 completion
->release();
4118 ObjectWriteOperation op
;
4120 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4121 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
4122 completion
->wait_for_complete();
4123 ASSERT_EQ(0, completion
->get_return_value());
4124 completion
->release();
4131 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 0));
4134 // read and verify the object (redirect)
4137 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
4138 ASSERT_EQ('a', bl
[0]);
4141 // read and verify the object (redirect)
4144 ASSERT_EQ(1, cache_ioctx
.read("bar", bl
, 1, 0));
4145 ASSERT_EQ('B', bl
[0]);
4149 TEST_F(LibRadosTwoPoolsPP
, ManifestCheckRefcountWhenModification
) {
4150 // skip test if not yet octopus
4151 if (_get_required_osd_release(cluster
) < "octopus") {
4152 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4156 ASSERT_EQ(0, cluster
.mon_command(
4157 set_pool_str(pool_name
, "fingerprint_algorithm", "sha1"),
4159 cluster
.wait_for_latest_osdmap();
4164 bl
.append("there hiHI");
4165 ObjectWriteOperation op
;
4167 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4170 string er_fp_oid
, hi_fp_oid
, HI_fp_oid
, ai_fp_oid
, bi_fp_oid
,
4171 Er_fp_oid
, Hi_fp_oid
, Si_fp_oid
;
4174 er_fp_oid
= get_fp_oid("er", "sha1");
4175 hi_fp_oid
= get_fp_oid("hi", "sha1");
4176 HI_fp_oid
= get_fp_oid("HI", "sha1");
4177 ai_fp_oid
= get_fp_oid("ai", "sha1");
4178 bi_fp_oid
= get_fp_oid("bi", "sha1");
4179 Er_fp_oid
= get_fp_oid("Er", "sha1");
4180 Hi_fp_oid
= get_fp_oid("Hi", "sha1");
4181 Si_fp_oid
= get_fp_oid("Si", "sha1");
4185 ObjectWriteOperation op
;
4189 ASSERT_EQ(0, cache_ioctx
.operate(er_fp_oid
, &op
));
4193 ObjectWriteOperation op
;
4197 ASSERT_EQ(0, cache_ioctx
.operate(hi_fp_oid
, &op
));
4201 ObjectWriteOperation op
;
4205 ASSERT_EQ(0, cache_ioctx
.operate(HI_fp_oid
, &op
));
4209 ObjectWriteOperation op
;
4213 ASSERT_EQ(0, cache_ioctx
.operate(ai_fp_oid
, &op
));
4217 ObjectWriteOperation op
;
4221 ASSERT_EQ(0, cache_ioctx
.operate(bi_fp_oid
, &op
));
4225 ObjectWriteOperation op
;
4229 ASSERT_EQ(0, cache_ioctx
.operate(Er_fp_oid
, &op
));
4233 ObjectWriteOperation op
;
4237 ASSERT_EQ(0, cache_ioctx
.operate(Hi_fp_oid
, &op
));
4241 ObjectWriteOperation op
;
4245 ASSERT_EQ(0, cache_ioctx
.operate(Si_fp_oid
, &op
));
4248 // set-chunk (dedup)
4249 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, er_fp_oid
, "foo");
4250 // set-chunk (dedup)
4251 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, hi_fp_oid
, "foo");
4252 // set-chunk (dedup)
4253 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, HI_fp_oid
, "foo");
4255 // foo head: [er] [hi] [HI]
4257 // create a snapshot, clone
4258 vector
<uint64_t> my_snaps(1);
4259 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4260 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4264 // foo snap[0]: [er] [hi] [HI]
4265 // foo head : [er] [ai] [HI]
4270 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 6));
4276 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 6));
4279 // set-chunk (dedup)
4280 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, ai_fp_oid
, "foo");
4282 // foo snap[0]: [er] [hi] [HI]
4283 // foo head : [er] [bi] [HI]
4288 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 6));
4294 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 6));
4297 // set-chunk (dedup)
4298 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, bi_fp_oid
, "foo");
4302 // check chunk's refcount
4303 // [ai]'s refcount should be 0
4307 int size
= strlen("ai");
4308 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
4309 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
4310 sha1_gen
.Update((const unsigned char *)"ai", size
);
4311 sha1_gen
.Final(fingerprint
);
4312 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
4313 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, p_str
, 0);
4316 // foo snap[0]: [er] [hi] [HI]
4317 // foo head : [Er] [Hi] [Si]
4321 bl
.append("thEre HiSi");
4322 ObjectWriteOperation op
;
4324 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4329 bl
.append("thEre HiSi");
4330 ObjectWriteOperation op
;
4332 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4335 // set-chunk (dedup)
4336 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, Er_fp_oid
, "foo");
4337 // set-chunk (dedup)
4338 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, Hi_fp_oid
, "foo");
4339 // set-chunk (dedup)
4340 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, Si_fp_oid
, "foo");
4342 // foo snap[0]: [er] [hi] [HI]
4343 // foo head : [ER] [HI] [SI]
4347 bl
.append("thERe HISI");
4348 ObjectWriteOperation op
;
4350 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4355 // check chunk's refcount
4356 // [Er]'s refcount should be 0
4360 int size
= strlen("Er");
4361 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1];
4362 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
4363 sha1_gen
.Update((const unsigned char *)"Er", size
);
4364 sha1_gen
.Final(fingerprint
);
4365 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
4366 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, p_str
, 0);
4370 TEST_F(LibRadosTwoPoolsPP
, ManifestSnapIncCount
) {
4371 // skip test if not yet octopus
4372 if (_get_required_osd_release(cluster
) < "octopus") {
4373 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
4380 bl
.append("there hiHI");
4381 ObjectWriteOperation op
;
4383 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4387 bl
.append("there hiHI");
4388 ObjectWriteOperation op
;
4390 ASSERT_EQ(0, cache_ioctx
.operate("chunk1", &op
));
4394 bl
.append("there hiHI");
4395 ObjectWriteOperation op
;
4397 ASSERT_EQ(0, cache_ioctx
.operate("chunk2", &op
));
4401 bl
.append("there hiHI");
4402 ObjectWriteOperation op
;
4404 ASSERT_EQ(0, cache_ioctx
.operate("chunk3", &op
));
4408 bl
.append("there hiHI");
4409 ObjectWriteOperation op
;
4411 ASSERT_EQ(0, cache_ioctx
.operate("chunk4", &op
));
4414 // wait for maps to settle
4415 cluster
.wait_for_latest_osdmap();
4417 // create a snapshot, clone
4418 vector
<uint64_t> my_snaps(1);
4419 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4420 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4425 bl
.append("there hiHI");
4426 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
4430 my_snaps
[1] = my_snaps
[0];
4431 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4432 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4437 bl
.append("there hiHI");
4438 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
4442 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, "chunk1", "foo");
4443 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, "chunk4", "foo");
4446 // foo head : [chunk1] [chunk4]
4448 ioctx
.snap_set_read(my_snaps
[1]);
4450 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, "chunk2", "foo");
4451 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, "chunk4", "foo");
4452 // foo snap[1]: [chunk2] [chunk4]
4454 // foo head : [chunk1] [chunk4]
4456 ioctx
.snap_set_read(my_snaps
[0]);
4458 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, "chunk2", "foo");
4459 // foo snap[1]: [chunk2] [chunk4]
4460 // foo snap[0]: [chunk2]
4461 // foo head : [chunk1] [chunk4]
4463 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, "chunk3", "foo");
4464 // foo snap[1]: [chunk2] [chunk4]
4465 // foo snap[0]: [chunk3] [chunk2]
4466 // foo head : [chunk1] [chunk4]
4467 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, "chunk4", "foo");
4468 // foo snap[1]: [chunk2] [chunk4]
4469 // foo snap[0]: [chunk3] [chunk2] [chunk4]
4470 // foo head : [chunk1] [chunk4]
4472 // check chunk's refcount
4473 check_fp_oid_refcount(cache_ioctx
, "chunk1", 1u, "");
4475 // check chunk's refcount
4476 check_fp_oid_refcount(cache_ioctx
, "chunk2", 1u, "");
4478 // check chunk's refcount
4479 check_fp_oid_refcount(cache_ioctx
, "chunk3", 1u, "");
4482 // check chunk's refcount
4483 is_intended_refcount_state(ioctx
, "foo", cache_ioctx
, "chunk4", 1);
4486 TEST_F(LibRadosTwoPoolsPP
, ManifestEvict
) {
4487 // skip test if not yet octopus
4488 if (_get_required_osd_release(cluster
) < "octopus") {
4489 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
4496 bl
.append("there hiHI");
4497 ObjectWriteOperation op
;
4499 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4503 bl
.append("there hiHI");
4504 ObjectWriteOperation op
;
4506 ASSERT_EQ(0, cache_ioctx
.operate("chunk1", &op
));
4510 bl
.append("there hiHI");
4511 ObjectWriteOperation op
;
4513 ASSERT_EQ(0, cache_ioctx
.operate("chunk2", &op
));
4517 bl
.append("there hiHI");
4518 ObjectWriteOperation op
;
4520 ASSERT_EQ(0, cache_ioctx
.operate("chunk3", &op
));
4524 bl
.append("there hiHI");
4525 ObjectWriteOperation op
;
4527 ASSERT_EQ(0, cache_ioctx
.operate("chunk4", &op
));
4530 // wait for maps to settle
4531 cluster
.wait_for_latest_osdmap();
4533 // create a snapshot, clone
4534 vector
<uint64_t> my_snaps(1);
4535 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4536 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4541 bl
.append("there hiHI");
4542 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
4546 my_snaps
[1] = my_snaps
[0];
4547 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4548 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4553 bl
.append("there hiHI");
4554 ASSERT_EQ(0, ioctx
.write("foo", bl
, bl
.length(), 0));
4558 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, "chunk1", "foo");
4559 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, "chunk4", "foo");
4562 // foo head : [chunk1] [chunk4]
4564 ioctx
.snap_set_read(my_snaps
[1]);
4566 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 0, 10, "chunk2", "foo");
4567 // foo snap[1]: [ chunk2 ]
4569 // foo head : [chunk1] [chunk4]
4571 ioctx
.snap_set_read(my_snaps
[0]);
4573 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, "chunk2", "foo");
4574 // foo snap[1]: [ chunk2 ]
4575 // foo snap[0]: [chunk2]
4576 // foo head : [chunk1] [chunk4]
4578 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, "chunk3", "foo");
4579 // foo snap[1]: [ chunk2 ]
4580 // foo snap[0]: [chunk3] [chunk2]
4581 // foo head : [chunk1] [chunk4]
4582 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, "chunk4", "foo");
4583 // foo snap[1]: [ chunk2 ]
4584 // foo snap[0]: [chunk3] [chunk2] [chunk4]
4585 // foo head : [chunk1] [chunk4]
4586 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 0, 2, "chunk4", "foo");
4587 // foo snap[1]: [ chunk2 ]
4588 // foo snap[0]: [chunk4] [chunk3] [chunk2] [chunk4]
4589 // foo head : [chunk1] [chunk4]
4590 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 4, 2, "chunk1", "foo");
4591 // foo snap[1]: [ chunk2 ]
4592 // foo snap[0]: [chunk4] [chunk3] [chunk1] [chunk2] [chunk4]
4593 // foo head : [chunk1] [chunk4]
4596 ObjectReadOperation op
, stat_op
;
4599 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4600 ASSERT_EQ(0, ioctx
.aio_operate(
4601 "foo", completion
, &op
,
4602 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4603 completion
->wait_for_complete();
4604 ASSERT_EQ(0, completion
->get_return_value());
4606 stat_op
.stat(&size
, NULL
, NULL
);
4607 ASSERT_EQ(0, ioctx
.operate("foo", &stat_op
, NULL
));
4608 ASSERT_EQ(10, size
);
4611 ioctx
.snap_set_read(librados::SNAP_HEAD
);
4613 ObjectReadOperation op
, stat_op
;
4616 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4617 ASSERT_EQ(0, ioctx
.aio_operate(
4618 "foo", completion
, &op
,
4619 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4620 completion
->wait_for_complete();
4621 ASSERT_EQ(0, completion
->get_return_value());
4623 stat_op
.stat(&size
, NULL
, NULL
);
4624 ASSERT_EQ(0, ioctx
.operate("foo", &stat_op
, NULL
));
4625 ASSERT_EQ(strlen("there hiHI"), size
);
4631 TEST_F(LibRadosTwoPoolsPP
, ManifestSnapSizeMismatch
) {
4632 // skip test if not yet octopus
4633 if (_get_required_osd_release(cluster
) < "octopus") {
4634 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
4641 bl
.append("there hiHI");
4642 ObjectWriteOperation op
;
4644 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
4648 bl
.append("there hiHI");
4649 ObjectWriteOperation op
;
4651 ASSERT_EQ(0, ioctx
.operate("chunk1", &op
));
4655 bl
.append("there HIHI");
4656 ObjectWriteOperation op
;
4658 ASSERT_EQ(0, ioctx
.operate("chunk2", &op
));
4661 // wait for maps to settle
4662 cluster
.wait_for_latest_osdmap();
4664 // create a snapshot, clone
4665 vector
<uint64_t> my_snaps(1);
4666 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4667 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4672 bl
.append("There hiHI");
4673 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, bl
.length(), 0));
4677 my_snaps
[1] = my_snaps
[0];
4678 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4679 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4684 bl
.append("tHere hiHI");
4685 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, bl
.length(), 0));
4689 manifest_set_chunk(cluster
, ioctx
, cache_ioctx
, 0, 10, "chunk1", "foo");
4691 cache_ioctx
.snap_set_read(my_snaps
[1]);
4694 manifest_set_chunk(cluster
, ioctx
, cache_ioctx
, 0, 10, "chunk2", "foo");
4698 ObjectReadOperation op
, stat_op
;
4700 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4701 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4702 "foo", completion
, &op
,
4703 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4704 completion
->wait_for_complete();
4705 ASSERT_EQ(0, completion
->get_return_value());
4709 ASSERT_EQ(0, cache_ioctx
.get_object_pg_hash_position2("foo", &hash
));
4713 for (int tries
= 0; tries
< 5; ++tries
) {
4716 ss
<< "{\"prefix\": \"pg deep-scrub\", \"pgid\": \""
4717 << cache_ioctx
.get_id() << "."
4720 int r
= cluster
.mon_command(ss
.str(), inbl
, NULL
, NULL
);
4729 cout
<< "waiting for scrubs..." << std::endl
;
4731 cout
<< "done waiting" << std::endl
;
4736 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
4737 ASSERT_EQ('t', bl
[0]);
4741 #include <common/CDC.h>
4742 TEST_F(LibRadosTwoPoolsPP
, DedupFlushRead
) {
4743 // skip test if not yet octopus
4744 if (_get_required_osd_release(cluster
) < "octopus") {
4745 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4749 ASSERT_EQ(0, cluster
.mon_command(
4750 set_pool_str(cache_pool_name
, "fingerprint_algorithm", "sha1"),
4752 ASSERT_EQ(0, cluster
.mon_command(
4753 set_pool_str(cache_pool_name
, "dedup_tier", pool_name
),
4755 ASSERT_EQ(0, cluster
.mon_command(
4756 set_pool_str(cache_pool_name
, "dedup_chunk_algorithm", "fastcdc"),
4758 ASSERT_EQ(0, cluster
.mon_command(
4759 set_pool_str(cache_pool_name
, "dedup_cdc_chunk_size", 1024),
4762 // wait for maps to settle
4763 cluster
.wait_for_latest_osdmap();
4768 generate_buffer(1024*8, &gbl
);
4769 ObjectWriteOperation op
;
4771 ASSERT_EQ(0, cache_ioctx
.operate("foo-chunk", &op
));
4775 bl
.append("DDse chunk");
4776 ObjectWriteOperation op
;
4778 ASSERT_EQ(0, ioctx
.operate("bar-chunk", &op
));
4781 // set-chunk to set manifest object
4783 ObjectReadOperation op
;
4784 op
.set_chunk(0, 2, ioctx
, "bar-chunk", 0,
4785 CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
4786 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4787 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo-chunk", completion
, &op
,
4788 librados::OPERATION_IGNORE_CACHE
, NULL
));
4789 completion
->wait_for_complete();
4790 ASSERT_EQ(0, completion
->get_return_value());
4791 completion
->release();
4795 ObjectReadOperation op
;
4797 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4798 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4799 "foo-chunk", completion
, &op
,
4800 librados::OPERATION_IGNORE_CACHE
, NULL
));
4801 completion
->wait_for_complete();
4802 ASSERT_EQ(0, completion
->get_return_value());
4803 completion
->release();
4806 std::unique_ptr
<CDC
> cdc
= CDC::create("fastcdc", cbits(1024)-1);
4807 vector
<pair
<uint64_t, uint64_t>> chunks
;
4809 cdc
->calc_chunks(gbl
, &chunks
);
4810 chunk
.substr_of(gbl
, chunks
[1].first
, chunks
[1].second
);
4813 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1] = {0};
4814 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
4816 int size
= chunk
.length();
4817 sha1_gen
.Update((const unsigned char *)chunk
.c_str(), size
);
4818 sha1_gen
.Final(fingerprint
);
4819 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
4820 tgt_oid
= string(p_str
);
4823 // read and verify the chunked object
4826 ASSERT_EQ(2, ioctx
.read(tgt_oid
, test_bl
, 2, 0));
4827 ASSERT_EQ(test_bl
[1], chunk
[1]);
4830 ASSERT_EQ(0, cluster
.mon_command(
4831 set_pool_str(cache_pool_name
, "dedup_cdc_chunk_size", 512),
4833 cluster
.wait_for_latest_osdmap();
4835 // make a dirty chunks
4839 ASSERT_EQ(0, cache_ioctx
.write("foo-chunk", bl
, bl
.length(), 0));
4844 ObjectReadOperation op
;
4846 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4847 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4848 "foo-chunk", completion
, &op
,
4849 librados::OPERATION_IGNORE_CACHE
, NULL
));
4850 completion
->wait_for_complete();
4851 ASSERT_EQ(0, completion
->get_return_value());
4852 completion
->release();
4855 cdc
= CDC::create("fastcdc", cbits(512)-1);
4857 cdc
->calc_chunks(gbl
, &chunks
);
4858 bufferlist chunk_512
;
4859 chunk_512
.substr_of(gbl
, chunks
[3].first
, chunks
[3].second
);
4861 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1] = {0};
4862 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
4864 int size
= chunk_512
.length();
4865 sha1_gen
.Update((const unsigned char *)chunk_512
.c_str(), size
);
4866 sha1_gen
.Final(fingerprint
);
4867 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
4868 tgt_oid
= string(p_str
);
4871 // read and verify the chunked object
4874 ASSERT_EQ(2, ioctx
.read(tgt_oid
, test_bl
, 2, 0));
4875 ASSERT_EQ(test_bl
[1], chunk_512
[1]);
4878 ASSERT_EQ(0, cluster
.mon_command(
4879 set_pool_str(cache_pool_name
, "dedup_cdc_chunk_size", 16384),
4881 cluster
.wait_for_latest_osdmap();
4883 // make a dirty chunks
4887 ASSERT_EQ(0, cache_ioctx
.write("foo-chunk", bl
, bl
.length(), 0));
4888 gbl
.begin(0).copy_in(bl
.length(), bl
);
4892 ObjectReadOperation op
;
4894 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4895 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4896 "foo-chunk", completion
, &op
,
4897 librados::OPERATION_IGNORE_CACHE
, NULL
));
4898 completion
->wait_for_complete();
4899 ASSERT_EQ(0, completion
->get_return_value());
4900 completion
->release();
4903 cdc
= CDC::create("fastcdc", cbits(16384)-1);
4905 cdc
->calc_chunks(gbl
, &chunks
);
4906 bufferlist chunk_16384
;
4907 chunk_16384
.substr_of(gbl
, chunks
[0].first
, chunks
[0].second
);
4909 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1] = {0};
4910 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
4912 int size
= chunk_16384
.length();
4913 sha1_gen
.Update((const unsigned char *)chunk_16384
.c_str(), size
);
4914 sha1_gen
.Final(fingerprint
);
4915 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
4916 tgt_oid
= string(p_str
);
4918 // read and verify the chunked object
4921 ASSERT_EQ(2, ioctx
.read(tgt_oid
, test_bl
, 2, 0));
4922 ASSERT_EQ(test_bl
[0], chunk_16384
[0]);
4925 // less than object size
4926 ASSERT_EQ(0, cluster
.mon_command(
4927 set_pool_str(cache_pool_name
, "dedup_cdc_chunk_size", 1024),
4929 cluster
.wait_for_latest_osdmap();
4931 // make a dirty chunks
4932 // a chunk_info is deleted by write, which converts the manifest object to non-manifest object
4936 ASSERT_EQ(0, cache_ioctx
.write("foo-chunk", bl
, bl
.length(), 0));
4942 bl
.append("DDse chunk");
4943 ObjectWriteOperation op
;
4945 ASSERT_EQ(0, ioctx
.operate("bar-chunk", &op
));
4947 // set-chunk to set manifest object
4949 ObjectReadOperation op
;
4950 op
.set_chunk(0, 2, ioctx
, "bar-chunk", 0,
4951 CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
4952 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4953 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo-chunk", completion
, &op
,
4954 librados::OPERATION_IGNORE_CACHE
, NULL
));
4955 completion
->wait_for_complete();
4956 ASSERT_EQ(0, completion
->get_return_value());
4957 completion
->release();
4961 ObjectReadOperation op
;
4963 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4964 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4965 "foo-chunk", completion
, &op
,
4966 librados::OPERATION_IGNORE_CACHE
, NULL
));
4967 completion
->wait_for_complete();
4968 ASSERT_EQ(0, completion
->get_return_value());
4969 completion
->release();
4972 cdc
= CDC::create("fastcdc", cbits(1024)-1);
4974 cdc
->calc_chunks(gbl
, &chunks
);
4975 bufferlist small_chunk
;
4976 small_chunk
.substr_of(gbl
, chunks
[1].first
, chunks
[1].second
);
4978 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1] = {0};
4979 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
4981 int size
= small_chunk
.length();
4982 sha1_gen
.Update((const unsigned char *)small_chunk
.c_str(), size
);
4983 sha1_gen
.Final(fingerprint
);
4984 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
4985 tgt_oid
= string(p_str
);
4987 // read and verify the chunked object
4990 ASSERT_EQ(2, ioctx
.read(tgt_oid
, test_bl
, 2, 0));
4991 ASSERT_EQ(test_bl
[0], small_chunk
[0]);
4996 TEST_F(LibRadosTwoPoolsPP
, ManifestFlushSnap
) {
4997 // skip test if not yet octopus
4998 if (_get_required_osd_release(cluster
) < "octopus") {
4999 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
5004 ASSERT_EQ(0, cluster
.mon_command(
5005 set_pool_str(cache_pool_name
, "fingerprint_algorithm", "sha1"),
5007 ASSERT_EQ(0, cluster
.mon_command(
5008 set_pool_str(cache_pool_name
, "dedup_tier", pool_name
),
5010 ASSERT_EQ(0, cluster
.mon_command(
5011 set_pool_str(cache_pool_name
, "dedup_chunk_algorithm", "fastcdc"),
5013 ASSERT_EQ(0, cluster
.mon_command(
5014 set_pool_str(cache_pool_name
, "dedup_cdc_chunk_size", 1024),
5017 // wait for maps to settle
5018 cluster
.wait_for_latest_osdmap();
5024 //bl.append("there hi");
5025 generate_buffer(1024*8, &gbl
);
5026 ObjectWriteOperation op
;
5028 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
5032 bl
.append("there hi");
5033 ObjectWriteOperation op
;
5035 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
5038 // set-chunk (dedup)
5039 manifest_set_chunk(cluster
, ioctx
, cache_ioctx
, 2, 2, "bar", "foo");
5041 // create a snapshot, clone
5042 vector
<uint64_t> my_snaps(1);
5043 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
5044 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
5047 // make a dirty chunks
5051 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, bl
.length(), 0));
5056 my_snaps
[1] = my_snaps
[0];
5057 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
5058 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
5061 // make a dirty chunks
5065 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, bl
.length(), 0));
5068 // flush on head (should fail)
5069 cache_ioctx
.snap_set_read(librados::SNAP_HEAD
);
5072 ObjectReadOperation op
;
5074 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5075 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5076 "foo", completion
, &op
,
5077 librados::OPERATION_IGNORE_CACHE
, NULL
));
5078 completion
->wait_for_complete();
5079 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
5080 completion
->release();
5083 // flush on recent snap (should fail)
5084 cache_ioctx
.snap_set_read(my_snaps
[0]);
5086 ObjectReadOperation op
;
5088 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5089 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5090 "foo", completion
, &op
,
5091 librados::OPERATION_IGNORE_CACHE
, NULL
));
5092 completion
->wait_for_complete();
5093 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
5094 completion
->release();
5097 // flush on oldest snap
5098 cache_ioctx
.snap_set_read(my_snaps
[1]);
5100 ObjectReadOperation op
;
5102 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5103 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5104 "foo", completion
, &op
,
5105 librados::OPERATION_IGNORE_CACHE
, NULL
));
5106 completion
->wait_for_complete();
5107 ASSERT_EQ(0, completion
->get_return_value());
5108 completion
->release();
5111 // flush on oldest snap
5112 cache_ioctx
.snap_set_read(my_snaps
[0]);
5114 ObjectReadOperation op
;
5116 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5117 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5118 "foo", completion
, &op
,
5119 librados::OPERATION_IGNORE_CACHE
, NULL
));
5120 completion
->wait_for_complete();
5121 ASSERT_EQ(0, completion
->get_return_value());
5122 completion
->release();
5125 // flush on oldest snap
5126 cache_ioctx
.snap_set_read(librados::SNAP_HEAD
);
5128 ObjectReadOperation op
;
5130 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5131 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5132 "foo", completion
, &op
,
5133 librados::OPERATION_IGNORE_CACHE
, NULL
));
5134 completion
->wait_for_complete();
5135 ASSERT_EQ(0, completion
->get_return_value());
5136 completion
->release();
5139 // check chunk's refcount
5140 std::unique_ptr
<CDC
> cdc
= CDC::create("fastcdc", cbits(1024)-1);
5141 vector
<pair
<uint64_t, uint64_t>> chunks
;
5143 cdc
->calc_chunks(gbl
, &chunks
);
5144 chunk
.substr_of(gbl
, chunks
[1].first
, chunks
[1].second
);
5147 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1] = {0};
5148 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
5150 int size
= chunk
.length();
5151 sha1_gen
.Update((const unsigned char *)chunk
.c_str(), size
);
5152 sha1_gen
.Final(fingerprint
);
5153 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
5154 tgt_oid
= string(p_str
);
5156 // read and verify the chunked object
5159 ASSERT_EQ(2, ioctx
.read(tgt_oid
, test_bl
, 2, 0));
5160 ASSERT_EQ(test_bl
[1], chunk
[1]);
5163 cache_ioctx
.snap_set_read(librados::SNAP_HEAD
);
5166 ASSERT_EQ(4, cache_ioctx
.read("foo", bl
, 4, 0));
5167 ASSERT_EQ('c', bl
[2]);
5170 cache_ioctx
.snap_set_read(my_snaps
[0]);
5173 ASSERT_EQ(4, cache_ioctx
.read("foo", bl
, 4, 0));
5174 ASSERT_EQ('b', bl
[2]);
5178 TEST_F(LibRadosTwoPoolsPP
, ManifestFlushDupCount
) {
5179 // skip test if not yet octopus
5180 if (_get_required_osd_release(cluster
) < "octopus") {
5181 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
5186 ASSERT_EQ(0, cluster
.mon_command(
5187 set_pool_str(cache_pool_name
, "fingerprint_algorithm", "sha1"),
5189 ASSERT_EQ(0, cluster
.mon_command(
5190 set_pool_str(cache_pool_name
, "dedup_tier", pool_name
),
5192 ASSERT_EQ(0, cluster
.mon_command(
5193 set_pool_str(cache_pool_name
, "dedup_chunk_algorithm", "fastcdc"),
5195 ASSERT_EQ(0, cluster
.mon_command(
5196 set_pool_str(cache_pool_name
, "dedup_cdc_chunk_size", 1024),
5203 generate_buffer(1024*8, &gbl
);
5204 ObjectWriteOperation op
;
5206 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
5210 bl
.append("there hiHI");
5211 ObjectWriteOperation op
;
5213 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
5216 // wait for maps to settle
5217 cluster
.wait_for_latest_osdmap();
5219 // set-chunk to set manifest object
5221 ObjectReadOperation op
;
5222 op
.set_chunk(0, 2, ioctx
, "bar", 0,
5223 CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
5224 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5225 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
5226 librados::OPERATION_IGNORE_CACHE
, NULL
));
5227 completion
->wait_for_complete();
5228 ASSERT_EQ(0, completion
->get_return_value());
5229 completion
->release();
5232 // create a snapshot, clone
5233 vector
<uint64_t> my_snaps(1);
5234 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
5235 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
5238 // make a dirty chunks
5241 bl
.append("Thbbe hi");
5242 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, bl
.length(), 0));
5247 my_snaps
[1] = my_snaps
[0];
5248 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
5249 ASSERT_EQ(0, cache_ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
5252 // make a dirty chunks
5255 bl
.append("Thcce hi");
5256 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, bl
.length(), 0));
5259 //flush on oldest snap
5260 cache_ioctx
.snap_set_read(my_snaps
[1]);
5263 ObjectReadOperation op
;
5265 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5266 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5267 "foo", completion
, &op
,
5268 librados::OPERATION_IGNORE_CACHE
, NULL
));
5269 completion
->wait_for_complete();
5270 ASSERT_EQ(0, completion
->get_return_value());
5271 completion
->release();
5274 // flush on oldest snap
5275 cache_ioctx
.snap_set_read(my_snaps
[0]);
5278 ObjectReadOperation op
;
5280 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5281 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5282 "foo", completion
, &op
,
5283 librados::OPERATION_IGNORE_CACHE
, NULL
));
5284 completion
->wait_for_complete();
5285 ASSERT_EQ(0, completion
->get_return_value());
5286 completion
->release();
5289 cache_ioctx
.snap_set_read(librados::SNAP_HEAD
);
5292 ObjectReadOperation op
;
5294 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5295 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5296 "foo", completion
, &op
,
5297 librados::OPERATION_IGNORE_CACHE
, NULL
));
5298 completion
->wait_for_complete();
5299 ASSERT_EQ(0, completion
->get_return_value());
5300 completion
->release();
5303 std::unique_ptr
<CDC
> cdc
= CDC::create("fastcdc", cbits(1024)-1);
5304 vector
<pair
<uint64_t, uint64_t>> chunks
;
5306 cdc
->calc_chunks(gbl
, &chunks
);
5307 chunk
.substr_of(gbl
, chunks
[1].first
, chunks
[1].second
);
5309 // check chunk's refcount
5311 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1] = {0};
5312 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
5315 int size
= chunk
.length();
5316 sha1_gen
.Update((const unsigned char *)chunk
.c_str(), size
);
5317 sha1_gen
.Final(fingerprint
);
5318 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
5319 tgt_oid
= string(p_str
);
5320 ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
5323 auto iter
= t
.cbegin();
5325 } catch (buffer::error
& err
) {
5328 ASSERT_EQ(1u, refs
.count());
5332 chunk2
.substr_of(gbl
, chunks
[0].first
, chunks
[0].second
);
5333 // check chunk's refcount
5335 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1] = {0};
5336 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
5339 int size
= chunk2
.length();
5340 sha1_gen
.Update((const unsigned char *)chunk2
.c_str(), size
);
5341 sha1_gen
.Final(fingerprint
);
5342 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
5343 tgt_oid
= string(p_str
);
5344 ioctx
.getxattr(p_str
, CHUNK_REFCOUNT_ATTR
, t
);
5347 auto iter
= t
.cbegin();
5349 } catch (buffer::error
& err
) {
5352 ASSERT_EQ(1u, refs
.count());
5355 // make a dirty chunks
5358 bl
.append("ThDDe hi");
5359 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, bl
.length(), 0));
5364 ObjectReadOperation op
;
5366 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5367 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5368 "foo", completion
, &op
,
5369 librados::OPERATION_IGNORE_CACHE
, NULL
));
5370 completion
->wait_for_complete();
5371 ASSERT_EQ(0, completion
->get_return_value());
5372 completion
->release();
5376 tmp
.append("Thcce hi");
5377 gbl
.begin(0).copy_in(tmp
.length(), tmp
);
5379 cdc
->calc_chunks(gbl
, &chunks
);
5380 chunk3
.substr_of(gbl
, chunks
[0].first
, chunks
[0].second
);
5381 // check chunk's refcount
5383 unsigned char fingerprint
[CEPH_CRYPTO_SHA1_DIGESTSIZE
+ 1] = {0};
5384 char p_str
[CEPH_CRYPTO_SHA1_DIGESTSIZE
*2+1] = {0};
5387 int size
= chunk2
.length();
5388 sha1_gen
.Update((const unsigned char *)chunk2
.c_str(), size
);
5389 sha1_gen
.Final(fingerprint
);
5390 buf_to_hex(fingerprint
, CEPH_CRYPTO_SHA1_DIGESTSIZE
, p_str
);
5391 is_intended_refcount_state(cache_ioctx
, "foo", ioctx
, p_str
, 0);
5395 TEST_F(LibRadosTwoPoolsPP
, TierFlushDuringFlush
) {
5396 // skip test if not yet octopus
5397 if (_get_required_osd_release(cluster
) < "octopus") {
5398 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
5404 // create a new pool
5405 std::string temp_pool_name
= get_temp_pool_name() + "-test-flush";
5406 ASSERT_EQ(0, cluster
.pool_create(temp_pool_name
.c_str()));
5408 ASSERT_EQ(0, cluster
.mon_command(
5409 set_pool_str(cache_pool_name
, "fingerprint_algorithm", "sha1"),
5411 ASSERT_EQ(0, cluster
.mon_command(
5412 set_pool_str(cache_pool_name
, "dedup_tier", temp_pool_name
),
5414 ASSERT_EQ(0, cluster
.mon_command(
5415 set_pool_str(cache_pool_name
, "dedup_chunk_algorithm", "fastcdc"),
5417 ASSERT_EQ(0, cluster
.mon_command(
5418 set_pool_str(cache_pool_name
, "dedup_cdc_chunk_size", 1024),
5425 generate_buffer(1024*8, &gbl
);
5426 ObjectWriteOperation op
;
5428 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
5432 bl
.append("there hiHI");
5433 ObjectWriteOperation op
;
5435 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
5438 // set-chunk to set manifest object
5440 ObjectReadOperation op
;
5441 op
.set_chunk(0, 2, ioctx
, "bar", 0,
5442 CEPH_OSD_OP_FLAG_WITH_REFERENCE
);
5443 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5444 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
5445 librados::OPERATION_IGNORE_CACHE
, NULL
));
5446 completion
->wait_for_complete();
5447 ASSERT_EQ(0, completion
->get_return_value());
5448 completion
->release();
5451 // delete temp pool, so flushing chunk will fail
5452 ASSERT_EQ(0, s_cluster
.pool_delete(temp_pool_name
.c_str()));
5454 // flush to check if proper error is returned
5456 ObjectReadOperation op
;
5458 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5459 ASSERT_EQ(0, cache_ioctx
.aio_operate(
5460 "foo", completion
, &op
,
5461 librados::OPERATION_IGNORE_CACHE
, NULL
));
5462 completion
->wait_for_complete();
5463 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
5464 completion
->release();
5469 TEST_F(LibRadosTwoPoolsPP
, ManifestSnapHasChunk
) {
5470 // skip test if not yet octopus
5471 if (_get_required_osd_release(cluster
) < "octopus") {
5472 cout
<< "cluster is not yet octopus, skipping test" << std::endl
;
5477 ASSERT_EQ(0, cluster
.mon_command(
5478 set_pool_str(pool_name
, "fingerprint_algorithm", "sha1"),
5480 cluster
.wait_for_latest_osdmap();
5485 bl
.append("there HIHI");
5486 ObjectWriteOperation op
;
5488 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5491 string er_fp_oid
, hi_fp_oid
, HI_fp_oid
, ai_fp_oid
, bi_fp_oid
,
5492 Er_fp_oid
, Hi_fp_oid
, SI_fp_oid
;
5495 er_fp_oid
= get_fp_oid("er", "sha1");
5496 hi_fp_oid
= get_fp_oid("hi", "sha1");
5497 HI_fp_oid
= get_fp_oid("HI", "sha1");
5498 ai_fp_oid
= get_fp_oid("ai", "sha1");
5499 bi_fp_oid
= get_fp_oid("bi", "sha1");
5500 Er_fp_oid
= get_fp_oid("Er", "sha1");
5501 Hi_fp_oid
= get_fp_oid("Hi", "sha1");
5502 SI_fp_oid
= get_fp_oid("SI", "sha1");
5506 ObjectWriteOperation op
;
5510 ASSERT_EQ(0, cache_ioctx
.operate(er_fp_oid
, &op
));
5514 ObjectWriteOperation op
;
5518 ASSERT_EQ(0, cache_ioctx
.operate(hi_fp_oid
, &op
));
5522 ObjectWriteOperation op
;
5526 ASSERT_EQ(0, cache_ioctx
.operate(HI_fp_oid
, &op
));
5530 ObjectWriteOperation op
;
5534 ASSERT_EQ(0, cache_ioctx
.operate(ai_fp_oid
, &op
));
5538 ObjectWriteOperation op
;
5542 ASSERT_EQ(0, cache_ioctx
.operate(bi_fp_oid
, &op
));
5546 ObjectWriteOperation op
;
5550 ASSERT_EQ(0, cache_ioctx
.operate(Er_fp_oid
, &op
));
5554 ObjectWriteOperation op
;
5558 ASSERT_EQ(0, cache_ioctx
.operate(Hi_fp_oid
, &op
));
5562 ObjectWriteOperation op
;
5566 ASSERT_EQ(0, cache_ioctx
.operate(SI_fp_oid
, &op
));
5569 // set-chunk (dedup)
5570 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, HI_fp_oid
, "foo");
5571 // set-chunk (dedup)
5572 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, HI_fp_oid
, "foo");
5574 // foo head: [hi] [HI]
5576 // create a snapshot, clone
5577 vector
<uint64_t> my_snaps(1);
5578 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
5579 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
5587 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 6));
5593 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 6));
5599 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 8));
5602 // foo snap[0]: [hi] [HI]
5603 // foo head : [er] [ai] [SI]
5605 // set-chunk (dedup)
5606 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 2, 2, er_fp_oid
, "foo");
5607 // set-chunk (dedup)
5608 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, ai_fp_oid
, "foo");
5609 // set-chunk (dedup)
5610 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 8, 2, SI_fp_oid
, "foo");
5613 my_snaps
[1] = my_snaps
[0];
5614 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
5615 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
5622 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 6));
5628 ASSERT_EQ(0, ioctx
.write("foo", bl
, 1, 6));
5631 // foo snap[1]: [HI] [HI]
5632 // foo snap[0]: [er] [ai] [SI]
5633 // foo head : [er] [bi] [SI]
5635 // set-chunk (dedup)
5636 manifest_set_chunk(cluster
, cache_ioctx
, ioctx
, 6, 2, bi_fp_oid
, "foo");
5639 ASSERT_EQ(1, cls_cas_references_chunk(ioctx
, "foo", SI_fp_oid
));
5640 ASSERT_EQ(1, cls_cas_references_chunk(ioctx
, "foo", er_fp_oid
));
5641 ASSERT_EQ(1, cls_cas_references_chunk(ioctx
, "foo", ai_fp_oid
));
5642 ASSERT_EQ(2, cls_cas_references_chunk(ioctx
, "foo", HI_fp_oid
));
5643 ASSERT_EQ(-ENOLINK
, cls_cas_references_chunk(ioctx
, "foo", Hi_fp_oid
));
5647 class LibRadosTwoPoolsECPP
: public RadosTestECPP
5650 LibRadosTwoPoolsECPP() {};
5651 ~LibRadosTwoPoolsECPP() override
{};
5653 static void SetUpTestCase() {
5654 pool_name
= get_temp_pool_name();
5655 ASSERT_EQ("", create_one_ec_pool_pp(pool_name
, s_cluster
));
5657 static void TearDownTestCase() {
5658 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name
, s_cluster
));
5660 static std::string cache_pool_name
;
5662 void SetUp() override
{
5663 cache_pool_name
= get_temp_pool_name();
5664 ASSERT_EQ(0, s_cluster
.pool_create(cache_pool_name
.c_str()));
5665 RadosTestECPP::SetUp();
5667 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
5668 cache_ioctx
.application_enable("rados", true);
5669 cache_ioctx
.set_namespace(nspace
);
5671 void TearDown() override
{
5672 // flush + evict cache
5673 flush_evict_all(cluster
, cache_ioctx
);
5677 ASSERT_EQ(0, cluster
.mon_command(
5678 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
5681 ASSERT_EQ(0, cluster
.mon_command(
5682 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
5683 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
5686 // wait for maps to settle before next test
5687 cluster
.wait_for_latest_osdmap();
5689 RadosTestECPP::TearDown();
5691 cleanup_default_namespace(cache_ioctx
);
5692 cleanup_namespace(cache_ioctx
, nspace
);
5694 cache_ioctx
.close();
5695 ASSERT_EQ(0, s_cluster
.pool_delete(cache_pool_name
.c_str()));
5698 librados::IoCtx cache_ioctx
;
5701 std::string
LibRadosTwoPoolsECPP::cache_pool_name
;
5703 TEST_F(LibRadosTierECPP
, Dirty
) {
5705 ObjectWriteOperation op
;
5707 ASSERT_EQ(0, ioctx
.operate("foo", &op
)); // still get 0 if it dne
5710 ObjectWriteOperation op
;
5712 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5717 ObjectReadOperation op
;
5718 op
.is_dirty(&dirty
, &r
);
5719 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
5724 ObjectWriteOperation op
;
5726 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5729 ObjectWriteOperation op
;
5731 ASSERT_EQ(0, ioctx
.operate("foo", &op
)); // still 0 if already clean
5736 ObjectReadOperation op
;
5737 op
.is_dirty(&dirty
, &r
);
5738 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
5739 ASSERT_FALSE(dirty
);
5743 // ObjectWriteOperation op;
5744 // op.truncate(0); // still a write even tho it is a no-op
5745 // ASSERT_EQ(0, ioctx.operate("foo", &op));
5748 // bool dirty = false;
5750 // ObjectReadOperation op;
5751 // op.is_dirty(&dirty, &r);
5752 // ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
5753 // ASSERT_TRUE(dirty);
5758 TEST_F(LibRadosTwoPoolsECPP
, Overlay
) {
5763 ObjectWriteOperation op
;
5765 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5770 ObjectWriteOperation op
;
5772 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
5777 ASSERT_EQ(0, cluster
.mon_command(
5778 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
5779 "\", \"tierpool\": \"" + cache_pool_name
+
5780 "\", \"force_nonempty\": \"--force-nonempty\" }",
5782 ASSERT_EQ(0, cluster
.mon_command(
5783 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
5784 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
5787 // wait for maps to settle
5788 cluster
.wait_for_latest_osdmap();
5790 // by default, the overlay sends us to cache pool
5793 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
5794 ASSERT_EQ('c', bl
[0]);
5798 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
5799 ASSERT_EQ('c', bl
[0]);
5802 // unless we say otherwise
5805 ObjectReadOperation op
;
5806 op
.read(0, 1, &bl
, NULL
);
5807 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5808 ASSERT_EQ(0, ioctx
.aio_operate(
5809 "foo", completion
, &op
,
5810 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
5811 completion
->wait_for_complete();
5812 ASSERT_EQ(0, completion
->get_return_value());
5813 completion
->release();
5814 ASSERT_EQ('b', bl
[0]);
5818 TEST_F(LibRadosTwoPoolsECPP
, Promote
) {
5822 bl
.append("hi there");
5823 ObjectWriteOperation op
;
5825 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5830 ASSERT_EQ(0, cluster
.mon_command(
5831 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
5832 "\", \"tierpool\": \"" + cache_pool_name
+
5833 "\", \"force_nonempty\": \"--force-nonempty\" }",
5835 ASSERT_EQ(0, cluster
.mon_command(
5836 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
5837 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
5839 ASSERT_EQ(0, cluster
.mon_command(
5840 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
5841 "\", \"mode\": \"writeback\"}",
5844 // wait for maps to settle
5845 cluster
.wait_for_latest_osdmap();
5847 // read, trigger a promote
5850 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
5853 // read, trigger a whiteout
5856 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
5857 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
5860 // verify the object is present in the cache tier
5862 NObjectIterator it
= cache_ioctx
.nobjects_begin();
5863 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
5864 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
5866 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
5868 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
5872 TEST_F(LibRadosTwoPoolsECPP
, PromoteSnap
) {
5876 bl
.append("hi there");
5877 ObjectWriteOperation op
;
5879 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5883 bl
.append("hi there");
5884 ObjectWriteOperation op
;
5886 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
5890 bl
.append("hi there");
5891 ObjectWriteOperation op
;
5893 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
5897 bl
.append("hi there");
5898 ObjectWriteOperation op
;
5900 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
5903 // create a snapshot, clone
5904 vector
<uint64_t> my_snaps(1);
5905 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
5906 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
5911 ObjectWriteOperation op
;
5913 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5918 ObjectWriteOperation op
;
5920 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
5923 ObjectWriteOperation op
;
5925 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
5930 ObjectWriteOperation op
;
5932 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
5937 ASSERT_EQ(0, cluster
.mon_command(
5938 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
5939 "\", \"tierpool\": \"" + cache_pool_name
+
5940 "\", \"force_nonempty\": \"--force-nonempty\" }",
5942 ASSERT_EQ(0, cluster
.mon_command(
5943 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
5944 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
5946 ASSERT_EQ(0, cluster
.mon_command(
5947 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
5948 "\", \"mode\": \"writeback\"}",
5951 // wait for maps to settle
5952 cluster
.wait_for_latest_osdmap();
5954 // read, trigger a promote on the head
5957 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
5958 ASSERT_EQ('c', bl
[0]);
5962 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
5963 ASSERT_EQ('c', bl
[0]);
5966 ioctx
.snap_set_read(my_snaps
[0]);
5968 // stop and scrub this pg (to make sure scrub can handle missing
5969 // clones in the cache tier)
5970 // This test requires cache tier and base tier to have the same pg_num/pgp_num
5972 for (int tries
= 0; tries
< 5; ++tries
) {
5974 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
5976 ASSERT_EQ(0, ioctx
.get_object_pg_hash_position2("foo", &hash
));
5978 ss
<< "{\"prefix\": \"pg scrub\", \"pgid\": \""
5979 << cache_ioctx
.get_id() << "."
5982 int r
= cluster
.mon_command(ss
.str(), inbl
, NULL
, NULL
);
5984 r
== -ENOENT
) { // in case mgr osdmap is a bit stale
5991 // give it a few seconds to go. this is sloppy but is usually enough time
5992 cout
<< "waiting for scrub..." << std::endl
;
5994 cout
<< "done waiting" << std::endl
;
6000 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
6001 ASSERT_EQ('h', bl
[0]);
6007 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
6008 ASSERT_EQ('h', bl
[0]);
6014 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
6015 ASSERT_EQ('h', bl
[0]);
6018 ioctx
.snap_set_read(librados::SNAP_HEAD
);
6023 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
6024 ASSERT_EQ('c', bl
[0]);
6030 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
6031 ASSERT_EQ('c', bl
[0]);
6037 ASSERT_EQ(-ENOENT
, ioctx
.read("baz", bl
, 1, 0));
6041 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
6044 TEST_F(LibRadosTwoPoolsECPP
, PromoteSnapTrimRace
) {
6048 bl
.append("hi there");
6049 ObjectWriteOperation op
;
6051 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6054 // create a snapshot, clone
6055 vector
<uint64_t> my_snaps(1);
6056 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
6057 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
6062 ObjectWriteOperation op
;
6064 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6069 ASSERT_EQ(0, cluster
.mon_command(
6070 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
6071 "\", \"tierpool\": \"" + cache_pool_name
+
6072 "\", \"force_nonempty\": \"--force-nonempty\" }",
6074 ASSERT_EQ(0, cluster
.mon_command(
6075 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
6076 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
6078 ASSERT_EQ(0, cluster
.mon_command(
6079 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
6080 "\", \"mode\": \"writeback\"}",
6083 // wait for maps to settle
6084 cluster
.wait_for_latest_osdmap();
6087 ASSERT_EQ(0, ioctx
.selfmanaged_snap_remove(my_snaps
[0]));
6089 ioctx
.snap_set_read(my_snaps
[0]);
6091 // read foo snap. the OSD may or may not realize that this snap has
6092 // been logically deleted; either response is valid.
6095 int r
= ioctx
.read("foo", bl
, 1, 0);
6096 ASSERT_TRUE(r
== 1 || r
== -ENOENT
);
6100 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
6103 TEST_F(LibRadosTwoPoolsECPP
, Whiteout
) {
6107 bl
.append("hi there");
6108 ObjectWriteOperation op
;
6110 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6115 ASSERT_EQ(0, cluster
.mon_command(
6116 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
6117 "\", \"tierpool\": \"" + cache_pool_name
+
6118 "\", \"force_nonempty\": \"--force-nonempty\" }",
6120 ASSERT_EQ(0, cluster
.mon_command(
6121 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
6122 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
6124 ASSERT_EQ(0, cluster
.mon_command(
6125 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
6126 "\", \"mode\": \"writeback\"}",
6129 // wait for maps to settle
6130 cluster
.wait_for_latest_osdmap();
6132 // create some whiteouts, verify they behave
6134 ObjectWriteOperation op
;
6137 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6141 ObjectWriteOperation op
;
6144 ASSERT_EQ(-ENOENT
, ioctx
.operate("bar", &op
));
6147 ObjectWriteOperation op
;
6150 ASSERT_EQ(-ENOENT
, ioctx
.operate("bar", &op
));
6153 // verify the whiteouts are there in the cache tier
6155 NObjectIterator it
= cache_ioctx
.nobjects_begin();
6156 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
6157 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
6159 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
6161 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
6164 // delete a whiteout and verify it goes away
6165 ASSERT_EQ(-ENOENT
, ioctx
.remove("foo"));
6167 ObjectWriteOperation op
;
6169 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6170 ASSERT_EQ(0, ioctx
.aio_operate("bar", completion
, &op
,
6171 librados::OPERATION_IGNORE_CACHE
));
6172 completion
->wait_for_complete();
6173 ASSERT_EQ(0, completion
->get_return_value());
6174 completion
->release();
6176 NObjectIterator it
= cache_ioctx
.nobjects_begin();
6177 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
6178 ASSERT_TRUE(it
->get_oid() == string("foo"));
6180 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
6183 // recreate an object and verify we can read it
6186 bl
.append("hi there");
6187 ObjectWriteOperation op
;
6189 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6193 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
6194 ASSERT_EQ('h', bl
[0]);
6198 TEST_F(LibRadosTwoPoolsECPP
, Evict
) {
6202 bl
.append("hi there");
6203 ObjectWriteOperation op
;
6205 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6210 ASSERT_EQ(0, cluster
.mon_command(
6211 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
6212 "\", \"tierpool\": \"" + cache_pool_name
+
6213 "\", \"force_nonempty\": \"--force-nonempty\" }",
6215 ASSERT_EQ(0, cluster
.mon_command(
6216 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
6217 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
6219 ASSERT_EQ(0, cluster
.mon_command(
6220 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
6221 "\", \"mode\": \"writeback\"}",
6224 // wait for maps to settle
6225 cluster
.wait_for_latest_osdmap();
6227 // read, trigger a promote
6230 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
6233 // read, trigger a whiteout, and a dirty object
6236 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
6237 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
6238 ASSERT_EQ(0, ioctx
.write("bar", bl
, bl
.length(), 0));
6241 // verify the object is present in the cache tier
6243 NObjectIterator it
= cache_ioctx
.nobjects_begin();
6244 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
6245 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
6247 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
6249 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
6254 ObjectWriteOperation op
;
6256 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6257 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
6258 completion
->wait_for_complete();
6259 ASSERT_EQ(0, completion
->get_return_value());
6260 completion
->release();
6263 // evict the pinned object with -EPERM
6265 ObjectReadOperation op
;
6267 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6268 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
6269 librados::OPERATION_IGNORE_CACHE
,
6271 completion
->wait_for_complete();
6272 ASSERT_EQ(-EPERM
, completion
->get_return_value());
6273 completion
->release();
6278 ObjectWriteOperation op
;
6280 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6281 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
6282 completion
->wait_for_complete();
6283 ASSERT_EQ(0, completion
->get_return_value());
6284 completion
->release();
6289 ObjectReadOperation op
;
6291 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6292 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6293 "foo", completion
, &op
,
6294 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
6295 completion
->wait_for_complete();
6296 ASSERT_EQ(0, completion
->get_return_value());
6297 completion
->release();
6304 ObjectReadOperation op
;
6305 op
.is_dirty(&dirty
, &r
);
6306 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
6307 ASSERT_FALSE(dirty
);
6313 ObjectReadOperation op
;
6315 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6316 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
6317 librados::OPERATION_IGNORE_CACHE
,
6319 completion
->wait_for_complete();
6320 ASSERT_EQ(0, completion
->get_return_value());
6321 completion
->release();
6324 ObjectReadOperation op
;
6326 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6327 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6328 "foo", completion
, &op
,
6329 librados::OPERATION_IGNORE_CACHE
, NULL
));
6330 completion
->wait_for_complete();
6331 ASSERT_EQ(0, completion
->get_return_value());
6332 completion
->release();
6335 ObjectReadOperation op
;
6337 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6338 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6339 "bar", completion
, &op
,
6340 librados::OPERATION_IGNORE_CACHE
, NULL
));
6341 completion
->wait_for_complete();
6342 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
6343 completion
->release();
6347 TEST_F(LibRadosTwoPoolsECPP
, EvictSnap
) {
6351 bl
.append("hi there");
6352 ObjectWriteOperation op
;
6354 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6358 bl
.append("hi there");
6359 ObjectWriteOperation op
;
6361 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
6365 bl
.append("hi there");
6366 ObjectWriteOperation op
;
6368 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
6372 bl
.append("hi there");
6373 ObjectWriteOperation op
;
6375 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
6378 // create a snapshot, clone
6379 vector
<uint64_t> my_snaps(1);
6380 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
6381 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
6386 ObjectWriteOperation op
;
6388 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6393 ObjectWriteOperation op
;
6395 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
6398 ObjectWriteOperation op
;
6400 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
6405 ObjectWriteOperation op
;
6407 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
6412 ASSERT_EQ(0, cluster
.mon_command(
6413 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
6414 "\", \"tierpool\": \"" + cache_pool_name
+
6415 "\", \"force_nonempty\": \"--force-nonempty\" }",
6417 ASSERT_EQ(0, cluster
.mon_command(
6418 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
6419 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
6421 ASSERT_EQ(0, cluster
.mon_command(
6422 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
6423 "\", \"mode\": \"writeback\"}",
6426 // wait for maps to settle
6427 cluster
.wait_for_latest_osdmap();
6429 // read, trigger a promote on the head
6432 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
6433 ASSERT_EQ('c', bl
[0]);
6437 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
6438 ASSERT_EQ('c', bl
[0]);
6443 ObjectReadOperation op
;
6445 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6446 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6447 "bam", completion
, &op
,
6448 librados::OPERATION_IGNORE_CACHE
, NULL
));
6449 completion
->wait_for_complete();
6450 ASSERT_EQ(0, completion
->get_return_value());
6451 completion
->release();
6455 ObjectReadOperation op
;
6456 op
.read(1, 0, &bl
, NULL
);
6457 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6458 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6459 "bam", completion
, &op
,
6460 librados::OPERATION_IGNORE_CACHE
, NULL
));
6461 completion
->wait_for_complete();
6462 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
6463 completion
->release();
6467 ioctx
.snap_set_read(my_snaps
[0]);
6470 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
6471 ASSERT_EQ('h', bl
[0]);
6476 ObjectReadOperation op
;
6478 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6479 ASSERT_EQ(0, ioctx
.aio_operate(
6480 "foo", completion
, &op
,
6481 librados::OPERATION_IGNORE_CACHE
, NULL
));
6482 completion
->wait_for_complete();
6483 ASSERT_EQ(0, completion
->get_return_value());
6484 completion
->release();
6489 ObjectReadOperation op
;
6490 op
.read(1, 0, &bl
, NULL
);
6491 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6492 ASSERT_EQ(0, ioctx
.aio_operate(
6493 "foo", completion
, &op
,
6494 librados::OPERATION_IGNORE_CACHE
, NULL
));
6495 completion
->wait_for_complete();
6496 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
6497 completion
->release();
6499 // head is still there...
6500 ioctx
.snap_set_read(librados::SNAP_HEAD
);
6503 ObjectReadOperation op
;
6504 op
.read(1, 0, &bl
, NULL
);
6505 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6506 ASSERT_EQ(0, ioctx
.aio_operate(
6507 "foo", completion
, &op
,
6508 librados::OPERATION_IGNORE_CACHE
, NULL
));
6509 completion
->wait_for_complete();
6510 ASSERT_EQ(0, completion
->get_return_value());
6511 completion
->release();
6514 // promote head + snap of bar
6515 ioctx
.snap_set_read(librados::SNAP_HEAD
);
6518 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
6519 ASSERT_EQ('c', bl
[0]);
6521 ioctx
.snap_set_read(my_snaps
[0]);
6524 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
6525 ASSERT_EQ('h', bl
[0]);
6528 // evict bar head (fail)
6529 ioctx
.snap_set_read(librados::SNAP_HEAD
);
6531 ObjectReadOperation op
;
6533 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6534 ASSERT_EQ(0, ioctx
.aio_operate(
6535 "bar", completion
, &op
,
6536 librados::OPERATION_IGNORE_CACHE
, NULL
));
6537 completion
->wait_for_complete();
6538 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
6539 completion
->release();
6543 ioctx
.snap_set_read(my_snaps
[0]);
6545 ObjectReadOperation op
;
6547 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6548 ASSERT_EQ(0, ioctx
.aio_operate(
6549 "bar", completion
, &op
,
6550 librados::OPERATION_IGNORE_CACHE
, NULL
));
6551 completion
->wait_for_complete();
6552 ASSERT_EQ(0, completion
->get_return_value());
6553 completion
->release();
6556 ioctx
.snap_set_read(librados::SNAP_HEAD
);
6559 ObjectReadOperation op
;
6560 op
.read(1, 0, &bl
, NULL
);
6561 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6562 ASSERT_EQ(0, ioctx
.aio_operate(
6563 "bar", completion
, &op
,
6564 librados::OPERATION_IGNORE_CACHE
, NULL
));
6565 completion
->wait_for_complete();
6566 ASSERT_EQ(0, completion
->get_return_value());
6567 completion
->release();
6570 ObjectReadOperation op
;
6572 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6573 ASSERT_EQ(0, ioctx
.aio_operate(
6574 "bar", completion
, &op
,
6575 librados::OPERATION_IGNORE_CACHE
, NULL
));
6576 completion
->wait_for_complete();
6577 ASSERT_EQ(0, completion
->get_return_value());
6578 completion
->release();
6582 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
6585 TEST_F(LibRadosTwoPoolsECPP
, TryFlush
) {
6588 ASSERT_EQ(0, cluster
.mon_command(
6589 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
6590 "\", \"tierpool\": \"" + cache_pool_name
+
6591 "\", \"force_nonempty\": \"--force-nonempty\" }",
6593 ASSERT_EQ(0, cluster
.mon_command(
6594 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
6595 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
6597 ASSERT_EQ(0, cluster
.mon_command(
6598 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
6599 "\", \"mode\": \"writeback\"}",
6602 // wait for maps to settle
6603 cluster
.wait_for_latest_osdmap();
6608 bl
.append("hi there");
6609 ObjectWriteOperation op
;
6611 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6614 // verify the object is present in the cache tier
6616 NObjectIterator it
= cache_ioctx
.nobjects_begin();
6617 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
6618 ASSERT_TRUE(it
->get_oid() == string("foo"));
6620 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
6623 // verify the object is NOT present in the base tier
6625 NObjectIterator it
= ioctx
.nobjects_begin();
6626 ASSERT_TRUE(it
== ioctx
.nobjects_end());
6633 ObjectReadOperation op
;
6634 op
.is_dirty(&dirty
, &r
);
6635 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
6642 ObjectWriteOperation op
;
6644 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6645 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
6646 completion
->wait_for_complete();
6647 ASSERT_EQ(0, completion
->get_return_value());
6648 completion
->release();
6651 // flush the pinned object with -EPERM
6653 ObjectReadOperation op
;
6654 op
.cache_try_flush();
6655 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6656 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6657 "foo", completion
, &op
,
6658 librados::OPERATION_IGNORE_OVERLAY
|
6659 librados::OPERATION_SKIPRWLOCKS
, NULL
));
6660 completion
->wait_for_complete();
6661 ASSERT_EQ(-EPERM
, completion
->get_return_value());
6662 completion
->release();
6667 ObjectWriteOperation op
;
6669 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6670 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
6671 completion
->wait_for_complete();
6672 ASSERT_EQ(0, completion
->get_return_value());
6673 completion
->release();
6678 ObjectReadOperation op
;
6679 op
.cache_try_flush();
6680 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6681 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6682 "foo", completion
, &op
,
6683 librados::OPERATION_IGNORE_OVERLAY
|
6684 librados::OPERATION_SKIPRWLOCKS
, NULL
));
6685 completion
->wait_for_complete();
6686 ASSERT_EQ(0, completion
->get_return_value());
6687 completion
->release();
6694 ObjectReadOperation op
;
6695 op
.is_dirty(&dirty
, &r
);
6696 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
6697 ASSERT_FALSE(dirty
);
6701 // verify in base tier
6703 NObjectIterator it
= ioctx
.nobjects_begin();
6704 ASSERT_TRUE(it
!= ioctx
.nobjects_end());
6705 ASSERT_TRUE(it
->get_oid() == string("foo"));
6707 ASSERT_TRUE(it
== ioctx
.nobjects_end());
6712 ObjectReadOperation op
;
6714 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6715 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6716 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
6717 completion
->wait_for_complete();
6718 ASSERT_EQ(0, completion
->get_return_value());
6719 completion
->release();
6722 // verify no longer in cache tier
6724 NObjectIterator it
= cache_ioctx
.nobjects_begin();
6725 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
6729 TEST_F(LibRadosTwoPoolsECPP
, FailedFlush
) {
6732 ASSERT_EQ(0, cluster
.mon_command(
6733 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
6734 "\", \"tierpool\": \"" + cache_pool_name
+
6735 "\", \"force_nonempty\": \"--force-nonempty\" }",
6737 ASSERT_EQ(0, cluster
.mon_command(
6738 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
6739 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
6741 ASSERT_EQ(0, cluster
.mon_command(
6742 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
6743 "\", \"mode\": \"writeback\"}",
6746 // wait for maps to settle
6747 cluster
.wait_for_latest_osdmap();
6752 bl
.append("hi there");
6753 ObjectWriteOperation op
;
6755 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6758 // verify the object is present in the cache tier
6760 NObjectIterator it
= cache_ioctx
.nobjects_begin();
6761 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
6762 ASSERT_TRUE(it
->get_oid() == string("foo"));
6764 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
6767 // verify the object is NOT present in the base tier
6769 NObjectIterator it
= ioctx
.nobjects_begin();
6770 ASSERT_TRUE(it
== ioctx
.nobjects_end());
6775 ObjectWriteOperation op
;
6776 std::map
<std::string
, bufferlist
> omap
;
6777 omap
["somekey"] = bufferlist();
6779 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6780 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
6781 completion
->wait_for_complete();
6782 ASSERT_EQ(0, completion
->get_return_value());
6783 completion
->release();
6788 ObjectReadOperation op
;
6790 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6791 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6792 "foo", completion
, &op
,
6793 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
6794 completion
->wait_for_complete();
6795 ASSERT_NE(0, completion
->get_return_value());
6796 completion
->release();
6801 ObjectReadOperation op
;
6804 std::set
<std::string
> keys
;
6805 keys
.insert("somekey");
6806 std::map
<std::string
, bufferlist
> map
;
6808 op
.omap_get_vals_by_keys(keys
, &map
, &prval
);
6809 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6810 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
, &bl
));
6812 bool completed
= completion
->is_complete();
6814 cache_ioctx
.aio_cancel(completion
);
6815 std::cerr
<< "Most probably test case will hang here, please reset manually" << std::endl
;
6816 ASSERT_TRUE(completed
); //in fact we are locked forever at test case shutdown unless fix for http://tracker.ceph.com/issues/14511 is applied. Seems there is no workaround for that
6818 completion
->release();
6820 // verify still not in base tier
6822 ASSERT_TRUE(ioctx
.nobjects_begin() == ioctx
.nobjects_end());
6826 ObjectWriteOperation op
;
6828 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6832 ObjectReadOperation op
;
6834 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6835 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6836 "foo", completion
, &op
,
6837 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
6838 completion
->wait_for_complete();
6839 ASSERT_EQ(0, completion
->get_return_value());
6840 completion
->release();
6844 ObjectReadOperation op
;
6846 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6847 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6848 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
6849 completion
->wait_for_complete();
6850 ASSERT_EQ(0, completion
->get_return_value());
6851 completion
->release();
6854 // verify no longer in cache tier
6856 NObjectIterator it
= cache_ioctx
.nobjects_begin();
6857 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
6861 NObjectIterator it
= ioctx
.nobjects_begin();
6862 ASSERT_TRUE(it
== ioctx
.nobjects_end());
6866 TEST_F(LibRadosTwoPoolsECPP
, Flush
) {
6869 ASSERT_EQ(0, cluster
.mon_command(
6870 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
6871 "\", \"tierpool\": \"" + cache_pool_name
+
6872 "\", \"force_nonempty\": \"--force-nonempty\" }",
6874 ASSERT_EQ(0, cluster
.mon_command(
6875 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
6876 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
6878 ASSERT_EQ(0, cluster
.mon_command(
6879 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
6880 "\", \"mode\": \"writeback\"}",
6883 // wait for maps to settle
6884 cluster
.wait_for_latest_osdmap();
6886 uint64_t user_version
= 0;
6891 bl
.append("hi there");
6892 ObjectWriteOperation op
;
6894 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
6897 // verify the object is present in the cache tier
6899 NObjectIterator it
= cache_ioctx
.nobjects_begin();
6900 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
6901 ASSERT_TRUE(it
->get_oid() == string("foo"));
6903 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
6906 // verify the object is NOT present in the base tier
6908 NObjectIterator it
= ioctx
.nobjects_begin();
6909 ASSERT_TRUE(it
== ioctx
.nobjects_end());
6916 ObjectReadOperation op
;
6917 op
.is_dirty(&dirty
, &r
);
6918 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
6921 user_version
= cache_ioctx
.get_last_version();
6926 ObjectWriteOperation op
;
6928 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6929 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
6930 completion
->wait_for_complete();
6931 ASSERT_EQ(0, completion
->get_return_value());
6932 completion
->release();
6935 // flush the pinned object with -EPERM
6937 ObjectReadOperation op
;
6938 op
.cache_try_flush();
6939 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6940 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6941 "foo", completion
, &op
,
6942 librados::OPERATION_IGNORE_OVERLAY
|
6943 librados::OPERATION_SKIPRWLOCKS
, NULL
));
6944 completion
->wait_for_complete();
6945 ASSERT_EQ(-EPERM
, completion
->get_return_value());
6946 completion
->release();
6951 ObjectWriteOperation op
;
6953 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6954 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
6955 completion
->wait_for_complete();
6956 ASSERT_EQ(0, completion
->get_return_value());
6957 completion
->release();
6962 ObjectReadOperation op
;
6964 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6965 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6966 "foo", completion
, &op
,
6967 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
6968 completion
->wait_for_complete();
6969 ASSERT_EQ(0, completion
->get_return_value());
6970 completion
->release();
6977 ObjectReadOperation op
;
6978 op
.is_dirty(&dirty
, &r
);
6979 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
6980 ASSERT_FALSE(dirty
);
6984 // verify in base tier
6986 NObjectIterator it
= ioctx
.nobjects_begin();
6987 ASSERT_TRUE(it
!= ioctx
.nobjects_end());
6988 ASSERT_TRUE(it
->get_oid() == string("foo"));
6990 ASSERT_TRUE(it
== ioctx
.nobjects_end());
6995 ObjectReadOperation op
;
6997 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
6998 ASSERT_EQ(0, cache_ioctx
.aio_operate(
6999 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
7000 completion
->wait_for_complete();
7001 ASSERT_EQ(0, completion
->get_return_value());
7002 completion
->release();
7005 // verify no longer in cache tier
7007 NObjectIterator it
= cache_ioctx
.nobjects_begin();
7008 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
7011 // read it again and verify the version is consistent
7014 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
7015 ASSERT_EQ(user_version
, cache_ioctx
.get_last_version());
7020 ObjectWriteOperation op
;
7022 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7027 ObjectReadOperation op
;
7029 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7030 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7031 "foo", completion
, &op
,
7032 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
7033 completion
->wait_for_complete();
7034 ASSERT_EQ(0, completion
->get_return_value());
7035 completion
->release();
7040 ObjectReadOperation op
;
7042 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7043 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7044 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
7045 completion
->wait_for_complete();
7046 ASSERT_EQ(0, completion
->get_return_value());
7047 completion
->release();
7050 // verify no longer in cache tier
7052 NObjectIterator it
= cache_ioctx
.nobjects_begin();
7053 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
7057 NObjectIterator it
= ioctx
.nobjects_begin();
7058 ASSERT_TRUE(it
== ioctx
.nobjects_end());
7062 TEST_F(LibRadosTwoPoolsECPP
, FlushSnap
) {
7065 ASSERT_EQ(0, cluster
.mon_command(
7066 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
7067 "\", \"tierpool\": \"" + cache_pool_name
+
7068 "\", \"force_nonempty\": \"--force-nonempty\" }",
7070 ASSERT_EQ(0, cluster
.mon_command(
7071 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
7072 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
7074 ASSERT_EQ(0, cluster
.mon_command(
7075 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
7076 "\", \"mode\": \"writeback\"}",
7079 // wait for maps to settle
7080 cluster
.wait_for_latest_osdmap();
7086 ObjectWriteOperation op
;
7088 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7091 // create a snapshot, clone
7092 vector
<uint64_t> my_snaps(1);
7093 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
7094 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
7099 ObjectWriteOperation op
;
7101 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7106 my_snaps
[1] = my_snaps
[0];
7107 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
7108 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
7113 ObjectWriteOperation op
;
7115 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7118 // verify the object is present in the cache tier
7120 NObjectIterator it
= cache_ioctx
.nobjects_begin();
7121 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
7122 ASSERT_TRUE(it
->get_oid() == string("foo"));
7124 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
7127 // verify the object is NOT present in the base tier
7129 NObjectIterator it
= ioctx
.nobjects_begin();
7130 ASSERT_TRUE(it
== ioctx
.nobjects_end());
7133 // flush on head (should fail)
7134 ioctx
.snap_set_read(librados::SNAP_HEAD
);
7136 ObjectReadOperation op
;
7138 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7139 ASSERT_EQ(0, ioctx
.aio_operate(
7140 "foo", completion
, &op
,
7141 librados::OPERATION_IGNORE_CACHE
, NULL
));
7142 completion
->wait_for_complete();
7143 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
7144 completion
->release();
7146 // flush on recent snap (should fail)
7147 ioctx
.snap_set_read(my_snaps
[0]);
7149 ObjectReadOperation op
;
7151 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7152 ASSERT_EQ(0, ioctx
.aio_operate(
7153 "foo", completion
, &op
,
7154 librados::OPERATION_IGNORE_CACHE
, NULL
));
7155 completion
->wait_for_complete();
7156 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
7157 completion
->release();
7159 // flush on oldest snap
7160 ioctx
.snap_set_read(my_snaps
[1]);
7162 ObjectReadOperation op
;
7164 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7165 ASSERT_EQ(0, ioctx
.aio_operate(
7166 "foo", completion
, &op
,
7167 librados::OPERATION_IGNORE_CACHE
, NULL
));
7168 completion
->wait_for_complete();
7169 ASSERT_EQ(0, completion
->get_return_value());
7170 completion
->release();
7172 // flush on next oldest snap
7173 ioctx
.snap_set_read(my_snaps
[0]);
7175 ObjectReadOperation op
;
7177 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7178 ASSERT_EQ(0, ioctx
.aio_operate(
7179 "foo", completion
, &op
,
7180 librados::OPERATION_IGNORE_CACHE
, NULL
));
7181 completion
->wait_for_complete();
7182 ASSERT_EQ(0, completion
->get_return_value());
7183 completion
->release();
7186 ioctx
.snap_set_read(librados::SNAP_HEAD
);
7188 ObjectReadOperation op
;
7190 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7191 ASSERT_EQ(0, ioctx
.aio_operate(
7192 "foo", completion
, &op
,
7193 librados::OPERATION_IGNORE_CACHE
, NULL
));
7194 completion
->wait_for_complete();
7195 ASSERT_EQ(0, completion
->get_return_value());
7196 completion
->release();
7199 // verify i can read the snaps from the cache pool
7200 ioctx
.snap_set_read(librados::SNAP_HEAD
);
7203 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
7204 ASSERT_EQ('c', bl
[0]);
7206 ioctx
.snap_set_read(my_snaps
[0]);
7209 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
7210 ASSERT_EQ('b', bl
[0]);
7212 ioctx
.snap_set_read(my_snaps
[1]);
7215 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
7216 ASSERT_EQ('a', bl
[0]);
7220 ASSERT_EQ(0, cluster
.mon_command(
7221 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
7225 // wait for maps to settle
7226 cluster
.wait_for_latest_osdmap();
7228 // verify i can read the snaps from the base pool
7229 ioctx
.snap_set_read(librados::SNAP_HEAD
);
7232 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
7233 ASSERT_EQ('c', bl
[0]);
7235 ioctx
.snap_set_read(my_snaps
[0]);
7238 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
7239 ASSERT_EQ('b', bl
[0]);
7241 ioctx
.snap_set_read(my_snaps
[1]);
7244 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
7245 ASSERT_EQ('a', bl
[0]);
7248 ASSERT_EQ(0, cluster
.mon_command(
7249 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
7250 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
7252 cluster
.wait_for_latest_osdmap();
7255 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
7258 TEST_F(LibRadosTierECPP
, FlushWriteRaces
) {
7260 std::string pool_name
= get_temp_pool_name();
7261 std::string cache_pool_name
= pool_name
+ "-cache";
7262 ASSERT_EQ("", create_one_pool_pp(pool_name
, cluster
));
7263 ASSERT_EQ(0, cluster
.pool_create(cache_pool_name
.c_str()));
7265 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
7266 cache_ioctx
.application_enable("rados", true);
7268 ASSERT_EQ(0, cluster
.ioctx_create(pool_name
.c_str(), ioctx
));
7272 ASSERT_EQ(0, cluster
.mon_command(
7273 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
7274 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
7276 ASSERT_EQ(0, cluster
.mon_command(
7277 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
7278 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
7280 ASSERT_EQ(0, cluster
.mon_command(
7281 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
7282 "\", \"mode\": \"writeback\"}",
7285 // wait for maps to settle
7286 cluster
.wait_for_latest_osdmap();
7288 // create/dirty object
7290 bl
.append("hi there");
7292 ObjectWriteOperation op
;
7294 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7299 ObjectReadOperation op
;
7301 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7302 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7303 "foo", completion
, &op
,
7304 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
7306 ObjectWriteOperation op2
;
7308 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
7309 ASSERT_EQ(0, ioctx
.aio_operate(
7310 "foo", completion2
, &op2
, 0));
7312 completion
->wait_for_complete();
7313 completion2
->wait_for_complete();
7314 ASSERT_EQ(0, completion
->get_return_value());
7315 ASSERT_EQ(0, completion2
->get_return_value());
7316 completion
->release();
7317 completion2
->release();
7322 // create/dirty object
7325 bl
.append("hi there");
7326 ObjectWriteOperation op
;
7328 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7331 // try-flush + write
7333 ObjectReadOperation op
;
7334 op
.cache_try_flush();
7335 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7336 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7337 "foo", completion
, &op
,
7338 librados::OPERATION_IGNORE_OVERLAY
|
7339 librados::OPERATION_SKIPRWLOCKS
, NULL
));
7341 ObjectWriteOperation op2
;
7343 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
7344 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion2
, &op2
, 0));
7346 completion
->wait_for_complete();
7347 completion2
->wait_for_complete();
7348 int r
= completion
->get_return_value();
7349 ASSERT_TRUE(r
== -EBUSY
|| r
== 0);
7350 ASSERT_EQ(0, completion2
->get_return_value());
7351 completion
->release();
7352 completion2
->release();
7355 cout
<< "didn't get EBUSY, trying again" << std::endl
;
7357 ASSERT_TRUE(--tries
);
7361 ASSERT_EQ(0, cluster
.mon_command(
7362 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
7365 ASSERT_EQ(0, cluster
.mon_command(
7366 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
7367 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
7370 // wait for maps to settle before next test
7371 cluster
.wait_for_latest_osdmap();
7373 ASSERT_EQ(0, cluster
.pool_delete(cache_pool_name
.c_str()));
7374 ASSERT_EQ(0, destroy_one_pool_pp(pool_name
, cluster
));
7377 TEST_F(LibRadosTwoPoolsECPP
, FlushTryFlushRaces
) {
7380 ASSERT_EQ(0, cluster
.mon_command(
7381 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
7382 "\", \"tierpool\": \"" + cache_pool_name
+
7383 "\", \"force_nonempty\": \"--force-nonempty\" }",
7385 ASSERT_EQ(0, cluster
.mon_command(
7386 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
7387 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
7389 ASSERT_EQ(0, cluster
.mon_command(
7390 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
7391 "\", \"mode\": \"writeback\"}",
7394 // wait for maps to settle
7395 cluster
.wait_for_latest_osdmap();
7397 // create/dirty object
7400 bl
.append("hi there");
7401 ObjectWriteOperation op
;
7403 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7408 ObjectReadOperation op
;
7410 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7411 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7412 "foo", completion
, &op
,
7413 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
7415 ObjectReadOperation op2
;
7417 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
7418 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7419 "foo", completion2
, &op2
,
7420 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
7422 completion
->wait_for_complete();
7423 completion2
->wait_for_complete();
7424 ASSERT_EQ(0, completion
->get_return_value());
7425 ASSERT_EQ(0, completion2
->get_return_value());
7426 completion
->release();
7427 completion2
->release();
7430 // create/dirty object
7433 bl
.append("hi there");
7434 ObjectWriteOperation op
;
7436 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7439 // flush + try-flush
7441 ObjectReadOperation op
;
7443 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7444 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7445 "foo", completion
, &op
,
7446 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
7448 ObjectReadOperation op2
;
7449 op2
.cache_try_flush();
7450 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
7451 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7452 "foo", completion2
, &op2
,
7453 librados::OPERATION_IGNORE_OVERLAY
|
7454 librados::OPERATION_SKIPRWLOCKS
, NULL
));
7456 completion
->wait_for_complete();
7457 completion2
->wait_for_complete();
7458 ASSERT_EQ(0, completion
->get_return_value());
7459 ASSERT_EQ(0, completion2
->get_return_value());
7460 completion
->release();
7461 completion2
->release();
7464 // create/dirty object
7469 bl
.append("hi there");
7470 ObjectWriteOperation op
;
7472 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7475 // try-flush + flush
7476 // (flush will not piggyback on try-flush)
7478 ObjectReadOperation op
;
7479 op
.cache_try_flush();
7480 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7481 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7482 "foo", completion
, &op
,
7483 librados::OPERATION_IGNORE_OVERLAY
|
7484 librados::OPERATION_SKIPRWLOCKS
, NULL
));
7486 ObjectReadOperation op2
;
7488 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
7489 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7490 "foo", completion2
, &op2
,
7491 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
7493 completion
->wait_for_complete();
7494 completion2
->wait_for_complete();
7495 int r
= completion
->get_return_value();
7496 ASSERT_TRUE(r
== -EBUSY
|| r
== 0);
7497 ASSERT_EQ(0, completion2
->get_return_value());
7498 completion
->release();
7499 completion2
->release();
7502 cout
<< "didn't get EBUSY, trying again" << std::endl
;
7504 ASSERT_TRUE(--tries
);
7507 // create/dirty object
7510 bl
.append("hi there");
7511 ObjectWriteOperation op
;
7513 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7516 // try-flush + try-flush
7518 ObjectReadOperation op
;
7519 op
.cache_try_flush();
7520 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7521 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7522 "foo", completion
, &op
,
7523 librados::OPERATION_IGNORE_OVERLAY
|
7524 librados::OPERATION_SKIPRWLOCKS
, NULL
));
7526 ObjectReadOperation op2
;
7527 op2
.cache_try_flush();
7528 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
7529 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7530 "foo", completion2
, &op2
,
7531 librados::OPERATION_IGNORE_OVERLAY
|
7532 librados::OPERATION_SKIPRWLOCKS
, NULL
));
7534 completion
->wait_for_complete();
7535 completion2
->wait_for_complete();
7536 ASSERT_EQ(0, completion
->get_return_value());
7537 ASSERT_EQ(0, completion2
->get_return_value());
7538 completion
->release();
7539 completion2
->release();
7543 TEST_F(LibRadosTwoPoolsECPP
, TryFlushReadRace
) {
7546 ASSERT_EQ(0, cluster
.mon_command(
7547 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
7548 "\", \"tierpool\": \"" + cache_pool_name
+
7549 "\", \"force_nonempty\": \"--force-nonempty\" }",
7551 ASSERT_EQ(0, cluster
.mon_command(
7552 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
7553 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
7555 ASSERT_EQ(0, cluster
.mon_command(
7556 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
7557 "\", \"mode\": \"writeback\"}",
7560 // wait for maps to settle
7561 cluster
.wait_for_latest_osdmap();
7563 // create/dirty object
7566 bl
.append("hi there");
7567 bufferptr
bp(4000000); // make it big!
7570 ObjectWriteOperation op
;
7572 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7575 // start a continuous stream of reads
7576 read_ioctx
= &ioctx
;
7578 for (int i
= 0; i
< max_reads
; ++i
) {
7585 ObjectReadOperation op
;
7586 op
.cache_try_flush();
7587 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7588 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7589 "foo", completion
, &op
,
7590 librados::OPERATION_IGNORE_OVERLAY
|
7591 librados::OPERATION_SKIPRWLOCKS
, NULL
));
7593 completion
->wait_for_complete();
7594 ASSERT_EQ(0, completion
->get_return_value());
7595 completion
->release();
7598 std::unique_lock locker
{test_lock
};
7600 cond
.wait(locker
, [] { return num_reads
== 0;});
7603 TEST_F(LibRadosTierECPP
, CallForcesPromote
) {
7605 std::string pool_name
= get_temp_pool_name();
7606 std::string cache_pool_name
= pool_name
+ "-cache";
7607 ASSERT_EQ("", create_one_ec_pool_pp(pool_name
, cluster
));
7608 ASSERT_EQ(0, cluster
.pool_create(cache_pool_name
.c_str()));
7610 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
7611 cache_ioctx
.application_enable("rados", true);
7613 ASSERT_EQ(0, cluster
.ioctx_create(pool_name
.c_str(), ioctx
));
7617 ASSERT_EQ(0, cluster
.mon_command(
7618 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
7619 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
7621 ASSERT_EQ(0, cluster
.mon_command(
7622 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
7623 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
7625 ASSERT_EQ(0, cluster
.mon_command(
7626 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
7627 "\", \"mode\": \"writeback\"}",
7630 // set things up such that the op would normally be proxied
7631 ASSERT_EQ(0, cluster
.mon_command(
7632 set_pool_str(cache_pool_name
, "hit_set_count", 2),
7634 ASSERT_EQ(0, cluster
.mon_command(
7635 set_pool_str(cache_pool_name
, "hit_set_period", 600),
7637 ASSERT_EQ(0, cluster
.mon_command(
7638 set_pool_str(cache_pool_name
, "hit_set_type",
7641 ASSERT_EQ(0, cluster
.mon_command(
7642 set_pool_str(cache_pool_name
, "min_read_recency_for_promote",
7646 // wait for maps to settle
7647 cluster
.wait_for_latest_osdmap();
7649 // create/dirty object
7651 bl
.append("hi there");
7653 ObjectWriteOperation op
;
7655 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
7660 ObjectReadOperation op
;
7662 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7663 ASSERT_EQ(0, cache_ioctx
.aio_operate(
7664 "foo", completion
, &op
,
7665 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
7666 completion
->wait_for_complete();
7667 ASSERT_EQ(0, completion
->get_return_value());
7668 completion
->release();
7673 ObjectReadOperation op
;
7675 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
7676 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
7677 librados::OPERATION_IGNORE_CACHE
,
7679 completion
->wait_for_complete();
7680 ASSERT_EQ(0, completion
->get_return_value());
7681 completion
->release();
7686 ObjectReadOperation op
;
7688 op
.exec("rbd", "get_id", bl
);
7690 // should get EIO (not an rbd object), not -EOPNOTSUPP (we didn't promote)
7691 ASSERT_EQ(-5, ioctx
.operate("foo", &op
, &out
));
7694 // make sure foo is back in the cache tier
7696 NObjectIterator it
= cache_ioctx
.nobjects_begin();
7697 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
7698 ASSERT_TRUE(it
->get_oid() == string("foo"));
7700 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
7704 ASSERT_EQ(0, cluster
.mon_command(
7705 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
7708 ASSERT_EQ(0, cluster
.mon_command(
7709 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
7710 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
7713 // wait for maps to settle before next test
7714 cluster
.wait_for_latest_osdmap();
7716 ASSERT_EQ(0, cluster
.pool_delete(cache_pool_name
.c_str()));
7717 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name
, cluster
));
7720 TEST_F(LibRadosTierECPP
, HitSetNone
) {
7722 list
< pair
<time_t,time_t> > ls
;
7723 AioCompletion
*c
= librados::Rados::aio_create_completion();
7724 ASSERT_EQ(0, ioctx
.hit_set_list(123, c
, &ls
));
7725 c
->wait_for_complete();
7726 ASSERT_EQ(0, c
->get_return_value());
7727 ASSERT_TRUE(ls
.empty());
7732 AioCompletion
*c
= librados::Rados::aio_create_completion();
7733 ASSERT_EQ(0, ioctx
.hit_set_get(123, c
, 12345, &bl
));
7734 c
->wait_for_complete();
7735 ASSERT_EQ(-ENOENT
, c
->get_return_value());
7740 TEST_F(LibRadosTwoPoolsECPP
, HitSetRead
) {
7743 ASSERT_EQ(0, cluster
.mon_command(
7744 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
7745 "\", \"tierpool\": \"" + cache_pool_name
+
7746 "\", \"force_nonempty\": \"--force-nonempty\" }",
7749 // enable hitset tracking for this pool
7750 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", 2),
7752 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", 600),
7754 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type",
7758 // wait for maps to settle
7759 cluster
.wait_for_latest_osdmap();
7761 cache_ioctx
.set_namespace("");
7763 // keep reading until we see our object appear in the HitSet
7764 utime_t start
= ceph_clock_now();
7765 utime_t hard_stop
= start
+ utime_t(600, 0);
7768 utime_t now
= ceph_clock_now();
7769 ASSERT_TRUE(now
< hard_stop
);
7771 string name
= "foo";
7773 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(name
, &hash
));
7774 hobject_t
oid(sobject_t(name
, CEPH_NOSNAP
), "", hash
,
7775 cluster
.pool_lookup(cache_pool_name
.c_str()), "");
7778 ASSERT_EQ(-ENOENT
, cache_ioctx
.read("foo", bl
, 1, 0));
7781 AioCompletion
*c
= librados::Rados::aio_create_completion();
7782 ASSERT_EQ(0, cache_ioctx
.hit_set_get(hash
, c
, now
.sec(), &hbl
));
7783 c
->wait_for_complete();
7787 auto p
= hbl
.cbegin();
7790 if (hs
.contains(oid
)) {
7791 cout
<< "ok, hit_set contains " << oid
<< std::endl
;
7794 cout
<< "hmm, not in HitSet yet" << std::endl
;
7796 cout
<< "hmm, no HitSet yet" << std::endl
;
7803 // disable this test until hitset-get reliably works on EC pools
7805 TEST_F(LibRadosTierECPP
, HitSetWrite
) {
7806 int num_pg
= _get_pg_num(cluster
, pool_name
);
7807 ceph_assert(num_pg
> 0);
7809 // enable hitset tracking for this pool
7811 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(pool_name
, "hit_set_count", 8),
7813 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(pool_name
, "hit_set_period", 600),
7815 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(pool_name
, "hit_set_type",
7819 // wait for maps to settle
7820 cluster
.wait_for_latest_osdmap();
7822 ioctx
.set_namespace("");
7824 // do a bunch of writes
7825 for (int i
=0; i
<1000; ++i
) {
7828 ASSERT_EQ(0, ioctx
.write(stringify(i
), bl
, 1, 0));
7832 std::map
<int,HitSet
> hitsets
;
7833 for (int i
=0; i
<num_pg
; ++i
) {
7834 list
< pair
<time_t,time_t> > ls
;
7835 AioCompletion
*c
= librados::Rados::aio_create_completion();
7836 ASSERT_EQ(0, ioctx
.hit_set_list(i
, c
, &ls
));
7837 c
->wait_for_complete();
7839 std::cout
<< "pg " << i
<< " ls " << ls
<< std::endl
;
7840 ASSERT_FALSE(ls
.empty());
7843 c
= librados::Rados::aio_create_completion();
7845 ASSERT_EQ(0, ioctx
.hit_set_get(i
, c
, ls
.back().first
, &bl
));
7846 c
->wait_for_complete();
7849 //std::cout << "bl len is " << bl.length() << "\n";
7850 //bl.hexdump(std::cout);
7851 //std::cout << std::endl;
7853 auto p
= bl
.cbegin();
7854 decode(hitsets
[i
], p
);
7856 // cope with racing splits by refreshing pg_num
7857 if (i
== num_pg
- 1)
7858 num_pg
= _get_pg_num(cluster
, pool_name
);
7861 for (int i
=0; i
<1000; ++i
) {
7862 string n
= stringify(i
);
7863 uint32_t hash
= ioctx
.get_object_hash_position(n
);
7864 hobject_t
oid(sobject_t(n
, CEPH_NOSNAP
), "", hash
,
7865 cluster
.pool_lookup(pool_name
.c_str()), "");
7866 std::cout
<< "checking for " << oid
<< std::endl
;
7868 for (int p
=0; p
<num_pg
; ++p
) {
7869 if (hitsets
[p
].contains(oid
)) {
7879 TEST_F(LibRadosTwoPoolsECPP
, HitSetTrim
) {
7881 unsigned period
= 3;
7885 ASSERT_EQ(0, cluster
.mon_command(
7886 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
7887 "\", \"tierpool\": \"" + cache_pool_name
+
7888 "\", \"force_nonempty\": \"--force-nonempty\" }",
7891 // enable hitset tracking for this pool
7892 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", count
),
7894 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", period
),
7896 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
7898 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_fpp", ".01"),
7901 // wait for maps to settle
7902 cluster
.wait_for_latest_osdmap();
7904 cache_ioctx
.set_namespace("");
7906 // do a bunch of writes and make sure the hitsets rotate
7907 utime_t start
= ceph_clock_now();
7908 utime_t hard_stop
= start
+ utime_t(count
* period
* 50, 0);
7911 int bsize
= alignment
;
7912 char *buf
= (char *)new char[bsize
];
7913 memset(buf
, 'f', bsize
);
7916 string name
= "foo";
7918 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(name
, &hash
));
7919 hobject_t
oid(sobject_t(name
, CEPH_NOSNAP
), "", hash
, -1, "");
7922 bl
.append(buf
, bsize
);
7923 ASSERT_EQ(0, cache_ioctx
.append("foo", bl
, bsize
));
7925 list
<pair
<time_t, time_t> > ls
;
7926 AioCompletion
*c
= librados::Rados::aio_create_completion();
7927 ASSERT_EQ(0, cache_ioctx
.hit_set_list(hash
, c
, &ls
));
7928 c
->wait_for_complete();
7931 cout
<< " got ls " << ls
<< std::endl
;
7934 first
= ls
.front().first
;
7935 cout
<< "first is " << first
<< std::endl
;
7937 if (ls
.front().first
!= first
) {
7938 cout
<< "first now " << ls
.front().first
<< ", trimmed" << std::endl
;
7944 utime_t now
= ceph_clock_now();
7945 ASSERT_TRUE(now
< hard_stop
);
7952 TEST_F(LibRadosTwoPoolsECPP
, PromoteOn2ndRead
) {
7954 for (int i
=0; i
<20; ++i
) {
7956 bl
.append("hi there");
7957 ObjectWriteOperation op
;
7959 ASSERT_EQ(0, ioctx
.operate("foo" + stringify(i
), &op
));
7964 ASSERT_EQ(0, cluster
.mon_command(
7965 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
7966 "\", \"tierpool\": \"" + cache_pool_name
+
7967 "\", \"force_nonempty\": \"--force-nonempty\" }",
7969 ASSERT_EQ(0, cluster
.mon_command(
7970 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
7971 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
7973 ASSERT_EQ(0, cluster
.mon_command(
7974 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
7975 "\", \"mode\": \"writeback\"}",
7978 // enable hitset tracking for this pool
7979 ASSERT_EQ(0, cluster
.mon_command(
7980 set_pool_str(cache_pool_name
, "hit_set_count", 2),
7982 ASSERT_EQ(0, cluster
.mon_command(
7983 set_pool_str(cache_pool_name
, "hit_set_period", 600),
7985 ASSERT_EQ(0, cluster
.mon_command(
7986 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
7988 ASSERT_EQ(0, cluster
.mon_command(
7989 set_pool_str(cache_pool_name
, "min_read_recency_for_promote", 1),
7991 ASSERT_EQ(0, cluster
.mon_command(
7992 set_pool_str(cache_pool_name
, "hit_set_grade_decay_rate", 20),
7994 ASSERT_EQ(0, cluster
.mon_command(
7995 set_pool_str(cache_pool_name
, "hit_set_search_last_n", 1),
7998 // wait for maps to settle
7999 cluster
.wait_for_latest_osdmap();
8001 int fake
= 0; // set this to non-zero to test spurious promotion,
8002 // e.g. from thrashing
8006 // 1st read, don't trigger a promote
8007 obj
= "foo" + stringify(attempt
);
8008 cout
<< obj
<< std::endl
;
8011 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
8014 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
8019 // verify the object is NOT present in the cache tier
8022 NObjectIterator it
= cache_ioctx
.nobjects_begin();
8023 while (it
!= cache_ioctx
.nobjects_end()) {
8024 cout
<< " see " << it
->get_oid() << std::endl
;
8025 if (it
->get_oid() == string(obj
.c_str())) {
8036 ASSERT_LE(attempt
, 20);
8037 cout
<< "hrm, object is present in cache on attempt " << attempt
8038 << ", retrying" << std::endl
;
8041 // Read until the object is present in the cache tier
8042 cout
<< "verifying " << obj
<< " is eventually promoted" << std::endl
;
8045 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
8048 NObjectIterator it
= cache_ioctx
.nobjects_begin();
8049 while (it
!= cache_ioctx
.nobjects_end()) {
8050 if (it
->get_oid() == string(obj
.c_str())) {
8063 ASSERT_EQ(0, cluster
.mon_command(
8064 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
8067 ASSERT_EQ(0, cluster
.mon_command(
8068 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
8069 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
8072 // wait for maps to settle before next test
8073 cluster
.wait_for_latest_osdmap();
8076 TEST_F(LibRadosTwoPoolsECPP
, ProxyRead
) {
8080 bl
.append("hi there");
8081 ObjectWriteOperation op
;
8083 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
8088 ASSERT_EQ(0, cluster
.mon_command(
8089 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
8090 "\", \"tierpool\": \"" + cache_pool_name
+
8091 "\", \"force_nonempty\": \"--force-nonempty\" }",
8093 ASSERT_EQ(0, cluster
.mon_command(
8094 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
8095 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
8097 ASSERT_EQ(0, cluster
.mon_command(
8098 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
8099 "\", \"mode\": \"readproxy\"}",
8102 // wait for maps to settle
8103 cluster
.wait_for_latest_osdmap();
8105 // read and verify the object
8108 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
8109 ASSERT_EQ('h', bl
[0]);
8112 // Verify 10 times the object is NOT present in the cache tier
8115 NObjectIterator it
= cache_ioctx
.nobjects_begin();
8116 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
8121 ASSERT_EQ(0, cluster
.mon_command(
8122 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
8125 ASSERT_EQ(0, cluster
.mon_command(
8126 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
8127 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
8130 // wait for maps to settle before next test
8131 cluster
.wait_for_latest_osdmap();
8134 TEST_F(LibRadosTwoPoolsECPP
, CachePin
) {
8138 bl
.append("hi there");
8139 ObjectWriteOperation op
;
8141 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
8145 bl
.append("hi there");
8146 ObjectWriteOperation op
;
8148 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
8152 bl
.append("hi there");
8153 ObjectWriteOperation op
;
8155 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
8159 bl
.append("hi there");
8160 ObjectWriteOperation op
;
8162 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
8167 ASSERT_EQ(0, cluster
.mon_command(
8168 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
8169 "\", \"tierpool\": \"" + cache_pool_name
+
8170 "\", \"force_nonempty\": \"--force-nonempty\" }",
8172 ASSERT_EQ(0, cluster
.mon_command(
8173 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
8174 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
8176 ASSERT_EQ(0, cluster
.mon_command(
8177 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
8178 "\", \"mode\": \"writeback\"}",
8181 // wait for maps to settle
8182 cluster
.wait_for_latest_osdmap();
8184 // read, trigger promote
8187 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
8188 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
8189 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
8190 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
8193 // verify the objects are present in the cache tier
8195 NObjectIterator it
= cache_ioctx
.nobjects_begin();
8196 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
8197 for (uint32_t i
= 0; i
< 4; i
++) {
8198 ASSERT_TRUE(it
->get_oid() == string("foo") ||
8199 it
->get_oid() == string("bar") ||
8200 it
->get_oid() == string("baz") ||
8201 it
->get_oid() == string("bam"));
8204 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
8209 ObjectWriteOperation op
;
8211 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8212 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
8213 completion
->wait_for_complete();
8214 ASSERT_EQ(0, completion
->get_return_value());
8215 completion
->release();
8218 ObjectWriteOperation op
;
8220 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8221 ASSERT_EQ(0, cache_ioctx
.aio_operate("baz", completion
, &op
));
8222 completion
->wait_for_complete();
8223 ASSERT_EQ(0, completion
->get_return_value());
8224 completion
->release();
8228 ASSERT_EQ(0, cluster
.mon_command(
8229 set_pool_str(cache_pool_name
, "hit_set_count", 2),
8231 ASSERT_EQ(0, cluster
.mon_command(
8232 set_pool_str(cache_pool_name
, "hit_set_period", 600),
8234 ASSERT_EQ(0, cluster
.mon_command(
8235 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
8237 ASSERT_EQ(0, cluster
.mon_command(
8238 set_pool_str(cache_pool_name
, "min_read_recency_for_promote", 1),
8240 ASSERT_EQ(0, cluster
.mon_command(
8241 set_pool_str(cache_pool_name
, "target_max_objects", 1),
8246 // Verify the pinned object 'foo' is not flushed/evicted
8250 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
8253 NObjectIterator it
= cache_ioctx
.nobjects_begin();
8254 while (it
!= cache_ioctx
.nobjects_end()) {
8255 ASSERT_TRUE(it
->get_oid() == string("foo") ||
8256 it
->get_oid() == string("bar") ||
8257 it
->get_oid() == string("baz") ||
8258 it
->get_oid() == string("bam"));
8263 ASSERT_TRUE(it
->get_oid() == string("foo") ||
8264 it
->get_oid() == string("baz"));
8272 ASSERT_EQ(0, cluster
.mon_command(
8273 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
8276 ASSERT_EQ(0, cluster
.mon_command(
8277 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
8278 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
8281 // wait for maps to settle before next test
8282 cluster
.wait_for_latest_osdmap();
8284 TEST_F(LibRadosTwoPoolsECPP
, SetRedirectRead
) {
8288 bl
.append("hi there");
8289 ObjectWriteOperation op
;
8291 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
8296 ObjectWriteOperation op
;
8298 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
8303 ASSERT_EQ(0, cluster
.mon_command(
8304 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
8305 "\", \"tierpool\": \"" + cache_pool_name
+
8306 "\", \"force_nonempty\": \"--force-nonempty\" }",
8309 // wait for maps to settle
8310 cluster
.wait_for_latest_osdmap();
8313 ObjectWriteOperation op
;
8314 op
.set_redirect("bar", cache_ioctx
, 0);
8315 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8316 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
8317 completion
->wait_for_complete();
8318 ASSERT_EQ(0, completion
->get_return_value());
8319 completion
->release();
8321 // read and verify the object
8324 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
8325 ASSERT_EQ('t', bl
[0]);
8328 ASSERT_EQ(0, cluster
.mon_command(
8329 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
8330 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
8333 // wait for maps to settle before next test
8334 cluster
.wait_for_latest_osdmap();
8337 TEST_F(LibRadosTwoPoolsECPP
, SetChunkRead
) {
8338 // note: require >= mimic
8342 bl
.append("there hi");
8343 ObjectWriteOperation op
;
8345 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
8350 bl
.append("There hi");
8351 ObjectWriteOperation op
;
8353 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
8356 // wait for maps to settle
8357 cluster
.wait_for_latest_osdmap();
8360 manifest_set_chunk(cluster
, ioctx
, cache_ioctx
, 0, 4, "bar", "foo");
8364 ObjectWriteOperation op
;
8366 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8367 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
8368 completion
->wait_for_complete();
8369 ASSERT_EQ(0, completion
->get_return_value());
8370 completion
->release();
8373 // read and verify the object
8376 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
8377 ASSERT_EQ('T', bl
[0]);
8380 // wait for maps to settle before next test
8381 cluster
.wait_for_latest_osdmap();
8384 TEST_F(LibRadosTwoPoolsECPP
, ManifestPromoteRead
) {
8385 // note: require >= mimic
8390 bl
.append("hiaa there");
8391 ObjectWriteOperation op
;
8393 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
8397 bl
.append("base chunk");
8398 ObjectWriteOperation op
;
8400 ASSERT_EQ(0, cache_ioctx
.operate("foo-chunk", &op
));
8404 bl
.append("HIaa there");
8405 ObjectWriteOperation op
;
8407 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
8411 bl
.append("BASE CHUNK");
8412 ObjectWriteOperation op
;
8414 ASSERT_EQ(0, ioctx
.operate("bar-chunk", &op
));
8419 ObjectWriteOperation op
;
8420 op
.set_redirect("bar", ioctx
, 0);
8421 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8422 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
8423 completion
->wait_for_complete();
8424 ASSERT_EQ(0, completion
->get_return_value());
8425 completion
->release();
8428 manifest_set_chunk(cluster
, ioctx
, cache_ioctx
, 0, 10, "bar-chunk", "foo-chunk");
8431 ObjectWriteOperation op
;
8433 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8434 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
8435 completion
->wait_for_complete();
8436 ASSERT_EQ(0, completion
->get_return_value());
8437 completion
->release();
8439 // read and verify the object (redirect)
8442 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
8443 ASSERT_EQ('H', bl
[0]);
8447 ObjectWriteOperation op
;
8449 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8450 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo-chunk", completion
, &op
));
8451 completion
->wait_for_complete();
8452 ASSERT_EQ(0, completion
->get_return_value());
8453 completion
->release();
8455 // read and verify the object
8458 ASSERT_EQ(1, cache_ioctx
.read("foo-chunk", bl
, 1, 0));
8459 ASSERT_EQ('B', bl
[0]);
8462 // wait for maps to settle before next test
8463 cluster
.wait_for_latest_osdmap();
8466 TEST_F(LibRadosTwoPoolsECPP
, TrySetDedupTier
) {
8467 // note: require >= mimic
8470 ASSERT_EQ(-EOPNOTSUPP
, cluster
.mon_command(
8471 set_pool_str(pool_name
, "dedup_tier", cache_pool_name
),
8475 TEST_F(LibRadosTwoPoolsPP
, PropagateBaseTierError
) {
8476 // write object to base tier
8478 encode(static_cast<uint32_t>(0U), omap_bl
);
8480 ObjectWriteOperation op1
;
8481 op1
.omap_set({{"somekey", omap_bl
}});
8482 ASSERT_EQ(0, ioctx
.operate("propagate-base-tier-error", &op1
));
8486 ASSERT_EQ(0, cluster
.mon_command(
8487 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
8488 "\", \"tierpool\": \"" + cache_pool_name
+
8489 "\", \"force_nonempty\": \"--force-nonempty\" }",
8491 ASSERT_EQ(0, cluster
.mon_command(
8492 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
8493 "\", \"mode\": \"writeback\"}",
8495 ASSERT_EQ(0, cluster
.mon_command(
8496 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
8497 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
8500 ASSERT_EQ(0, cluster
.mon_command(
8501 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
8503 ASSERT_EQ(0, cluster
.mon_command(
8504 set_pool_str(cache_pool_name
, "hit_set_count", 1),
8506 ASSERT_EQ(0, cluster
.mon_command(
8507 set_pool_str(cache_pool_name
, "hit_set_period", 600),
8509 ASSERT_EQ(0, cluster
.mon_command(
8510 set_pool_str(cache_pool_name
, "target_max_objects", 250),
8513 // wait for maps to settle
8514 cluster
.wait_for_latest_osdmap();
8516 // guarded op should fail so expect error to propagate to cache tier
8517 bufferlist test_omap_bl
;
8518 encode(static_cast<uint32_t>(1U), test_omap_bl
);
8520 ObjectWriteOperation op2
;
8521 op2
.omap_cmp({{"somekey", {test_omap_bl
, CEPH_OSD_CMPXATTR_OP_EQ
}}}, nullptr);
8522 op2
.omap_set({{"somekey", test_omap_bl
}});
8524 ASSERT_EQ(-ECANCELED
, ioctx
.operate("propagate-base-tier-error", &op2
));
8527 TEST_F(LibRadosTwoPoolsPP
, HelloWriteReturn
) {
8530 ASSERT_EQ(0, cluster
.mon_command(
8531 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
8532 "\", \"tierpool\": \"" + cache_pool_name
+
8533 "\", \"force_nonempty\": \"--force-nonempty\" }",
8535 ASSERT_EQ(0, cluster
.mon_command(
8536 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
8537 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
8539 ASSERT_EQ(0, cluster
.mon_command(
8540 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
8541 "\", \"mode\": \"writeback\"}",
8544 // set things up such that the op would normally be proxied
8545 ASSERT_EQ(0, cluster
.mon_command(
8546 set_pool_str(cache_pool_name
, "hit_set_count", 2),
8548 ASSERT_EQ(0, cluster
.mon_command(
8549 set_pool_str(cache_pool_name
, "hit_set_period", 600),
8551 ASSERT_EQ(0, cluster
.mon_command(
8552 set_pool_str(cache_pool_name
, "hit_set_type",
8555 ASSERT_EQ(0, cluster
.mon_command(
8556 set_pool_str(cache_pool_name
, "min_read_recency_for_promote",
8560 // wait for maps to settle
8561 cluster
.wait_for_latest_osdmap();
8563 // this *will* return data due to the RETURNVEC flag
8567 ObjectWriteOperation o
;
8568 o
.exec("hello", "write_return_data", in
, &out
, &rval
);
8569 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8570 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &o
,
8571 librados::OPERATION_RETURNVEC
));
8572 completion
->wait_for_complete();
8573 ASSERT_EQ(42, completion
->get_return_value());
8574 ASSERT_EQ(42, rval
);
8575 out
.hexdump(std::cout
);
8576 ASSERT_EQ("you might see this", std::string(out
.c_str(), out
.length()));
8579 // this will overflow because the return data is too big
8583 ObjectWriteOperation o
;
8584 o
.exec("hello", "write_too_much_return_data", in
, &out
, &rval
);
8585 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
8586 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &o
,
8587 librados::OPERATION_RETURNVEC
));
8588 completion
->wait_for_complete();
8589 ASSERT_EQ(-EOVERFLOW
, completion
->get_return_value());
8590 ASSERT_EQ(-EOVERFLOW
, rval
);
8591 ASSERT_EQ("", std::string(out
.c_str(), out
.length()));