1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
3 #include "gtest/gtest.h"
5 #include "mds/mdstypes.h"
6 #include "include/buffer.h"
7 #include "include/rbd_types.h"
8 #include "include/rados/librados.h"
9 #include "include/rados/librados.hpp"
10 #include "include/stringify.h"
11 #include "include/types.h"
12 #include "global/global_context.h"
13 #include "common/Cond.h"
14 #include "test/librados/test.h"
15 #include "test/librados/TestCase.h"
16 #include "json_spirit/json_spirit.h"
18 #include "osd/HitSet.h"
25 using namespace librados
;
27 using std::ostringstream
;
30 typedef RadosTestPP LibRadosTierPP
;
31 typedef RadosTestECPP LibRadosTierECPP
;
33 void flush_evict_all(librados::Rados
& cluster
, librados::IoCtx
& cache_ioctx
)
36 cache_ioctx
.set_namespace(all_nspaces
);
37 for (NObjectIterator it
= cache_ioctx
.nobjects_begin();
38 it
!= cache_ioctx
.nobjects_end(); ++it
) {
39 cache_ioctx
.locator_set_key(it
->get_locator());
40 cache_ioctx
.set_namespace(it
->get_nspace());
42 ObjectReadOperation op
;
44 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
45 cache_ioctx
.aio_operate(
46 it
->get_oid(), completion
, &op
,
47 librados::OPERATION_IGNORE_OVERLAY
, NULL
);
48 completion
->wait_for_safe();
49 completion
->get_return_value();
50 completion
->release();
53 ObjectReadOperation op
;
55 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
56 cache_ioctx
.aio_operate(
57 it
->get_oid(), completion
, &op
,
58 librados::OPERATION_IGNORE_OVERLAY
, NULL
);
59 completion
->wait_for_safe();
60 completion
->get_return_value();
61 completion
->release();
66 class LibRadosTwoPoolsPP
: public RadosTestPP
69 LibRadosTwoPoolsPP() {};
70 ~LibRadosTwoPoolsPP() override
{};
72 static void SetUpTestCase() {
73 pool_name
= get_temp_pool_name();
74 ASSERT_EQ("", create_one_pool_pp(pool_name
, s_cluster
));
76 static void TearDownTestCase() {
77 ASSERT_EQ(0, destroy_one_pool_pp(pool_name
, s_cluster
));
79 static std::string cache_pool_name
;
81 void SetUp() override
{
82 cache_pool_name
= get_temp_pool_name();
83 ASSERT_EQ(0, s_cluster
.pool_create(cache_pool_name
.c_str()));
86 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
87 cache_ioctx
.application_enable("rados", true);
88 cache_ioctx
.set_namespace(nspace
);
90 void TearDown() override
{
91 // flush + evict cache
92 flush_evict_all(cluster
, cache_ioctx
);
96 ASSERT_EQ(0, cluster
.mon_command(
97 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
100 ASSERT_EQ(0, cluster
.mon_command(
101 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
102 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
105 // wait for maps to settle before next test
106 cluster
.wait_for_latest_osdmap();
108 RadosTestPP::TearDown();
110 cleanup_default_namespace(cache_ioctx
);
111 cleanup_namespace(cache_ioctx
, nspace
);
114 ASSERT_EQ(0, s_cluster
.pool_delete(cache_pool_name
.c_str()));
116 librados::IoCtx cache_ioctx
;
122 Completions() = default;
123 librados::AioCompletion
* getCompletion() {
124 librados::AioCompletion
* comp
= librados::Rados::aio_create_completion();
125 m_completions
.push_back(comp
);
130 for (auto& comp
: m_completions
) {
136 vector
<librados::AioCompletion
*> m_completions
;
139 Completions completions
;
141 std::string
LibRadosTwoPoolsPP::cache_pool_name
;
143 TEST_F(LibRadosTierPP
, Dirty
) {
145 ObjectWriteOperation op
;
147 ASSERT_EQ(0, ioctx
.operate("foo", &op
)); // still get 0 if it dne
150 ObjectWriteOperation op
;
152 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
157 ObjectReadOperation op
;
158 op
.is_dirty(&dirty
, &r
);
159 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
164 ObjectWriteOperation op
;
166 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
169 ObjectWriteOperation op
;
171 ASSERT_EQ(0, ioctx
.operate("foo", &op
)); // still 0 if already clean
176 ObjectReadOperation op
;
177 op
.is_dirty(&dirty
, &r
);
178 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
183 ObjectWriteOperation op
;
184 op
.truncate(0); // still a write even tho it is a no-op
185 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
190 ObjectReadOperation op
;
191 op
.is_dirty(&dirty
, &r
);
192 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
198 TEST_F(LibRadosTwoPoolsPP
, Overlay
) {
203 ObjectWriteOperation op
;
205 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
210 ObjectWriteOperation op
;
212 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
217 ASSERT_EQ(0, cluster
.mon_command(
218 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
219 "\", \"tierpool\": \"" + cache_pool_name
+
220 "\", \"force_nonempty\": \"--force-nonempty\" }",
222 ASSERT_EQ(0, cluster
.mon_command(
223 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
224 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
227 // wait for maps to settle
228 cluster
.wait_for_latest_osdmap();
230 // by default, the overlay sends us to cache pool
233 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
234 ASSERT_EQ('c', bl
[0]);
238 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
239 ASSERT_EQ('c', bl
[0]);
242 // unless we say otherwise
245 ObjectReadOperation op
;
246 op
.read(0, 1, &bl
, NULL
);
247 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
248 ASSERT_EQ(0, ioctx
.aio_operate(
249 "foo", completion
, &op
,
250 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
251 completion
->wait_for_safe();
252 ASSERT_EQ(0, completion
->get_return_value());
253 completion
->release();
254 ASSERT_EQ('b', bl
[0]);
258 TEST_F(LibRadosTwoPoolsPP
, Promote
) {
262 bl
.append("hi there");
263 ObjectWriteOperation op
;
265 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
270 ASSERT_EQ(0, cluster
.mon_command(
271 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
272 "\", \"tierpool\": \"" + cache_pool_name
+
273 "\", \"force_nonempty\": \"--force-nonempty\" }",
275 ASSERT_EQ(0, cluster
.mon_command(
276 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
277 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
279 ASSERT_EQ(0, cluster
.mon_command(
280 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
281 "\", \"mode\": \"writeback\"}",
284 // wait for maps to settle
285 cluster
.wait_for_latest_osdmap();
287 // read, trigger a promote
290 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
293 // read, trigger a whiteout
296 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
297 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
300 // verify the object is present in the cache tier
302 NObjectIterator it
= cache_ioctx
.nobjects_begin();
303 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
304 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
306 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
308 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
312 TEST_F(LibRadosTwoPoolsPP
, PromoteSnap
) {
316 bl
.append("hi there");
317 ObjectWriteOperation op
;
319 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
323 bl
.append("hi there");
324 ObjectWriteOperation op
;
326 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
330 bl
.append("hi there");
331 ObjectWriteOperation op
;
333 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
337 bl
.append("hi there");
338 ObjectWriteOperation op
;
340 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
343 // create a snapshot, clone
344 vector
<uint64_t> my_snaps(1);
345 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
346 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
351 ObjectWriteOperation op
;
353 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
358 ObjectWriteOperation op
;
360 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
363 ObjectWriteOperation op
;
365 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
370 ObjectWriteOperation op
;
372 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
377 ASSERT_EQ(0, cluster
.mon_command(
378 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
379 "\", \"tierpool\": \"" + cache_pool_name
+
380 "\", \"force_nonempty\": \"--force-nonempty\" }",
382 ASSERT_EQ(0, cluster
.mon_command(
383 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
384 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
386 ASSERT_EQ(0, cluster
.mon_command(
387 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
388 "\", \"mode\": \"writeback\"}",
391 // wait for maps to settle
392 cluster
.wait_for_latest_osdmap();
394 // read, trigger a promote on the head
397 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
398 ASSERT_EQ('c', bl
[0]);
402 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
403 ASSERT_EQ('c', bl
[0]);
406 ioctx
.snap_set_read(my_snaps
[0]);
411 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
412 ASSERT_EQ('h', bl
[0]);
418 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
419 ASSERT_EQ('h', bl
[0]);
425 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
426 ASSERT_EQ('h', bl
[0]);
429 ioctx
.snap_set_read(librados::SNAP_HEAD
);
434 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
435 ASSERT_EQ('c', bl
[0]);
441 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
442 ASSERT_EQ('c', bl
[0]);
448 ASSERT_EQ(-ENOENT
, ioctx
.read("baz", bl
, 1, 0));
452 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
455 TEST_F(LibRadosTwoPoolsPP
, PromoteSnapScrub
) {
459 for (int i
=0; i
<num
; ++i
) {
461 bl
.append("hi there");
462 ObjectWriteOperation op
;
464 ASSERT_EQ(0, ioctx
.operate(string("foo") + stringify(i
), &op
));
467 vector
<uint64_t> my_snaps
;
468 for (int snap
=0; snap
<4; ++snap
) {
469 // create a snapshot, clone
470 vector
<uint64_t> ns(1);
471 ns
.insert(ns
.end(), my_snaps
.begin(), my_snaps
.end());
473 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
474 cout
<< "my_snaps " << my_snaps
<< std::endl
;
475 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
477 for (int i
=0; i
<num
; ++i
) {
479 bl
.append(string("ciao! snap") + stringify(snap
));
480 ObjectWriteOperation op
;
482 ASSERT_EQ(0, ioctx
.operate(string("foo") + stringify(i
), &op
));
488 ASSERT_EQ(0, cluster
.mon_command(
489 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
490 "\", \"tierpool\": \"" + cache_pool_name
+
491 "\", \"force_nonempty\": \"--force-nonempty\" }",
493 ASSERT_EQ(0, cluster
.mon_command(
494 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
495 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
497 ASSERT_EQ(0, cluster
.mon_command(
498 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
499 "\", \"mode\": \"writeback\"}",
502 // wait for maps to settle
503 cluster
.wait_for_latest_osdmap();
505 // read, trigger a promote on _some_ heads to make sure we handle cases
506 // where snaps are present and where they are not.
507 cout
<< "promoting some heads" << std::endl
;
508 for (int i
=0; i
<num
; ++i
) {
509 if (i
% 5 == 0 || i
> num
- 3) {
511 ASSERT_EQ(1, ioctx
.read(string("foo") + stringify(i
), bl
, 1, 0));
512 ASSERT_EQ('c', bl
[0]);
516 for (unsigned snap
= 0; snap
< my_snaps
.size(); ++snap
) {
517 cout
<< "promoting from clones for snap " << my_snaps
[snap
] << std::endl
;
518 ioctx
.snap_set_read(my_snaps
[snap
]);
520 // read some snaps, semi-randomly
521 for (int i
=0; i
<50; ++i
) {
523 string o
= string("foo") + stringify((snap
* i
* 137) % 80);
524 //cout << o << std::endl;
525 ASSERT_EQ(1, ioctx
.read(o
, bl
, 1, 0));
529 // ok, stop and scrub this pool (to make sure scrub can handle
530 // missing clones in the cache tier).
533 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
534 for (int i
=0; i
<10; ++i
) {
537 ss
<< "{\"prefix\": \"pg scrub\", \"pgid\": \""
538 << cache_ioctx
.get_id() << "." << i
540 int r
= cluster
.mon_command(ss
.str(), inbl
, NULL
, NULL
);
541 if (r
== -ENOENT
|| // in case mgr osdmap is stale
549 // give it a few seconds to go. this is sloppy but is usually enough time
550 cout
<< "waiting for scrubs..." << std::endl
;
552 cout
<< "done waiting" << std::endl
;
555 ioctx
.snap_set_read(librados::SNAP_HEAD
);
558 for (unsigned snap
= 0; snap
< my_snaps
.size(); ++snap
) {
559 ioctx
.selfmanaged_snap_remove(my_snaps
[snap
]);
563 TEST_F(LibRadosTwoPoolsPP
, PromoteSnapTrimRace
) {
567 bl
.append("hi there");
568 ObjectWriteOperation op
;
570 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
573 // create a snapshot, clone
574 vector
<uint64_t> my_snaps(1);
575 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
576 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
581 ObjectWriteOperation op
;
583 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
588 ASSERT_EQ(0, cluster
.mon_command(
589 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
590 "\", \"tierpool\": \"" + cache_pool_name
+
591 "\", \"force_nonempty\": \"--force-nonempty\" }",
593 ASSERT_EQ(0, cluster
.mon_command(
594 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
595 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
597 ASSERT_EQ(0, cluster
.mon_command(
598 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
599 "\", \"mode\": \"writeback\"}",
602 // wait for maps to settle
603 cluster
.wait_for_latest_osdmap();
606 ASSERT_EQ(0, ioctx
.selfmanaged_snap_remove(my_snaps
[0]));
608 ioctx
.snap_set_read(my_snaps
[0]);
613 ASSERT_EQ(-ENOENT
, ioctx
.read("foo", bl
, 1, 0));
617 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
620 TEST_F(LibRadosTwoPoolsPP
, Whiteout
) {
624 bl
.append("hi there");
625 ObjectWriteOperation op
;
627 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
632 ASSERT_EQ(0, cluster
.mon_command(
633 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
634 "\", \"tierpool\": \"" + cache_pool_name
+
635 "\", \"force_nonempty\": \"--force-nonempty\" }",
637 ASSERT_EQ(0, cluster
.mon_command(
638 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
639 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
641 ASSERT_EQ(0, cluster
.mon_command(
642 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
643 "\", \"mode\": \"writeback\"}",
646 // wait for maps to settle
647 cluster
.wait_for_latest_osdmap();
649 // create some whiteouts, verify they behave
651 ObjectWriteOperation op
;
654 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
658 ObjectWriteOperation op
;
661 ASSERT_EQ(-ENOENT
, ioctx
.operate("bar", &op
));
664 ObjectWriteOperation op
;
667 ASSERT_EQ(-ENOENT
, ioctx
.operate("bar", &op
));
670 // verify the whiteouts are there in the cache tier
672 NObjectIterator it
= cache_ioctx
.nobjects_begin();
673 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
674 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
676 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
678 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
681 // delete a whiteout and verify it goes away
682 ASSERT_EQ(-ENOENT
, ioctx
.remove("foo"));
684 ObjectWriteOperation op
;
686 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
687 ASSERT_EQ(0, ioctx
.aio_operate("bar", completion
, &op
,
688 librados::OPERATION_IGNORE_CACHE
));
689 completion
->wait_for_safe();
690 ASSERT_EQ(0, completion
->get_return_value());
691 completion
->release();
693 NObjectIterator it
= cache_ioctx
.nobjects_begin();
694 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
695 ASSERT_TRUE(it
->get_oid() == string("foo"));
697 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
700 // recreate an object and verify we can read it
703 bl
.append("hi there");
704 ObjectWriteOperation op
;
706 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
710 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
711 ASSERT_EQ('h', bl
[0]);
715 TEST_F(LibRadosTwoPoolsPP
, WhiteoutDeleteCreate
) {
718 ASSERT_EQ(0, cluster
.mon_command(
719 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
720 "\", \"tierpool\": \"" + cache_pool_name
+
721 "\", \"force_nonempty\": \"--force-nonempty\" }",
723 ASSERT_EQ(0, cluster
.mon_command(
724 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
725 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
727 ASSERT_EQ(0, cluster
.mon_command(
728 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
729 "\", \"mode\": \"writeback\"}",
732 // wait for maps to settle
733 cluster
.wait_for_latest_osdmap();
739 ASSERT_EQ(0, ioctx
.write_full("foo", bl
));
742 // do delete + create operation
744 ObjectWriteOperation op
;
749 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
752 // verify it still "exists" (w/ new content)
755 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
756 ASSERT_EQ('b', bl
[0]);
760 TEST_F(LibRadosTwoPoolsPP
, Evict
) {
764 bl
.append("hi there");
765 ObjectWriteOperation op
;
767 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
772 ASSERT_EQ(0, cluster
.mon_command(
773 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
774 "\", \"tierpool\": \"" + cache_pool_name
+
775 "\", \"force_nonempty\": \"--force-nonempty\" }",
777 ASSERT_EQ(0, cluster
.mon_command(
778 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
779 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
781 ASSERT_EQ(0, cluster
.mon_command(
782 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
783 "\", \"mode\": \"writeback\"}",
786 // wait for maps to settle
787 cluster
.wait_for_latest_osdmap();
789 // read, trigger a promote
792 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
795 // read, trigger a whiteout, and a dirty object
798 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
799 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
800 ASSERT_EQ(0, ioctx
.write("bar", bl
, bl
.length(), 0));
803 // verify the object is present in the cache tier
805 NObjectIterator it
= cache_ioctx
.nobjects_begin();
806 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
807 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
809 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
811 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
816 ObjectWriteOperation op
;
818 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
819 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
820 completion
->wait_for_safe();
821 ASSERT_EQ(0, completion
->get_return_value());
822 completion
->release();
825 // evict the pinned object with -EPERM
827 ObjectReadOperation op
;
829 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
830 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
831 librados::OPERATION_IGNORE_CACHE
,
833 completion
->wait_for_safe();
834 ASSERT_EQ(-EPERM
, completion
->get_return_value());
835 completion
->release();
840 ObjectWriteOperation op
;
842 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
843 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
844 completion
->wait_for_safe();
845 ASSERT_EQ(0, completion
->get_return_value());
846 completion
->release();
851 ObjectReadOperation op
;
853 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
854 ASSERT_EQ(0, cache_ioctx
.aio_operate(
855 "foo", completion
, &op
,
856 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
857 completion
->wait_for_safe();
858 ASSERT_EQ(0, completion
->get_return_value());
859 completion
->release();
866 ObjectReadOperation op
;
867 op
.is_dirty(&dirty
, &r
);
868 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
875 ObjectReadOperation op
;
877 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
878 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
879 librados::OPERATION_IGNORE_CACHE
,
881 completion
->wait_for_safe();
882 ASSERT_EQ(0, completion
->get_return_value());
883 completion
->release();
886 ObjectReadOperation op
;
888 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
889 ASSERT_EQ(0, cache_ioctx
.aio_operate(
890 "foo", completion
, &op
,
891 librados::OPERATION_IGNORE_CACHE
, NULL
));
892 completion
->wait_for_safe();
893 ASSERT_EQ(0, completion
->get_return_value());
894 completion
->release();
897 ObjectReadOperation op
;
899 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
900 ASSERT_EQ(0, cache_ioctx
.aio_operate(
901 "bar", completion
, &op
,
902 librados::OPERATION_IGNORE_CACHE
, NULL
));
903 completion
->wait_for_safe();
904 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
905 completion
->release();
909 TEST_F(LibRadosTwoPoolsPP
, EvictSnap
) {
913 bl
.append("hi there");
914 ObjectWriteOperation op
;
916 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
920 bl
.append("hi there");
921 ObjectWriteOperation op
;
923 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
927 bl
.append("hi there");
928 ObjectWriteOperation op
;
930 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
934 bl
.append("hi there");
935 ObjectWriteOperation op
;
937 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
940 // create a snapshot, clone
941 vector
<uint64_t> my_snaps(1);
942 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
943 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
948 ObjectWriteOperation op
;
950 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
955 ObjectWriteOperation op
;
957 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
960 ObjectWriteOperation op
;
962 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
967 ObjectWriteOperation op
;
969 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
974 ASSERT_EQ(0, cluster
.mon_command(
975 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
976 "\", \"tierpool\": \"" + cache_pool_name
+
977 "\", \"force_nonempty\": \"--force-nonempty\" }",
979 ASSERT_EQ(0, cluster
.mon_command(
980 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
981 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
983 ASSERT_EQ(0, cluster
.mon_command(
984 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
985 "\", \"mode\": \"writeback\"}",
988 // wait for maps to settle
989 cluster
.wait_for_latest_osdmap();
991 // read, trigger a promote on the head
994 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
995 ASSERT_EQ('c', bl
[0]);
999 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
1000 ASSERT_EQ('c', bl
[0]);
1005 ObjectReadOperation op
;
1007 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1008 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1009 "bam", completion
, &op
,
1010 librados::OPERATION_IGNORE_CACHE
, NULL
));
1011 completion
->wait_for_safe();
1012 ASSERT_EQ(0, completion
->get_return_value());
1013 completion
->release();
1017 ObjectReadOperation op
;
1018 op
.read(1, 0, &bl
, NULL
);
1019 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1020 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1021 "bam", completion
, &op
,
1022 librados::OPERATION_IGNORE_CACHE
, NULL
));
1023 completion
->wait_for_safe();
1024 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
1025 completion
->release();
1029 ioctx
.snap_set_read(my_snaps
[0]);
1032 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1033 ASSERT_EQ('h', bl
[0]);
1038 ObjectReadOperation op
;
1040 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1041 ASSERT_EQ(0, ioctx
.aio_operate(
1042 "foo", completion
, &op
,
1043 librados::OPERATION_IGNORE_CACHE
, NULL
));
1044 completion
->wait_for_safe();
1045 ASSERT_EQ(0, completion
->get_return_value());
1046 completion
->release();
1051 ObjectReadOperation op
;
1052 op
.read(1, 0, &bl
, NULL
);
1053 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1054 ASSERT_EQ(0, ioctx
.aio_operate(
1055 "foo", completion
, &op
,
1056 librados::OPERATION_IGNORE_CACHE
, NULL
));
1057 completion
->wait_for_safe();
1058 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
1059 completion
->release();
1061 // head is still there...
1062 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1065 ObjectReadOperation op
;
1066 op
.read(1, 0, &bl
, NULL
);
1067 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1068 ASSERT_EQ(0, ioctx
.aio_operate(
1069 "foo", completion
, &op
,
1070 librados::OPERATION_IGNORE_CACHE
, NULL
));
1071 completion
->wait_for_safe();
1072 ASSERT_EQ(0, completion
->get_return_value());
1073 completion
->release();
1076 // promote head + snap of bar
1077 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1080 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
1081 ASSERT_EQ('c', bl
[0]);
1083 ioctx
.snap_set_read(my_snaps
[0]);
1086 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
1087 ASSERT_EQ('h', bl
[0]);
1090 // evict bar head (fail)
1091 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1093 ObjectReadOperation op
;
1095 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1096 ASSERT_EQ(0, ioctx
.aio_operate(
1097 "bar", completion
, &op
,
1098 librados::OPERATION_IGNORE_CACHE
, NULL
));
1099 completion
->wait_for_safe();
1100 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
1101 completion
->release();
1105 ioctx
.snap_set_read(my_snaps
[0]);
1107 ObjectReadOperation op
;
1109 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1110 ASSERT_EQ(0, ioctx
.aio_operate(
1111 "bar", completion
, &op
,
1112 librados::OPERATION_IGNORE_CACHE
, NULL
));
1113 completion
->wait_for_safe();
1114 ASSERT_EQ(0, completion
->get_return_value());
1115 completion
->release();
1118 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1121 ObjectReadOperation op
;
1122 op
.read(1, 0, &bl
, NULL
);
1123 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1124 ASSERT_EQ(0, ioctx
.aio_operate(
1125 "bar", completion
, &op
,
1126 librados::OPERATION_IGNORE_CACHE
, NULL
));
1127 completion
->wait_for_safe();
1128 ASSERT_EQ(0, completion
->get_return_value());
1129 completion
->release();
1132 ObjectReadOperation op
;
1134 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1135 ASSERT_EQ(0, ioctx
.aio_operate(
1136 "bar", completion
, &op
,
1137 librados::OPERATION_IGNORE_CACHE
, NULL
));
1138 completion
->wait_for_safe();
1139 ASSERT_EQ(0, completion
->get_return_value());
1140 completion
->release();
1144 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
1147 // this test case reproduces http://tracker.ceph.com/issues/8629
1148 TEST_F(LibRadosTwoPoolsPP
, EvictSnap2
) {
1152 bl
.append("hi there");
1153 ObjectWriteOperation op
;
1155 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1157 // create a snapshot, clone
1158 vector
<uint64_t> my_snaps(1);
1159 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
1160 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
1165 ObjectWriteOperation op
;
1167 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1171 ASSERT_EQ(0, cluster
.mon_command(
1172 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1173 "\", \"tierpool\": \"" + cache_pool_name
+
1174 "\", \"force_nonempty\": \"--force-nonempty\" }",
1176 ASSERT_EQ(0, cluster
.mon_command(
1177 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1178 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1180 ASSERT_EQ(0, cluster
.mon_command(
1181 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1182 "\", \"mode\": \"writeback\"}",
1185 // wait for maps to settle
1186 cluster
.wait_for_latest_osdmap();
1188 // read, trigger a promote on the head
1191 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1192 ASSERT_EQ('c', bl
[0]);
1197 ObjectReadOperation op
;
1199 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1200 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1201 "foo", completion
, &op
,
1202 librados::OPERATION_IGNORE_CACHE
, NULL
));
1203 completion
->wait_for_safe();
1204 ASSERT_EQ(0, completion
->get_return_value());
1205 completion
->release();
1208 // verify the snapdir is not present in the cache pool
1210 ObjectReadOperation op
;
1211 librados::snap_set_t snapset
;
1212 op
.list_snaps(&snapset
, NULL
);
1213 ioctx
.snap_set_read(librados::SNAP_DIR
);
1214 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1215 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
,
1216 librados::OPERATION_IGNORE_CACHE
, NULL
));
1217 completion
->wait_for_safe();
1218 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
1219 completion
->release();
1223 TEST_F(LibRadosTwoPoolsPP
, TryFlush
) {
1226 ASSERT_EQ(0, cluster
.mon_command(
1227 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1228 "\", \"tierpool\": \"" + cache_pool_name
+
1229 "\", \"force_nonempty\": \"--force-nonempty\" }",
1231 ASSERT_EQ(0, cluster
.mon_command(
1232 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1233 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1235 ASSERT_EQ(0, cluster
.mon_command(
1236 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1237 "\", \"mode\": \"writeback\"}",
1240 // wait for maps to settle
1241 cluster
.wait_for_latest_osdmap();
1246 bl
.append("hi there");
1247 ObjectWriteOperation op
;
1249 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1252 // verify the object is present in the cache tier
1254 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1255 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
1256 ASSERT_TRUE(it
->get_oid() == string("foo"));
1258 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1261 // verify the object is NOT present in the base tier
1263 NObjectIterator it
= ioctx
.nobjects_begin();
1264 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1271 ObjectReadOperation op
;
1272 op
.is_dirty(&dirty
, &r
);
1273 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1280 ObjectWriteOperation op
;
1282 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1283 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
1284 completion
->wait_for_safe();
1285 ASSERT_EQ(0, completion
->get_return_value());
1286 completion
->release();
1289 // flush the pinned object with -EPERM
1291 ObjectReadOperation op
;
1292 op
.cache_try_flush();
1293 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1294 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1295 "foo", completion
, &op
,
1296 librados::OPERATION_IGNORE_OVERLAY
|
1297 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1298 completion
->wait_for_safe();
1299 ASSERT_EQ(-EPERM
, completion
->get_return_value());
1300 completion
->release();
1305 ObjectWriteOperation op
;
1307 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1308 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
1309 completion
->wait_for_safe();
1310 ASSERT_EQ(0, completion
->get_return_value());
1311 completion
->release();
1316 ObjectReadOperation op
;
1317 op
.cache_try_flush();
1318 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1319 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1320 "foo", completion
, &op
,
1321 librados::OPERATION_IGNORE_OVERLAY
|
1322 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1323 completion
->wait_for_safe();
1324 ASSERT_EQ(0, completion
->get_return_value());
1325 completion
->release();
1332 ObjectReadOperation op
;
1333 op
.is_dirty(&dirty
, &r
);
1334 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1335 ASSERT_FALSE(dirty
);
1339 // verify in base tier
1341 NObjectIterator it
= ioctx
.nobjects_begin();
1342 ASSERT_TRUE(it
!= ioctx
.nobjects_end());
1343 ASSERT_TRUE(it
->get_oid() == string("foo"));
1345 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1350 ObjectReadOperation op
;
1352 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1353 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1354 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
1355 completion
->wait_for_safe();
1356 ASSERT_EQ(0, completion
->get_return_value());
1357 completion
->release();
1360 // verify no longer in cache tier
1362 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1363 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1367 TEST_F(LibRadosTwoPoolsPP
, Flush
) {
1370 ASSERT_EQ(0, cluster
.mon_command(
1371 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1372 "\", \"tierpool\": \"" + cache_pool_name
+
1373 "\", \"force_nonempty\": \"--force-nonempty\" }",
1375 ASSERT_EQ(0, cluster
.mon_command(
1376 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1377 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1379 ASSERT_EQ(0, cluster
.mon_command(
1380 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1381 "\", \"mode\": \"writeback\"}",
1384 // wait for maps to settle
1385 cluster
.wait_for_latest_osdmap();
1387 uint64_t user_version
= 0;
1392 bl
.append("hi there");
1393 ObjectWriteOperation op
;
1395 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1398 // verify the object is present in the cache tier
1400 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1401 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
1402 ASSERT_TRUE(it
->get_oid() == string("foo"));
1404 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1407 // verify the object is NOT present in the base tier
1409 NObjectIterator it
= ioctx
.nobjects_begin();
1410 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1417 ObjectReadOperation op
;
1418 op
.is_dirty(&dirty
, &r
);
1419 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1422 user_version
= cache_ioctx
.get_last_version();
1427 ObjectWriteOperation op
;
1429 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1430 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
1431 completion
->wait_for_safe();
1432 ASSERT_EQ(0, completion
->get_return_value());
1433 completion
->release();
1436 // flush the pinned object with -EPERM
1438 ObjectReadOperation op
;
1439 op
.cache_try_flush();
1440 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1441 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1442 "foo", completion
, &op
,
1443 librados::OPERATION_IGNORE_OVERLAY
|
1444 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1445 completion
->wait_for_safe();
1446 ASSERT_EQ(-EPERM
, completion
->get_return_value());
1447 completion
->release();
1452 ObjectWriteOperation op
;
1454 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1455 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
1456 completion
->wait_for_safe();
1457 ASSERT_EQ(0, completion
->get_return_value());
1458 completion
->release();
1463 ObjectReadOperation op
;
1465 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1466 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1467 "foo", completion
, &op
,
1468 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1469 completion
->wait_for_safe();
1470 ASSERT_EQ(0, completion
->get_return_value());
1471 completion
->release();
1478 ObjectReadOperation op
;
1479 op
.is_dirty(&dirty
, &r
);
1480 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
1481 ASSERT_FALSE(dirty
);
1485 // verify in base tier
1487 NObjectIterator it
= ioctx
.nobjects_begin();
1488 ASSERT_TRUE(it
!= ioctx
.nobjects_end());
1489 ASSERT_TRUE(it
->get_oid() == string("foo"));
1491 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1496 ObjectReadOperation op
;
1498 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1499 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1500 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
1501 completion
->wait_for_safe();
1502 ASSERT_EQ(0, completion
->get_return_value());
1503 completion
->release();
1506 // verify no longer in cache tier
1508 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1509 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1512 // read it again and verify the version is consistent
1515 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
1516 ASSERT_EQ(user_version
, cache_ioctx
.get_last_version());
1521 ObjectWriteOperation op
;
1523 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1528 ObjectReadOperation op
;
1530 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1531 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1532 "foo", completion
, &op
,
1533 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1534 completion
->wait_for_safe();
1535 ASSERT_EQ(0, completion
->get_return_value());
1536 completion
->release();
1541 ObjectReadOperation op
;
1543 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1544 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1545 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
1546 completion
->wait_for_safe();
1547 ASSERT_EQ(0, completion
->get_return_value());
1548 completion
->release();
1551 // verify no longer in cache tier
1553 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1554 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1558 NObjectIterator it
= ioctx
.nobjects_begin();
1559 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1563 TEST_F(LibRadosTwoPoolsPP
, FlushSnap
) {
1566 ASSERT_EQ(0, cluster
.mon_command(
1567 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1568 "\", \"tierpool\": \"" + cache_pool_name
+
1569 "\", \"force_nonempty\": \"--force-nonempty\" }",
1571 ASSERT_EQ(0, cluster
.mon_command(
1572 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1573 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1575 ASSERT_EQ(0, cluster
.mon_command(
1576 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1577 "\", \"mode\": \"writeback\"}",
1580 // wait for maps to settle
1581 cluster
.wait_for_latest_osdmap();
1587 ObjectWriteOperation op
;
1589 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1592 // create a snapshot, clone
1593 vector
<uint64_t> my_snaps(1);
1594 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
1595 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
1600 ObjectWriteOperation op
;
1602 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1607 my_snaps
[1] = my_snaps
[0];
1608 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
1609 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
1614 ObjectWriteOperation op
;
1616 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1619 // verify the object is present in the cache tier
1621 NObjectIterator it
= cache_ioctx
.nobjects_begin();
1622 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
1623 ASSERT_TRUE(it
->get_oid() == string("foo"));
1625 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
1628 // verify the object is NOT present in the base tier
1630 NObjectIterator it
= ioctx
.nobjects_begin();
1631 ASSERT_TRUE(it
== ioctx
.nobjects_end());
1634 // flush on head (should fail)
1635 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1637 ObjectReadOperation op
;
1639 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1640 ASSERT_EQ(0, ioctx
.aio_operate(
1641 "foo", completion
, &op
,
1642 librados::OPERATION_IGNORE_CACHE
, NULL
));
1643 completion
->wait_for_safe();
1644 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
1645 completion
->release();
1647 // flush on recent snap (should fail)
1648 ioctx
.snap_set_read(my_snaps
[0]);
1650 ObjectReadOperation op
;
1652 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1653 ASSERT_EQ(0, ioctx
.aio_operate(
1654 "foo", completion
, &op
,
1655 librados::OPERATION_IGNORE_CACHE
, NULL
));
1656 completion
->wait_for_safe();
1657 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
1658 completion
->release();
1660 // flush on oldest snap
1661 ioctx
.snap_set_read(my_snaps
[1]);
1663 ObjectReadOperation op
;
1665 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1666 ASSERT_EQ(0, ioctx
.aio_operate(
1667 "foo", completion
, &op
,
1668 librados::OPERATION_IGNORE_CACHE
, NULL
));
1669 completion
->wait_for_safe();
1670 ASSERT_EQ(0, completion
->get_return_value());
1671 completion
->release();
1673 // flush on next oldest snap
1674 ioctx
.snap_set_read(my_snaps
[0]);
1676 ObjectReadOperation op
;
1678 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1679 ASSERT_EQ(0, ioctx
.aio_operate(
1680 "foo", completion
, &op
,
1681 librados::OPERATION_IGNORE_CACHE
, NULL
));
1682 completion
->wait_for_safe();
1683 ASSERT_EQ(0, completion
->get_return_value());
1684 completion
->release();
1687 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1689 ObjectReadOperation op
;
1691 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1692 ASSERT_EQ(0, ioctx
.aio_operate(
1693 "foo", completion
, &op
,
1694 librados::OPERATION_IGNORE_CACHE
, NULL
));
1695 completion
->wait_for_safe();
1696 ASSERT_EQ(0, completion
->get_return_value());
1697 completion
->release();
1700 // verify i can read the snaps from the cache pool
1701 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1704 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1705 ASSERT_EQ('c', bl
[0]);
1707 ioctx
.snap_set_read(my_snaps
[0]);
1710 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1711 ASSERT_EQ('b', bl
[0]);
1713 ioctx
.snap_set_read(my_snaps
[1]);
1716 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1717 ASSERT_EQ('a', bl
[0]);
1721 ASSERT_EQ(0, cluster
.mon_command(
1722 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
1726 // wait for maps to settle
1727 cluster
.wait_for_latest_osdmap();
1729 // verify i can read the snaps from the base pool
1730 ioctx
.snap_set_read(librados::SNAP_HEAD
);
1733 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1734 ASSERT_EQ('c', bl
[0]);
1736 ioctx
.snap_set_read(my_snaps
[0]);
1739 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1740 ASSERT_EQ('b', bl
[0]);
1742 ioctx
.snap_set_read(my_snaps
[1]);
1745 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
1746 ASSERT_EQ('a', bl
[0]);
1749 ASSERT_EQ(0, cluster
.mon_command(
1750 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1751 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1755 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
1758 TEST_F(LibRadosTierPP
, FlushWriteRaces
) {
1760 std::string pool_name
= get_temp_pool_name();
1761 std::string cache_pool_name
= pool_name
+ "-cache";
1762 ASSERT_EQ("", create_one_pool_pp(pool_name
, cluster
));
1763 ASSERT_EQ(0, cluster
.pool_create(cache_pool_name
.c_str()));
1765 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
1766 cache_ioctx
.application_enable("rados", true);
1768 ASSERT_EQ(0, cluster
.ioctx_create(pool_name
.c_str(), ioctx
));
1772 ASSERT_EQ(0, cluster
.mon_command(
1773 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1774 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
1776 ASSERT_EQ(0, cluster
.mon_command(
1777 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1778 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1780 ASSERT_EQ(0, cluster
.mon_command(
1781 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1782 "\", \"mode\": \"writeback\"}",
1785 // wait for maps to settle
1786 cluster
.wait_for_latest_osdmap();
1788 // create/dirty object
1790 bl
.append("hi there");
1792 ObjectWriteOperation op
;
1794 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1799 ObjectReadOperation op
;
1801 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1802 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1803 "foo", completion
, &op
,
1804 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1806 ObjectWriteOperation op2
;
1808 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
1809 ASSERT_EQ(0, ioctx
.aio_operate(
1810 "foo", completion2
, &op2
, 0));
1812 completion
->wait_for_safe();
1813 completion2
->wait_for_safe();
1814 ASSERT_EQ(0, completion
->get_return_value());
1815 ASSERT_EQ(0, completion2
->get_return_value());
1816 completion
->release();
1817 completion2
->release();
1822 // create/dirty object
1825 bl
.append("hi there");
1826 ObjectWriteOperation op
;
1828 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1831 // try-flush + write
1833 ObjectReadOperation op
;
1834 op
.cache_try_flush();
1835 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1836 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1837 "foo", completion
, &op
,
1838 librados::OPERATION_IGNORE_OVERLAY
|
1839 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1841 ObjectWriteOperation op2
;
1843 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
1844 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion2
, &op2
, 0));
1846 completion
->wait_for_safe();
1847 completion2
->wait_for_safe();
1848 int r
= completion
->get_return_value();
1849 ASSERT_TRUE(r
== -EBUSY
|| r
== 0);
1850 ASSERT_EQ(0, completion2
->get_return_value());
1851 completion
->release();
1852 completion2
->release();
1855 cout
<< "didn't get EBUSY, trying again" << std::endl
;
1857 ASSERT_TRUE(--tries
);
1861 ASSERT_EQ(0, cluster
.mon_command(
1862 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
1865 ASSERT_EQ(0, cluster
.mon_command(
1866 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
1867 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
1870 // wait for maps to settle before next test
1871 cluster
.wait_for_latest_osdmap();
1873 ASSERT_EQ(0, cluster
.pool_delete(cache_pool_name
.c_str()));
1874 ASSERT_EQ(0, destroy_one_pool_pp(pool_name
, cluster
));
1877 TEST_F(LibRadosTwoPoolsPP
, FlushTryFlushRaces
) {
1880 ASSERT_EQ(0, cluster
.mon_command(
1881 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
1882 "\", \"tierpool\": \"" + cache_pool_name
+
1883 "\", \"force_nonempty\": \"--force-nonempty\" }",
1885 ASSERT_EQ(0, cluster
.mon_command(
1886 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
1887 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
1889 ASSERT_EQ(0, cluster
.mon_command(
1890 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
1891 "\", \"mode\": \"writeback\"}",
1894 // wait for maps to settle
1895 cluster
.wait_for_latest_osdmap();
1897 // create/dirty object
1900 bl
.append("hi there");
1901 ObjectWriteOperation op
;
1903 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1908 ObjectReadOperation op
;
1910 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1911 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1912 "foo", completion
, &op
,
1913 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1915 ObjectReadOperation op2
;
1917 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
1918 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1919 "foo", completion2
, &op2
,
1920 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1922 completion
->wait_for_safe();
1923 completion2
->wait_for_safe();
1924 ASSERT_EQ(0, completion
->get_return_value());
1925 ASSERT_EQ(0, completion2
->get_return_value());
1926 completion
->release();
1927 completion2
->release();
1930 // create/dirty object
1933 bl
.append("hi there");
1934 ObjectWriteOperation op
;
1936 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1939 // flush + try-flush
1941 ObjectReadOperation op
;
1943 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1944 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1945 "foo", completion
, &op
,
1946 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1948 ObjectReadOperation op2
;
1949 op2
.cache_try_flush();
1950 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
1951 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1952 "foo", completion2
, &op2
,
1953 librados::OPERATION_IGNORE_OVERLAY
|
1954 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1956 completion
->wait_for_safe();
1957 completion2
->wait_for_safe();
1958 ASSERT_EQ(0, completion
->get_return_value());
1959 ASSERT_EQ(0, completion2
->get_return_value());
1960 completion
->release();
1961 completion2
->release();
1964 // create/dirty object
1969 bl
.append("hi there");
1970 ObjectWriteOperation op
;
1972 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
1975 // try-flush + flush
1976 // (flush will not piggyback on try-flush)
1978 ObjectReadOperation op
;
1979 op
.cache_try_flush();
1980 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
1981 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1982 "foo", completion
, &op
,
1983 librados::OPERATION_IGNORE_OVERLAY
|
1984 librados::OPERATION_SKIPRWLOCKS
, NULL
));
1986 ObjectReadOperation op2
;
1988 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
1989 ASSERT_EQ(0, cache_ioctx
.aio_operate(
1990 "foo", completion2
, &op2
,
1991 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
1993 completion
->wait_for_safe();
1994 completion2
->wait_for_safe();
1995 int r
= completion
->get_return_value();
1996 ASSERT_TRUE(r
== -EBUSY
|| r
== 0);
1997 ASSERT_EQ(0, completion2
->get_return_value());
1998 completion
->release();
1999 completion2
->release();
2002 cout
<< "didn't get EBUSY, trying again" << std::endl
;
2004 ASSERT_TRUE(--tries
);
2007 // create/dirty object
2010 bl
.append("hi there");
2011 ObjectWriteOperation op
;
2013 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2016 // try-flush + try-flush
2018 ObjectReadOperation op
;
2019 op
.cache_try_flush();
2020 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2021 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2022 "foo", completion
, &op
,
2023 librados::OPERATION_IGNORE_OVERLAY
|
2024 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2026 ObjectReadOperation op2
;
2027 op2
.cache_try_flush();
2028 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
2029 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2030 "foo", completion2
, &op2
,
2031 librados::OPERATION_IGNORE_OVERLAY
|
2032 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2034 completion
->wait_for_safe();
2035 completion2
->wait_for_safe();
2036 ASSERT_EQ(0, completion
->get_return_value());
2037 ASSERT_EQ(0, completion2
->get_return_value());
2038 completion
->release();
2039 completion2
->release();
2044 IoCtx
*read_ioctx
= 0;
2045 Mutex
test_lock("FlushReadRaces::lock");
2047 int max_reads
= 100;
2048 int num_reads
= 0; // in progress
2050 void flush_read_race_cb(completion_t cb
, void *arg
);
2052 void start_flush_read()
2054 //cout << " starting read" << std::endl;
2055 ObjectReadOperation op
;
2056 op
.stat(NULL
, NULL
, NULL
);
2057 librados::AioCompletion
*completion
= completions
.getCompletion();
2058 completion
->set_complete_callback(0, flush_read_race_cb
);
2059 read_ioctx
->aio_operate("foo", completion
, &op
, NULL
);
2062 void flush_read_race_cb(completion_t cb
, void *arg
)
2064 //cout << " finished read" << std::endl;
2066 if (num_reads
> max_reads
) {
2075 TEST_F(LibRadosTwoPoolsPP
, TryFlushReadRace
) {
2078 ASSERT_EQ(0, cluster
.mon_command(
2079 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2080 "\", \"tierpool\": \"" + cache_pool_name
+
2081 "\", \"force_nonempty\": \"--force-nonempty\" }",
2083 ASSERT_EQ(0, cluster
.mon_command(
2084 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2085 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2087 ASSERT_EQ(0, cluster
.mon_command(
2088 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2089 "\", \"mode\": \"writeback\"}",
2092 // wait for maps to settle
2093 cluster
.wait_for_latest_osdmap();
2095 // create/dirty object
2098 bl
.append("hi there");
2099 bufferptr
bp(4000000); // make it big!
2102 ObjectWriteOperation op
;
2104 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2107 // start a continuous stream of reads
2108 read_ioctx
= &ioctx
;
2110 for (int i
= 0; i
< max_reads
; ++i
) {
2117 ObjectReadOperation op
;
2118 op
.cache_try_flush();
2119 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2120 ASSERT_EQ(0, cache_ioctx
.aio_operate(
2121 "foo", completion
, &op
,
2122 librados::OPERATION_IGNORE_OVERLAY
|
2123 librados::OPERATION_SKIPRWLOCKS
, NULL
));
2125 completion
->wait_for_safe();
2126 ASSERT_EQ(0, completion
->get_return_value());
2127 completion
->release();
2132 while (num_reads
> 0)
2133 cond
.Wait(test_lock
);
2137 TEST_F(LibRadosTierPP
, HitSetNone
) {
2139 list
< pair
<time_t,time_t> > ls
;
2140 AioCompletion
*c
= librados::Rados::aio_create_completion();
2141 ASSERT_EQ(0, ioctx
.hit_set_list(123, c
, &ls
));
2142 c
->wait_for_complete();
2143 ASSERT_EQ(0, c
->get_return_value());
2144 ASSERT_TRUE(ls
.empty());
2149 AioCompletion
*c
= librados::Rados::aio_create_completion();
2150 ASSERT_EQ(0, ioctx
.hit_set_get(123, c
, 12345, &bl
));
2151 c
->wait_for_complete();
2152 ASSERT_EQ(-ENOENT
, c
->get_return_value());
2157 string
set_pool_str(string pool
, string var
, string val
)
2159 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2160 + string("\",\"var\": \"") + var
+ string("\",\"val\": \"")
2161 + val
+ string("\"}");
2164 string
set_pool_str(string pool
, string var
, int val
)
2166 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2167 + string("\",\"var\": \"") + var
+ string("\",\"val\": \"")
2168 + stringify(val
) + string("\"}");
2171 TEST_F(LibRadosTwoPoolsPP
, HitSetRead
) {
2174 ASSERT_EQ(0, cluster
.mon_command(
2175 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2176 "\", \"tierpool\": \"" + cache_pool_name
+
2177 "\", \"force_nonempty\": \"--force-nonempty\" }",
2180 // enable hitset tracking for this pool
2181 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", 2),
2183 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", 600),
2185 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type",
2189 // wait for maps to settle
2190 cluster
.wait_for_latest_osdmap();
2192 cache_ioctx
.set_namespace("");
2194 // keep reading until we see our object appear in the HitSet
2195 utime_t start
= ceph_clock_now();
2196 utime_t hard_stop
= start
+ utime_t(600, 0);
2199 utime_t now
= ceph_clock_now();
2200 ASSERT_TRUE(now
< hard_stop
);
2202 string name
= "foo";
2204 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(name
, &hash
));
2205 hobject_t
oid(sobject_t(name
, CEPH_NOSNAP
), "", hash
,
2206 cluster
.pool_lookup(cache_pool_name
.c_str()), "");
2209 ASSERT_EQ(-ENOENT
, cache_ioctx
.read("foo", bl
, 1, 0));
2212 AioCompletion
*c
= librados::Rados::aio_create_completion();
2213 ASSERT_EQ(0, cache_ioctx
.hit_set_get(hash
, c
, now
.sec(), &hbl
));
2214 c
->wait_for_complete();
2218 bufferlist::iterator p
= hbl
.begin();
2221 if (hs
.contains(oid
)) {
2222 cout
<< "ok, hit_set contains " << oid
<< std::endl
;
2225 cout
<< "hmm, not in HitSet yet" << std::endl
;
2227 cout
<< "hmm, no HitSet yet" << std::endl
;
2234 static int _get_pg_num(Rados
& cluster
, string pool_name
)
2237 string cmd
= string("{\"prefix\": \"osd pool get\",\"pool\":\"")
2239 + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
2241 int r
= cluster
.mon_command(cmd
, inbl
, &outbl
, NULL
);
2243 string
outstr(outbl
.c_str(), outbl
.length());
2244 json_spirit::Value v
;
2245 if (!json_spirit::read(outstr
, v
)) {
2246 cerr
<<" unable to parse json " << outstr
<< std::endl
;
2250 json_spirit::Object
& o
= v
.get_obj();
2251 for (json_spirit::Object::size_type i
=0; i
<o
.size(); i
++) {
2252 json_spirit::Pair
& p
= o
[i
];
2253 if (p
.name_
== "pg_num") {
2254 cout
<< "pg_num = " << p
.value_
.get_int() << std::endl
;
2255 return p
.value_
.get_int();
2258 cerr
<< "didn't find pg_num in " << outstr
<< std::endl
;
2263 TEST_F(LibRadosTwoPoolsPP
, HitSetWrite
) {
2264 int num_pg
= _get_pg_num(cluster
, pool_name
);
2269 ASSERT_EQ(0, cluster
.mon_command(
2270 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2271 "\", \"tierpool\": \"" + cache_pool_name
+
2272 "\", \"force_nonempty\": \"--force-nonempty\" }",
2275 // enable hitset tracking for this pool
2276 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", 8),
2278 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", 600),
2280 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type",
2284 // wait for maps to settle
2285 cluster
.wait_for_latest_osdmap();
2287 cache_ioctx
.set_namespace("");
2291 // do a bunch of writes
2292 for (int i
=0; i
<num
; ++i
) {
2295 ASSERT_EQ(0, cache_ioctx
.write(stringify(i
), bl
, 1, 0));
2299 std::map
<int,HitSet
> hitsets
;
2300 for (int i
=0; i
<num_pg
; ++i
) {
2301 list
< pair
<time_t,time_t> > ls
;
2302 AioCompletion
*c
= librados::Rados::aio_create_completion();
2303 ASSERT_EQ(0, cache_ioctx
.hit_set_list(i
, c
, &ls
));
2304 c
->wait_for_complete();
2306 std::cout
<< "pg " << i
<< " ls " << ls
<< std::endl
;
2307 ASSERT_FALSE(ls
.empty());
2310 c
= librados::Rados::aio_create_completion();
2312 ASSERT_EQ(0, cache_ioctx
.hit_set_get(i
, c
, ls
.back().first
, &bl
));
2313 c
->wait_for_complete();
2317 bufferlist::iterator p
= bl
.begin();
2318 ::decode(hitsets
[i
], p
);
2320 catch (buffer::error
& e
) {
2321 std::cout
<< "failed to decode hit set; bl len is " << bl
.length() << "\n";
2322 bl
.hexdump(std::cout
);
2323 std::cout
<< std::endl
;
2327 // cope with racing splits by refreshing pg_num
2328 if (i
== num_pg
- 1)
2329 num_pg
= _get_pg_num(cluster
, cache_pool_name
);
2332 for (int i
=0; i
<num
; ++i
) {
2333 string n
= stringify(i
);
2335 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(n
, &hash
));
2336 hobject_t
oid(sobject_t(n
, CEPH_NOSNAP
), "", hash
,
2337 cluster
.pool_lookup(cache_pool_name
.c_str()), "");
2338 std::cout
<< "checking for " << oid
<< std::endl
;
2340 for (int p
=0; p
<num_pg
; ++p
) {
2341 if (hitsets
[p
].contains(oid
)) {
2350 TEST_F(LibRadosTwoPoolsPP
, HitSetTrim
) {
2352 unsigned period
= 3;
2356 ASSERT_EQ(0, cluster
.mon_command(
2357 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2358 "\", \"tierpool\": \"" + cache_pool_name
+
2359 "\", \"force_nonempty\": \"--force-nonempty\" }",
2362 // enable hitset tracking for this pool
2363 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", count
),
2365 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", period
),
2367 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
2369 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_fpp", ".01"),
2372 // wait for maps to settle
2373 cluster
.wait_for_latest_osdmap();
2375 cache_ioctx
.set_namespace("");
2377 // do a bunch of writes and make sure the hitsets rotate
2378 utime_t start
= ceph_clock_now();
2379 utime_t hard_stop
= start
+ utime_t(count
* period
* 50, 0);
2383 string name
= "foo";
2385 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(name
, &hash
));
2386 hobject_t
oid(sobject_t(name
, CEPH_NOSNAP
), "", hash
, -1, "");
2390 ASSERT_EQ(0, cache_ioctx
.write("foo", bl
, 1, 0));
2392 list
<pair
<time_t, time_t> > ls
;
2393 AioCompletion
*c
= librados::Rados::aio_create_completion();
2394 ASSERT_EQ(0, cache_ioctx
.hit_set_list(hash
, c
, &ls
));
2395 c
->wait_for_complete();
2398 cout
<< " got ls " << ls
<< std::endl
;
2401 first
= ls
.front().first
;
2402 cout
<< "first is " << first
<< std::endl
;
2404 if (ls
.front().first
!= first
) {
2405 cout
<< "first now " << ls
.front().first
<< ", trimmed" << std::endl
;
2411 utime_t now
= ceph_clock_now();
2412 ASSERT_TRUE(now
< hard_stop
);
2418 TEST_F(LibRadosTwoPoolsPP
, PromoteOn2ndRead
) {
2420 for (int i
=0; i
<20; ++i
) {
2422 bl
.append("hi there");
2423 ObjectWriteOperation op
;
2425 ASSERT_EQ(0, ioctx
.operate("foo" + stringify(i
), &op
));
2430 ASSERT_EQ(0, cluster
.mon_command(
2431 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2432 "\", \"tierpool\": \"" + cache_pool_name
+
2433 "\", \"force_nonempty\": \"--force-nonempty\" }",
2435 ASSERT_EQ(0, cluster
.mon_command(
2436 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2437 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2439 ASSERT_EQ(0, cluster
.mon_command(
2440 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2441 "\", \"mode\": \"writeback\"}",
2444 // enable hitset tracking for this pool
2445 ASSERT_EQ(0, cluster
.mon_command(
2446 set_pool_str(cache_pool_name
, "hit_set_count", 2),
2448 ASSERT_EQ(0, cluster
.mon_command(
2449 set_pool_str(cache_pool_name
, "hit_set_period", 600),
2451 ASSERT_EQ(0, cluster
.mon_command(
2452 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
2454 ASSERT_EQ(0, cluster
.mon_command(
2455 set_pool_str(cache_pool_name
, "min_read_recency_for_promote", 1),
2457 ASSERT_EQ(0, cluster
.mon_command(
2458 set_pool_str(cache_pool_name
, "hit_set_grade_decay_rate", 20),
2460 ASSERT_EQ(0, cluster
.mon_command(
2461 set_pool_str(cache_pool_name
, "hit_set_search_last_n", 1),
2464 // wait for maps to settle
2465 cluster
.wait_for_latest_osdmap();
2467 int fake
= 0; // set this to non-zero to test spurious promotion,
2468 // e.g. from thrashing
2472 // 1st read, don't trigger a promote
2473 obj
= "foo" + stringify(attempt
);
2474 cout
<< obj
<< std::endl
;
2477 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
2480 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
2485 // verify the object is NOT present in the cache tier
2488 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2489 while (it
!= cache_ioctx
.nobjects_end()) {
2490 cout
<< " see " << it
->get_oid() << std::endl
;
2491 if (it
->get_oid() == string(obj
.c_str())) {
2502 ASSERT_LE(attempt
, 20);
2503 cout
<< "hrm, object is present in cache on attempt " << attempt
2504 << ", retrying" << std::endl
;
2507 // Read until the object is present in the cache tier
2508 cout
<< "verifying " << obj
<< " is eventually promoted" << std::endl
;
2511 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
2514 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2515 while (it
!= cache_ioctx
.nobjects_end()) {
2516 if (it
->get_oid() == string(obj
.c_str())) {
2529 ASSERT_EQ(0, cluster
.mon_command(
2530 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
2533 ASSERT_EQ(0, cluster
.mon_command(
2534 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
2535 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2538 // wait for maps to settle before next test
2539 cluster
.wait_for_latest_osdmap();
2542 TEST_F(LibRadosTwoPoolsPP
, ProxyRead
) {
2546 bl
.append("hi there");
2547 ObjectWriteOperation op
;
2549 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2554 ASSERT_EQ(0, cluster
.mon_command(
2555 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2556 "\", \"tierpool\": \"" + cache_pool_name
+
2557 "\", \"force_nonempty\": \"--force-nonempty\" }",
2559 ASSERT_EQ(0, cluster
.mon_command(
2560 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2561 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2563 ASSERT_EQ(0, cluster
.mon_command(
2564 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2565 "\", \"mode\": \"readproxy\"}",
2568 // wait for maps to settle
2569 cluster
.wait_for_latest_osdmap();
2571 // read and verify the object
2574 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2575 ASSERT_EQ('h', bl
[0]);
2578 // Verify 10 times the object is NOT present in the cache tier
2581 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2582 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
2587 ASSERT_EQ(0, cluster
.mon_command(
2588 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
2591 ASSERT_EQ(0, cluster
.mon_command(
2592 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
2593 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2596 // wait for maps to settle before next test
2597 cluster
.wait_for_latest_osdmap();
2600 TEST_F(LibRadosTwoPoolsPP
, CachePin
) {
2604 bl
.append("hi there");
2605 ObjectWriteOperation op
;
2607 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2611 bl
.append("hi there");
2612 ObjectWriteOperation op
;
2614 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
2618 bl
.append("hi there");
2619 ObjectWriteOperation op
;
2621 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
2625 bl
.append("hi there");
2626 ObjectWriteOperation op
;
2628 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
2633 ASSERT_EQ(0, cluster
.mon_command(
2634 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2635 "\", \"tierpool\": \"" + cache_pool_name
+
2636 "\", \"force_nonempty\": \"--force-nonempty\" }",
2638 ASSERT_EQ(0, cluster
.mon_command(
2639 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2640 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2642 ASSERT_EQ(0, cluster
.mon_command(
2643 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
2644 "\", \"mode\": \"writeback\"}",
2647 // wait for maps to settle
2648 cluster
.wait_for_latest_osdmap();
2650 // read, trigger promote
2653 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2654 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
2655 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
2656 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
2659 // verify the objects are present in the cache tier
2661 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2662 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
2663 for (uint32_t i
= 0; i
< 4; i
++) {
2664 ASSERT_TRUE(it
->get_oid() == string("foo") ||
2665 it
->get_oid() == string("bar") ||
2666 it
->get_oid() == string("baz") ||
2667 it
->get_oid() == string("bam"));
2670 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
2675 ObjectWriteOperation op
;
2677 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2678 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
2679 completion
->wait_for_safe();
2680 ASSERT_EQ(0, completion
->get_return_value());
2681 completion
->release();
2684 ObjectWriteOperation op
;
2686 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2687 ASSERT_EQ(0, cache_ioctx
.aio_operate("baz", completion
, &op
));
2688 completion
->wait_for_safe();
2689 ASSERT_EQ(0, completion
->get_return_value());
2690 completion
->release();
2694 ASSERT_EQ(0, cluster
.mon_command(
2695 set_pool_str(cache_pool_name
, "hit_set_count", 2),
2697 ASSERT_EQ(0, cluster
.mon_command(
2698 set_pool_str(cache_pool_name
, "hit_set_period", 600),
2700 ASSERT_EQ(0, cluster
.mon_command(
2701 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
2703 ASSERT_EQ(0, cluster
.mon_command(
2704 set_pool_str(cache_pool_name
, "min_read_recency_for_promote", 1),
2706 ASSERT_EQ(0, cluster
.mon_command(
2707 set_pool_str(cache_pool_name
, "target_max_objects", 1),
2712 // Verify the pinned object 'foo' is not flushed/evicted
2716 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
2719 NObjectIterator it
= cache_ioctx
.nobjects_begin();
2720 while (it
!= cache_ioctx
.nobjects_end()) {
2721 ASSERT_TRUE(it
->get_oid() == string("foo") ||
2722 it
->get_oid() == string("bar") ||
2723 it
->get_oid() == string("baz") ||
2724 it
->get_oid() == string("bam"));
2729 ASSERT_TRUE(it
->get_oid() == string("foo") ||
2730 it
->get_oid() == string("baz"));
2738 ASSERT_EQ(0, cluster
.mon_command(
2739 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
2742 ASSERT_EQ(0, cluster
.mon_command(
2743 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
2744 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2747 // wait for maps to settle before next test
2748 cluster
.wait_for_latest_osdmap();
2751 TEST_F(LibRadosTwoPoolsPP
, SetRedirectRead
) {
2752 // skip test if not yet luminous
2754 bufferlist inbl
, outbl
;
2755 ASSERT_EQ(0, cluster
.mon_command(
2756 "{\"prefix\": \"osd dump\"}",
2757 inbl
, &outbl
, NULL
));
2758 string
s(outbl
.c_str(), outbl
.length());
2759 if (s
.find("luminous") == std::string::npos
) {
2760 cout
<< "cluster is not yet luminous, skipping test" << std::endl
;
2768 bl
.append("hi there");
2769 ObjectWriteOperation op
;
2771 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2776 ObjectWriteOperation op
;
2778 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
2783 ASSERT_EQ(0, cluster
.mon_command(
2784 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2785 "\", \"tierpool\": \"" + cache_pool_name
+
2786 "\", \"force_nonempty\": \"--force-nonempty\" }",
2789 // wait for maps to settle
2790 cluster
.wait_for_latest_osdmap();
2793 ObjectWriteOperation op
;
2794 op
.set_redirect("bar", cache_ioctx
, 0);
2795 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2796 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
2797 completion
->wait_for_safe();
2798 ASSERT_EQ(0, completion
->get_return_value());
2799 completion
->release();
2801 // read and verify the object
2804 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2805 ASSERT_EQ('t', bl
[0]);
2808 ASSERT_EQ(0, cluster
.mon_command(
2809 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
2810 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2813 // wait for maps to settle before next test
2814 cluster
.wait_for_latest_osdmap();
2817 class LibRadosTwoPoolsECPP
: public RadosTestECPP
2820 LibRadosTwoPoolsECPP() {};
2821 ~LibRadosTwoPoolsECPP() override
{};
2823 static void SetUpTestCase() {
2824 pool_name
= get_temp_pool_name();
2825 ASSERT_EQ("", create_one_ec_pool_pp(pool_name
, s_cluster
));
2827 static void TearDownTestCase() {
2828 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name
, s_cluster
));
2830 static std::string cache_pool_name
;
2832 void SetUp() override
{
2833 cache_pool_name
= get_temp_pool_name();
2834 ASSERT_EQ(0, s_cluster
.pool_create(cache_pool_name
.c_str()));
2835 RadosTestECPP::SetUp();
2837 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
2838 cache_ioctx
.application_enable("rados", true);
2839 cache_ioctx
.set_namespace(nspace
);
2841 void TearDown() override
{
2842 // flush + evict cache
2843 flush_evict_all(cluster
, cache_ioctx
);
2847 ASSERT_EQ(0, cluster
.mon_command(
2848 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
2851 ASSERT_EQ(0, cluster
.mon_command(
2852 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
2853 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
2856 // wait for maps to settle before next test
2857 cluster
.wait_for_latest_osdmap();
2859 RadosTestECPP::TearDown();
2861 cleanup_default_namespace(cache_ioctx
);
2862 cleanup_namespace(cache_ioctx
, nspace
);
2864 cache_ioctx
.close();
2865 ASSERT_EQ(0, s_cluster
.pool_delete(cache_pool_name
.c_str()));
2868 librados::IoCtx cache_ioctx
;
2871 std::string
LibRadosTwoPoolsECPP::cache_pool_name
;
2873 TEST_F(LibRadosTierECPP
, Dirty
) {
2875 ObjectWriteOperation op
;
2877 ASSERT_EQ(0, ioctx
.operate("foo", &op
)); // still get 0 if it dne
2880 ObjectWriteOperation op
;
2882 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2887 ObjectReadOperation op
;
2888 op
.is_dirty(&dirty
, &r
);
2889 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
2894 ObjectWriteOperation op
;
2896 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2899 ObjectWriteOperation op
;
2901 ASSERT_EQ(0, ioctx
.operate("foo", &op
)); // still 0 if already clean
2906 ObjectReadOperation op
;
2907 op
.is_dirty(&dirty
, &r
);
2908 ASSERT_EQ(0, ioctx
.operate("foo", &op
, NULL
));
2909 ASSERT_FALSE(dirty
);
2913 // ObjectWriteOperation op;
2914 // op.truncate(0); // still a write even tho it is a no-op
2915 // ASSERT_EQ(0, ioctx.operate("foo", &op));
2918 // bool dirty = false;
2920 // ObjectReadOperation op;
2921 // op.is_dirty(&dirty, &r);
2922 // ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
2923 // ASSERT_TRUE(dirty);
2928 TEST_F(LibRadosTwoPoolsECPP
, Overlay
) {
2933 ObjectWriteOperation op
;
2935 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
2940 ObjectWriteOperation op
;
2942 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
));
2947 ASSERT_EQ(0, cluster
.mon_command(
2948 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
2949 "\", \"tierpool\": \"" + cache_pool_name
+
2950 "\", \"force_nonempty\": \"--force-nonempty\" }",
2952 ASSERT_EQ(0, cluster
.mon_command(
2953 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
2954 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
2957 // wait for maps to settle
2958 cluster
.wait_for_latest_osdmap();
2960 // by default, the overlay sends us to cache pool
2963 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
2964 ASSERT_EQ('c', bl
[0]);
2968 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
2969 ASSERT_EQ('c', bl
[0]);
2972 // unless we say otherwise
2975 ObjectReadOperation op
;
2976 op
.read(0, 1, &bl
, NULL
);
2977 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
2978 ASSERT_EQ(0, ioctx
.aio_operate(
2979 "foo", completion
, &op
,
2980 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
2981 completion
->wait_for_safe();
2982 ASSERT_EQ(0, completion
->get_return_value());
2983 completion
->release();
2984 ASSERT_EQ('b', bl
[0]);
2988 TEST_F(LibRadosTwoPoolsECPP
, Promote
) {
2992 bl
.append("hi there");
2993 ObjectWriteOperation op
;
2995 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3000 ASSERT_EQ(0, cluster
.mon_command(
3001 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
3002 "\", \"tierpool\": \"" + cache_pool_name
+
3003 "\", \"force_nonempty\": \"--force-nonempty\" }",
3005 ASSERT_EQ(0, cluster
.mon_command(
3006 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
3007 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
3009 ASSERT_EQ(0, cluster
.mon_command(
3010 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
3011 "\", \"mode\": \"writeback\"}",
3014 // wait for maps to settle
3015 cluster
.wait_for_latest_osdmap();
3017 // read, trigger a promote
3020 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3023 // read, trigger a whiteout
3026 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
3027 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
3030 // verify the object is present in the cache tier
3032 NObjectIterator it
= cache_ioctx
.nobjects_begin();
3033 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
3034 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
3036 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
3038 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
3042 TEST_F(LibRadosTwoPoolsECPP
, PromoteSnap
) {
3046 bl
.append("hi there");
3047 ObjectWriteOperation op
;
3049 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3053 bl
.append("hi there");
3054 ObjectWriteOperation op
;
3056 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
3060 bl
.append("hi there");
3061 ObjectWriteOperation op
;
3063 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
3067 bl
.append("hi there");
3068 ObjectWriteOperation op
;
3070 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
3073 // create a snapshot, clone
3074 vector
<uint64_t> my_snaps(1);
3075 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
3076 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
3081 ObjectWriteOperation op
;
3083 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3088 ObjectWriteOperation op
;
3090 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
3093 ObjectWriteOperation op
;
3095 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
3100 ObjectWriteOperation op
;
3102 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
3107 ASSERT_EQ(0, cluster
.mon_command(
3108 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
3109 "\", \"tierpool\": \"" + cache_pool_name
+
3110 "\", \"force_nonempty\": \"--force-nonempty\" }",
3112 ASSERT_EQ(0, cluster
.mon_command(
3113 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
3114 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
3116 ASSERT_EQ(0, cluster
.mon_command(
3117 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
3118 "\", \"mode\": \"writeback\"}",
3121 // wait for maps to settle
3122 cluster
.wait_for_latest_osdmap();
3124 // read, trigger a promote on the head
3127 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3128 ASSERT_EQ('c', bl
[0]);
3132 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
3133 ASSERT_EQ('c', bl
[0]);
3136 ioctx
.snap_set_read(my_snaps
[0]);
3138 // stop and scrub this pg (to make sure scrub can handle missing
3139 // clones in the cache tier)
3140 // This test requires cache tier and base tier to have the same pg_num/pgp_num
3142 for (int tries
= 0; tries
< 5; ++tries
) {
3144 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
3146 ASSERT_EQ(0, ioctx
.get_object_pg_hash_position2("foo", &hash
));
3148 ss
<< "{\"prefix\": \"pg scrub\", \"pgid\": \""
3149 << cache_ioctx
.get_id() << "."
3152 int r
= cluster
.mon_command(ss
.str(), inbl
, NULL
, NULL
);
3154 r
== -ENOENT
) { // in case mgr osdmap is a bit stale
3161 // give it a few seconds to go. this is sloppy but is usually enough time
3162 cout
<< "waiting for scrub..." << std::endl
;
3164 cout
<< "done waiting" << std::endl
;
3170 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3171 ASSERT_EQ('h', bl
[0]);
3177 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
3178 ASSERT_EQ('h', bl
[0]);
3184 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
3185 ASSERT_EQ('h', bl
[0]);
3188 ioctx
.snap_set_read(librados::SNAP_HEAD
);
3193 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3194 ASSERT_EQ('c', bl
[0]);
3200 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
3201 ASSERT_EQ('c', bl
[0]);
3207 ASSERT_EQ(-ENOENT
, ioctx
.read("baz", bl
, 1, 0));
3211 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
3214 TEST_F(LibRadosTwoPoolsECPP
, PromoteSnapTrimRace
) {
3218 bl
.append("hi there");
3219 ObjectWriteOperation op
;
3221 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3224 // create a snapshot, clone
3225 vector
<uint64_t> my_snaps(1);
3226 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
3227 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
3232 ObjectWriteOperation op
;
3234 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3239 ASSERT_EQ(0, cluster
.mon_command(
3240 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
3241 "\", \"tierpool\": \"" + cache_pool_name
+
3242 "\", \"force_nonempty\": \"--force-nonempty\" }",
3244 ASSERT_EQ(0, cluster
.mon_command(
3245 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
3246 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
3248 ASSERT_EQ(0, cluster
.mon_command(
3249 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
3250 "\", \"mode\": \"writeback\"}",
3253 // wait for maps to settle
3254 cluster
.wait_for_latest_osdmap();
3257 ASSERT_EQ(0, ioctx
.selfmanaged_snap_remove(my_snaps
[0]));
3259 ioctx
.snap_set_read(my_snaps
[0]);
3264 ASSERT_EQ(-ENOENT
, ioctx
.read("foo", bl
, 1, 0));
3268 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
3271 TEST_F(LibRadosTwoPoolsECPP
, Whiteout
) {
3275 bl
.append("hi there");
3276 ObjectWriteOperation op
;
3278 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3283 ASSERT_EQ(0, cluster
.mon_command(
3284 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
3285 "\", \"tierpool\": \"" + cache_pool_name
+
3286 "\", \"force_nonempty\": \"--force-nonempty\" }",
3288 ASSERT_EQ(0, cluster
.mon_command(
3289 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
3290 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
3292 ASSERT_EQ(0, cluster
.mon_command(
3293 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
3294 "\", \"mode\": \"writeback\"}",
3297 // wait for maps to settle
3298 cluster
.wait_for_latest_osdmap();
3300 // create some whiteouts, verify they behave
3302 ObjectWriteOperation op
;
3305 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3309 ObjectWriteOperation op
;
3312 ASSERT_EQ(-ENOENT
, ioctx
.operate("bar", &op
));
3315 ObjectWriteOperation op
;
3318 ASSERT_EQ(-ENOENT
, ioctx
.operate("bar", &op
));
3321 // verify the whiteouts are there in the cache tier
3323 NObjectIterator it
= cache_ioctx
.nobjects_begin();
3324 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
3325 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
3327 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
3329 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
3332 // delete a whiteout and verify it goes away
3333 ASSERT_EQ(-ENOENT
, ioctx
.remove("foo"));
3335 ObjectWriteOperation op
;
3337 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3338 ASSERT_EQ(0, ioctx
.aio_operate("bar", completion
, &op
,
3339 librados::OPERATION_IGNORE_CACHE
));
3340 completion
->wait_for_safe();
3341 ASSERT_EQ(0, completion
->get_return_value());
3342 completion
->release();
3344 NObjectIterator it
= cache_ioctx
.nobjects_begin();
3345 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
3346 ASSERT_TRUE(it
->get_oid() == string("foo"));
3348 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
3351 // recreate an object and verify we can read it
3354 bl
.append("hi there");
3355 ObjectWriteOperation op
;
3357 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3361 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3362 ASSERT_EQ('h', bl
[0]);
3366 TEST_F(LibRadosTwoPoolsECPP
, Evict
) {
3370 bl
.append("hi there");
3371 ObjectWriteOperation op
;
3373 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3378 ASSERT_EQ(0, cluster
.mon_command(
3379 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
3380 "\", \"tierpool\": \"" + cache_pool_name
+
3381 "\", \"force_nonempty\": \"--force-nonempty\" }",
3383 ASSERT_EQ(0, cluster
.mon_command(
3384 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
3385 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
3387 ASSERT_EQ(0, cluster
.mon_command(
3388 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
3389 "\", \"mode\": \"writeback\"}",
3392 // wait for maps to settle
3393 cluster
.wait_for_latest_osdmap();
3395 // read, trigger a promote
3398 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3401 // read, trigger a whiteout, and a dirty object
3404 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
3405 ASSERT_EQ(-ENOENT
, ioctx
.read("bar", bl
, 1, 0));
3406 ASSERT_EQ(0, ioctx
.write("bar", bl
, bl
.length(), 0));
3409 // verify the object is present in the cache tier
3411 NObjectIterator it
= cache_ioctx
.nobjects_begin();
3412 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
3413 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
3415 ASSERT_TRUE(it
->get_oid() == string("foo") || it
->get_oid() == string("bar"));
3417 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
3422 ObjectWriteOperation op
;
3424 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3425 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
3426 completion
->wait_for_safe();
3427 ASSERT_EQ(0, completion
->get_return_value());
3428 completion
->release();
3431 // evict the pinned object with -EPERM
3433 ObjectReadOperation op
;
3435 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3436 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
3437 librados::OPERATION_IGNORE_CACHE
,
3439 completion
->wait_for_safe();
3440 ASSERT_EQ(-EPERM
, completion
->get_return_value());
3441 completion
->release();
3446 ObjectWriteOperation op
;
3448 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3449 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
3450 completion
->wait_for_safe();
3451 ASSERT_EQ(0, completion
->get_return_value());
3452 completion
->release();
3457 ObjectReadOperation op
;
3459 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3460 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3461 "foo", completion
, &op
,
3462 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
3463 completion
->wait_for_safe();
3464 ASSERT_EQ(0, completion
->get_return_value());
3465 completion
->release();
3472 ObjectReadOperation op
;
3473 op
.is_dirty(&dirty
, &r
);
3474 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
3475 ASSERT_FALSE(dirty
);
3481 ObjectReadOperation op
;
3483 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3484 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
3485 librados::OPERATION_IGNORE_CACHE
,
3487 completion
->wait_for_safe();
3488 ASSERT_EQ(0, completion
->get_return_value());
3489 completion
->release();
3492 ObjectReadOperation op
;
3494 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3495 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3496 "foo", completion
, &op
,
3497 librados::OPERATION_IGNORE_CACHE
, NULL
));
3498 completion
->wait_for_safe();
3499 ASSERT_EQ(0, completion
->get_return_value());
3500 completion
->release();
3503 ObjectReadOperation op
;
3505 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3506 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3507 "bar", completion
, &op
,
3508 librados::OPERATION_IGNORE_CACHE
, NULL
));
3509 completion
->wait_for_safe();
3510 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
3511 completion
->release();
3515 TEST_F(LibRadosTwoPoolsECPP
, EvictSnap
) {
3519 bl
.append("hi there");
3520 ObjectWriteOperation op
;
3522 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3526 bl
.append("hi there");
3527 ObjectWriteOperation op
;
3529 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
3533 bl
.append("hi there");
3534 ObjectWriteOperation op
;
3536 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
3540 bl
.append("hi there");
3541 ObjectWriteOperation op
;
3543 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
3546 // create a snapshot, clone
3547 vector
<uint64_t> my_snaps(1);
3548 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
3549 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
3554 ObjectWriteOperation op
;
3556 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3561 ObjectWriteOperation op
;
3563 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
3566 ObjectWriteOperation op
;
3568 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
3573 ObjectWriteOperation op
;
3575 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
3580 ASSERT_EQ(0, cluster
.mon_command(
3581 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
3582 "\", \"tierpool\": \"" + cache_pool_name
+
3583 "\", \"force_nonempty\": \"--force-nonempty\" }",
3585 ASSERT_EQ(0, cluster
.mon_command(
3586 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
3587 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
3589 ASSERT_EQ(0, cluster
.mon_command(
3590 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
3591 "\", \"mode\": \"writeback\"}",
3594 // wait for maps to settle
3595 cluster
.wait_for_latest_osdmap();
3597 // read, trigger a promote on the head
3600 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3601 ASSERT_EQ('c', bl
[0]);
3605 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
3606 ASSERT_EQ('c', bl
[0]);
3611 ObjectReadOperation op
;
3613 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3614 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3615 "bam", completion
, &op
,
3616 librados::OPERATION_IGNORE_CACHE
, NULL
));
3617 completion
->wait_for_safe();
3618 ASSERT_EQ(0, completion
->get_return_value());
3619 completion
->release();
3623 ObjectReadOperation op
;
3624 op
.read(1, 0, &bl
, NULL
);
3625 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3626 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3627 "bam", completion
, &op
,
3628 librados::OPERATION_IGNORE_CACHE
, NULL
));
3629 completion
->wait_for_safe();
3630 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
3631 completion
->release();
3635 ioctx
.snap_set_read(my_snaps
[0]);
3638 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
3639 ASSERT_EQ('h', bl
[0]);
3644 ObjectReadOperation op
;
3646 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3647 ASSERT_EQ(0, ioctx
.aio_operate(
3648 "foo", completion
, &op
,
3649 librados::OPERATION_IGNORE_CACHE
, NULL
));
3650 completion
->wait_for_safe();
3651 ASSERT_EQ(0, completion
->get_return_value());
3652 completion
->release();
3657 ObjectReadOperation op
;
3658 op
.read(1, 0, &bl
, NULL
);
3659 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3660 ASSERT_EQ(0, ioctx
.aio_operate(
3661 "foo", completion
, &op
,
3662 librados::OPERATION_IGNORE_CACHE
, NULL
));
3663 completion
->wait_for_safe();
3664 ASSERT_EQ(-ENOENT
, completion
->get_return_value());
3665 completion
->release();
3667 // head is still there...
3668 ioctx
.snap_set_read(librados::SNAP_HEAD
);
3671 ObjectReadOperation op
;
3672 op
.read(1, 0, &bl
, NULL
);
3673 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3674 ASSERT_EQ(0, ioctx
.aio_operate(
3675 "foo", completion
, &op
,
3676 librados::OPERATION_IGNORE_CACHE
, NULL
));
3677 completion
->wait_for_safe();
3678 ASSERT_EQ(0, completion
->get_return_value());
3679 completion
->release();
3682 // promote head + snap of bar
3683 ioctx
.snap_set_read(librados::SNAP_HEAD
);
3686 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
3687 ASSERT_EQ('c', bl
[0]);
3689 ioctx
.snap_set_read(my_snaps
[0]);
3692 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
3693 ASSERT_EQ('h', bl
[0]);
3696 // evict bar head (fail)
3697 ioctx
.snap_set_read(librados::SNAP_HEAD
);
3699 ObjectReadOperation op
;
3701 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3702 ASSERT_EQ(0, ioctx
.aio_operate(
3703 "bar", completion
, &op
,
3704 librados::OPERATION_IGNORE_CACHE
, NULL
));
3705 completion
->wait_for_safe();
3706 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
3707 completion
->release();
3711 ioctx
.snap_set_read(my_snaps
[0]);
3713 ObjectReadOperation op
;
3715 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3716 ASSERT_EQ(0, ioctx
.aio_operate(
3717 "bar", completion
, &op
,
3718 librados::OPERATION_IGNORE_CACHE
, NULL
));
3719 completion
->wait_for_safe();
3720 ASSERT_EQ(0, completion
->get_return_value());
3721 completion
->release();
3724 ioctx
.snap_set_read(librados::SNAP_HEAD
);
3727 ObjectReadOperation op
;
3728 op
.read(1, 0, &bl
, NULL
);
3729 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3730 ASSERT_EQ(0, ioctx
.aio_operate(
3731 "bar", completion
, &op
,
3732 librados::OPERATION_IGNORE_CACHE
, NULL
));
3733 completion
->wait_for_safe();
3734 ASSERT_EQ(0, completion
->get_return_value());
3735 completion
->release();
3738 ObjectReadOperation op
;
3740 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3741 ASSERT_EQ(0, ioctx
.aio_operate(
3742 "bar", completion
, &op
,
3743 librados::OPERATION_IGNORE_CACHE
, NULL
));
3744 completion
->wait_for_safe();
3745 ASSERT_EQ(0, completion
->get_return_value());
3746 completion
->release();
3750 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
3753 TEST_F(LibRadosTwoPoolsECPP
, TryFlush
) {
3756 ASSERT_EQ(0, cluster
.mon_command(
3757 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
3758 "\", \"tierpool\": \"" + cache_pool_name
+
3759 "\", \"force_nonempty\": \"--force-nonempty\" }",
3761 ASSERT_EQ(0, cluster
.mon_command(
3762 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
3763 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
3765 ASSERT_EQ(0, cluster
.mon_command(
3766 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
3767 "\", \"mode\": \"writeback\"}",
3770 // wait for maps to settle
3771 cluster
.wait_for_latest_osdmap();
3776 bl
.append("hi there");
3777 ObjectWriteOperation op
;
3779 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3782 // verify the object is present in the cache tier
3784 NObjectIterator it
= cache_ioctx
.nobjects_begin();
3785 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
3786 ASSERT_TRUE(it
->get_oid() == string("foo"));
3788 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
3791 // verify the object is NOT present in the base tier
3793 NObjectIterator it
= ioctx
.nobjects_begin();
3794 ASSERT_TRUE(it
== ioctx
.nobjects_end());
3801 ObjectReadOperation op
;
3802 op
.is_dirty(&dirty
, &r
);
3803 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
3810 ObjectWriteOperation op
;
3812 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3813 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
3814 completion
->wait_for_safe();
3815 ASSERT_EQ(0, completion
->get_return_value());
3816 completion
->release();
3819 // flush the pinned object with -EPERM
3821 ObjectReadOperation op
;
3822 op
.cache_try_flush();
3823 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3824 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3825 "foo", completion
, &op
,
3826 librados::OPERATION_IGNORE_OVERLAY
|
3827 librados::OPERATION_SKIPRWLOCKS
, NULL
));
3828 completion
->wait_for_safe();
3829 ASSERT_EQ(-EPERM
, completion
->get_return_value());
3830 completion
->release();
3835 ObjectWriteOperation op
;
3837 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3838 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
3839 completion
->wait_for_safe();
3840 ASSERT_EQ(0, completion
->get_return_value());
3841 completion
->release();
3846 ObjectReadOperation op
;
3847 op
.cache_try_flush();
3848 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3849 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3850 "foo", completion
, &op
,
3851 librados::OPERATION_IGNORE_OVERLAY
|
3852 librados::OPERATION_SKIPRWLOCKS
, NULL
));
3853 completion
->wait_for_safe();
3854 ASSERT_EQ(0, completion
->get_return_value());
3855 completion
->release();
3862 ObjectReadOperation op
;
3863 op
.is_dirty(&dirty
, &r
);
3864 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
3865 ASSERT_FALSE(dirty
);
3869 // verify in base tier
3871 NObjectIterator it
= ioctx
.nobjects_begin();
3872 ASSERT_TRUE(it
!= ioctx
.nobjects_end());
3873 ASSERT_TRUE(it
->get_oid() == string("foo"));
3875 ASSERT_TRUE(it
== ioctx
.nobjects_end());
3880 ObjectReadOperation op
;
3882 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3883 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3884 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
3885 completion
->wait_for_safe();
3886 ASSERT_EQ(0, completion
->get_return_value());
3887 completion
->release();
3890 // verify no longer in cache tier
3892 NObjectIterator it
= cache_ioctx
.nobjects_begin();
3893 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
3897 TEST_F(LibRadosTwoPoolsECPP
, FailedFlush
) {
3900 ASSERT_EQ(0, cluster
.mon_command(
3901 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
3902 "\", \"tierpool\": \"" + cache_pool_name
+
3903 "\", \"force_nonempty\": \"--force-nonempty\" }",
3905 ASSERT_EQ(0, cluster
.mon_command(
3906 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
3907 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
3909 ASSERT_EQ(0, cluster
.mon_command(
3910 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
3911 "\", \"mode\": \"writeback\"}",
3914 // wait for maps to settle
3915 cluster
.wait_for_latest_osdmap();
3920 bl
.append("hi there");
3921 ObjectWriteOperation op
;
3923 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
3926 // verify the object is present in the cache tier
3928 NObjectIterator it
= cache_ioctx
.nobjects_begin();
3929 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
3930 ASSERT_TRUE(it
->get_oid() == string("foo"));
3932 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
3935 // verify the object is NOT present in the base tier
3937 NObjectIterator it
= ioctx
.nobjects_begin();
3938 ASSERT_TRUE(it
== ioctx
.nobjects_end());
3943 ObjectWriteOperation op
;
3944 std::map
<std::string
, bufferlist
> omap
;
3945 omap
["somekey"] = bufferlist();
3947 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3948 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
3949 completion
->wait_for_safe();
3950 ASSERT_EQ(0, completion
->get_return_value());
3951 completion
->release();
3956 ObjectReadOperation op
;
3958 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3959 ASSERT_EQ(0, cache_ioctx
.aio_operate(
3960 "foo", completion
, &op
,
3961 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
3962 completion
->wait_for_safe();
3963 ASSERT_NE(0, completion
->get_return_value());
3964 completion
->release();
3969 ObjectReadOperation op
;
3972 std::set
<std::string
> keys
;
3973 keys
.insert("somekey");
3974 std::map
<std::string
, bufferlist
> map
;
3976 op
.omap_get_vals_by_keys(keys
, &map
, &prval
);
3977 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
3978 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
, &bl
));
3980 bool completed
= completion
->is_complete();
3982 cache_ioctx
.aio_cancel(completion
);
3983 std::cerr
<< "Most probably test case will hang here, please reset manually" << std::endl
;
3984 ASSERT_TRUE(completed
); //in fact we are locked forever at test case shutdown unless fix for http://tracker.ceph.com/issues/14511 is applied. Seems there is no workaround for that
3986 completion
->release();
3988 // verify still not in base tier
3990 ASSERT_TRUE(ioctx
.nobjects_begin() == ioctx
.nobjects_end());
3994 ObjectWriteOperation op
;
3996 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4000 ObjectReadOperation op
;
4002 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4003 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4004 "foo", completion
, &op
,
4005 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4006 completion
->wait_for_safe();
4007 ASSERT_EQ(0, completion
->get_return_value());
4008 completion
->release();
4012 ObjectReadOperation op
;
4014 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4015 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4016 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
4017 completion
->wait_for_safe();
4018 ASSERT_EQ(0, completion
->get_return_value());
4019 completion
->release();
4022 // verify no longer in cache tier
4024 NObjectIterator it
= cache_ioctx
.nobjects_begin();
4025 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
4029 NObjectIterator it
= ioctx
.nobjects_begin();
4030 ASSERT_TRUE(it
== ioctx
.nobjects_end());
4034 TEST_F(LibRadosTwoPoolsECPP
, Flush
) {
4037 ASSERT_EQ(0, cluster
.mon_command(
4038 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
4039 "\", \"tierpool\": \"" + cache_pool_name
+
4040 "\", \"force_nonempty\": \"--force-nonempty\" }",
4042 ASSERT_EQ(0, cluster
.mon_command(
4043 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
4044 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
4046 ASSERT_EQ(0, cluster
.mon_command(
4047 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
4048 "\", \"mode\": \"writeback\"}",
4051 // wait for maps to settle
4052 cluster
.wait_for_latest_osdmap();
4054 uint64_t user_version
= 0;
4059 bl
.append("hi there");
4060 ObjectWriteOperation op
;
4062 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4065 // verify the object is present in the cache tier
4067 NObjectIterator it
= cache_ioctx
.nobjects_begin();
4068 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
4069 ASSERT_TRUE(it
->get_oid() == string("foo"));
4071 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
4074 // verify the object is NOT present in the base tier
4076 NObjectIterator it
= ioctx
.nobjects_begin();
4077 ASSERT_TRUE(it
== ioctx
.nobjects_end());
4084 ObjectReadOperation op
;
4085 op
.is_dirty(&dirty
, &r
);
4086 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
4089 user_version
= cache_ioctx
.get_last_version();
4094 ObjectWriteOperation op
;
4096 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4097 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
4098 completion
->wait_for_safe();
4099 ASSERT_EQ(0, completion
->get_return_value());
4100 completion
->release();
4103 // flush the pinned object with -EPERM
4105 ObjectReadOperation op
;
4106 op
.cache_try_flush();
4107 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4108 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4109 "foo", completion
, &op
,
4110 librados::OPERATION_IGNORE_OVERLAY
|
4111 librados::OPERATION_SKIPRWLOCKS
, NULL
));
4112 completion
->wait_for_safe();
4113 ASSERT_EQ(-EPERM
, completion
->get_return_value());
4114 completion
->release();
4119 ObjectWriteOperation op
;
4121 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4122 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
4123 completion
->wait_for_safe();
4124 ASSERT_EQ(0, completion
->get_return_value());
4125 completion
->release();
4130 ObjectReadOperation op
;
4132 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4133 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4134 "foo", completion
, &op
,
4135 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4136 completion
->wait_for_safe();
4137 ASSERT_EQ(0, completion
->get_return_value());
4138 completion
->release();
4145 ObjectReadOperation op
;
4146 op
.is_dirty(&dirty
, &r
);
4147 ASSERT_EQ(0, cache_ioctx
.operate("foo", &op
, NULL
));
4148 ASSERT_FALSE(dirty
);
4152 // verify in base tier
4154 NObjectIterator it
= ioctx
.nobjects_begin();
4155 ASSERT_TRUE(it
!= ioctx
.nobjects_end());
4156 ASSERT_TRUE(it
->get_oid() == string("foo"));
4158 ASSERT_TRUE(it
== ioctx
.nobjects_end());
4163 ObjectReadOperation op
;
4165 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4166 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4167 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
4168 completion
->wait_for_safe();
4169 ASSERT_EQ(0, completion
->get_return_value());
4170 completion
->release();
4173 // verify no longer in cache tier
4175 NObjectIterator it
= cache_ioctx
.nobjects_begin();
4176 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
4179 // read it again and verify the version is consistent
4182 ASSERT_EQ(1, cache_ioctx
.read("foo", bl
, 1, 0));
4183 ASSERT_EQ(user_version
, cache_ioctx
.get_last_version());
4188 ObjectWriteOperation op
;
4190 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4195 ObjectReadOperation op
;
4197 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4198 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4199 "foo", completion
, &op
,
4200 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4201 completion
->wait_for_safe();
4202 ASSERT_EQ(0, completion
->get_return_value());
4203 completion
->release();
4208 ObjectReadOperation op
;
4210 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4211 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4212 "foo", completion
, &op
, librados::OPERATION_IGNORE_CACHE
, NULL
));
4213 completion
->wait_for_safe();
4214 ASSERT_EQ(0, completion
->get_return_value());
4215 completion
->release();
4218 // verify no longer in cache tier
4220 NObjectIterator it
= cache_ioctx
.nobjects_begin();
4221 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
4225 NObjectIterator it
= ioctx
.nobjects_begin();
4226 ASSERT_TRUE(it
== ioctx
.nobjects_end());
4230 TEST_F(LibRadosTwoPoolsECPP
, FlushSnap
) {
4233 ASSERT_EQ(0, cluster
.mon_command(
4234 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
4235 "\", \"tierpool\": \"" + cache_pool_name
+
4236 "\", \"force_nonempty\": \"--force-nonempty\" }",
4238 ASSERT_EQ(0, cluster
.mon_command(
4239 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
4240 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
4242 ASSERT_EQ(0, cluster
.mon_command(
4243 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
4244 "\", \"mode\": \"writeback\"}",
4247 // wait for maps to settle
4248 cluster
.wait_for_latest_osdmap();
4254 ObjectWriteOperation op
;
4256 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4259 // create a snapshot, clone
4260 vector
<uint64_t> my_snaps(1);
4261 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4262 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4267 ObjectWriteOperation op
;
4269 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4274 my_snaps
[1] = my_snaps
[0];
4275 ASSERT_EQ(0, ioctx
.selfmanaged_snap_create(&my_snaps
[0]));
4276 ASSERT_EQ(0, ioctx
.selfmanaged_snap_set_write_ctx(my_snaps
[0],
4281 ObjectWriteOperation op
;
4283 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4286 // verify the object is present in the cache tier
4288 NObjectIterator it
= cache_ioctx
.nobjects_begin();
4289 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
4290 ASSERT_TRUE(it
->get_oid() == string("foo"));
4292 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
4295 // verify the object is NOT present in the base tier
4297 NObjectIterator it
= ioctx
.nobjects_begin();
4298 ASSERT_TRUE(it
== ioctx
.nobjects_end());
4301 // flush on head (should fail)
4302 ioctx
.snap_set_read(librados::SNAP_HEAD
);
4304 ObjectReadOperation op
;
4306 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4307 ASSERT_EQ(0, ioctx
.aio_operate(
4308 "foo", completion
, &op
,
4309 librados::OPERATION_IGNORE_CACHE
, NULL
));
4310 completion
->wait_for_safe();
4311 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
4312 completion
->release();
4314 // flush on recent snap (should fail)
4315 ioctx
.snap_set_read(my_snaps
[0]);
4317 ObjectReadOperation op
;
4319 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4320 ASSERT_EQ(0, ioctx
.aio_operate(
4321 "foo", completion
, &op
,
4322 librados::OPERATION_IGNORE_CACHE
, NULL
));
4323 completion
->wait_for_safe();
4324 ASSERT_EQ(-EBUSY
, completion
->get_return_value());
4325 completion
->release();
4327 // flush on oldest snap
4328 ioctx
.snap_set_read(my_snaps
[1]);
4330 ObjectReadOperation op
;
4332 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4333 ASSERT_EQ(0, ioctx
.aio_operate(
4334 "foo", completion
, &op
,
4335 librados::OPERATION_IGNORE_CACHE
, NULL
));
4336 completion
->wait_for_safe();
4337 ASSERT_EQ(0, completion
->get_return_value());
4338 completion
->release();
4340 // flush on next oldest snap
4341 ioctx
.snap_set_read(my_snaps
[0]);
4343 ObjectReadOperation op
;
4345 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4346 ASSERT_EQ(0, ioctx
.aio_operate(
4347 "foo", completion
, &op
,
4348 librados::OPERATION_IGNORE_CACHE
, NULL
));
4349 completion
->wait_for_safe();
4350 ASSERT_EQ(0, completion
->get_return_value());
4351 completion
->release();
4354 ioctx
.snap_set_read(librados::SNAP_HEAD
);
4356 ObjectReadOperation op
;
4358 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4359 ASSERT_EQ(0, ioctx
.aio_operate(
4360 "foo", completion
, &op
,
4361 librados::OPERATION_IGNORE_CACHE
, NULL
));
4362 completion
->wait_for_safe();
4363 ASSERT_EQ(0, completion
->get_return_value());
4364 completion
->release();
4367 // verify i can read the snaps from the cache pool
4368 ioctx
.snap_set_read(librados::SNAP_HEAD
);
4371 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
4372 ASSERT_EQ('c', bl
[0]);
4374 ioctx
.snap_set_read(my_snaps
[0]);
4377 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
4378 ASSERT_EQ('b', bl
[0]);
4380 ioctx
.snap_set_read(my_snaps
[1]);
4383 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
4384 ASSERT_EQ('a', bl
[0]);
4388 ASSERT_EQ(0, cluster
.mon_command(
4389 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
4393 // wait for maps to settle
4394 cluster
.wait_for_latest_osdmap();
4396 // verify i can read the snaps from the base pool
4397 ioctx
.snap_set_read(librados::SNAP_HEAD
);
4400 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
4401 ASSERT_EQ('c', bl
[0]);
4403 ioctx
.snap_set_read(my_snaps
[0]);
4406 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
4407 ASSERT_EQ('b', bl
[0]);
4409 ioctx
.snap_set_read(my_snaps
[1]);
4412 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
4413 ASSERT_EQ('a', bl
[0]);
4416 ASSERT_EQ(0, cluster
.mon_command(
4417 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
4418 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
4420 cluster
.wait_for_latest_osdmap();
4423 ioctx
.selfmanaged_snap_remove(my_snaps
[0]);
4426 TEST_F(LibRadosTierECPP
, FlushWriteRaces
) {
4428 std::string pool_name
= get_temp_pool_name();
4429 std::string cache_pool_name
= pool_name
+ "-cache";
4430 ASSERT_EQ("", create_one_pool_pp(pool_name
, cluster
));
4431 ASSERT_EQ(0, cluster
.pool_create(cache_pool_name
.c_str()));
4433 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
4434 cache_ioctx
.application_enable("rados", true);
4436 ASSERT_EQ(0, cluster
.ioctx_create(pool_name
.c_str(), ioctx
));
4440 ASSERT_EQ(0, cluster
.mon_command(
4441 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
4442 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
4444 ASSERT_EQ(0, cluster
.mon_command(
4445 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
4446 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
4448 ASSERT_EQ(0, cluster
.mon_command(
4449 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
4450 "\", \"mode\": \"writeback\"}",
4453 // wait for maps to settle
4454 cluster
.wait_for_latest_osdmap();
4456 // create/dirty object
4458 bl
.append("hi there");
4460 ObjectWriteOperation op
;
4462 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4467 ObjectReadOperation op
;
4469 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4470 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4471 "foo", completion
, &op
,
4472 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4474 ObjectWriteOperation op2
;
4476 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
4477 ASSERT_EQ(0, ioctx
.aio_operate(
4478 "foo", completion2
, &op2
, 0));
4480 completion
->wait_for_safe();
4481 completion2
->wait_for_safe();
4482 ASSERT_EQ(0, completion
->get_return_value());
4483 ASSERT_EQ(0, completion2
->get_return_value());
4484 completion
->release();
4485 completion2
->release();
4490 // create/dirty object
4493 bl
.append("hi there");
4494 ObjectWriteOperation op
;
4496 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4499 // try-flush + write
4501 ObjectReadOperation op
;
4502 op
.cache_try_flush();
4503 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4504 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4505 "foo", completion
, &op
,
4506 librados::OPERATION_IGNORE_OVERLAY
|
4507 librados::OPERATION_SKIPRWLOCKS
, NULL
));
4509 ObjectWriteOperation op2
;
4511 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
4512 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion2
, &op2
, 0));
4514 completion
->wait_for_safe();
4515 completion2
->wait_for_safe();
4516 int r
= completion
->get_return_value();
4517 ASSERT_TRUE(r
== -EBUSY
|| r
== 0);
4518 ASSERT_EQ(0, completion2
->get_return_value());
4519 completion
->release();
4520 completion2
->release();
4523 cout
<< "didn't get EBUSY, trying again" << std::endl
;
4525 ASSERT_TRUE(--tries
);
4529 ASSERT_EQ(0, cluster
.mon_command(
4530 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
4533 ASSERT_EQ(0, cluster
.mon_command(
4534 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
4535 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
4538 // wait for maps to settle before next test
4539 cluster
.wait_for_latest_osdmap();
4541 ASSERT_EQ(0, cluster
.pool_delete(cache_pool_name
.c_str()));
4542 ASSERT_EQ(0, destroy_one_pool_pp(pool_name
, cluster
));
4545 TEST_F(LibRadosTwoPoolsECPP
, FlushTryFlushRaces
) {
4548 ASSERT_EQ(0, cluster
.mon_command(
4549 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
4550 "\", \"tierpool\": \"" + cache_pool_name
+
4551 "\", \"force_nonempty\": \"--force-nonempty\" }",
4553 ASSERT_EQ(0, cluster
.mon_command(
4554 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
4555 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
4557 ASSERT_EQ(0, cluster
.mon_command(
4558 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
4559 "\", \"mode\": \"writeback\"}",
4562 // wait for maps to settle
4563 cluster
.wait_for_latest_osdmap();
4565 // create/dirty object
4568 bl
.append("hi there");
4569 ObjectWriteOperation op
;
4571 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4576 ObjectReadOperation op
;
4578 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4579 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4580 "foo", completion
, &op
,
4581 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4583 ObjectReadOperation op2
;
4585 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
4586 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4587 "foo", completion2
, &op2
,
4588 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4590 completion
->wait_for_safe();
4591 completion2
->wait_for_safe();
4592 ASSERT_EQ(0, completion
->get_return_value());
4593 ASSERT_EQ(0, completion2
->get_return_value());
4594 completion
->release();
4595 completion2
->release();
4598 // create/dirty object
4601 bl
.append("hi there");
4602 ObjectWriteOperation op
;
4604 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4607 // flush + try-flush
4609 ObjectReadOperation op
;
4611 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4612 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4613 "foo", completion
, &op
,
4614 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4616 ObjectReadOperation op2
;
4617 op2
.cache_try_flush();
4618 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
4619 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4620 "foo", completion2
, &op2
,
4621 librados::OPERATION_IGNORE_OVERLAY
|
4622 librados::OPERATION_SKIPRWLOCKS
, NULL
));
4624 completion
->wait_for_safe();
4625 completion2
->wait_for_safe();
4626 ASSERT_EQ(0, completion
->get_return_value());
4627 ASSERT_EQ(0, completion2
->get_return_value());
4628 completion
->release();
4629 completion2
->release();
4632 // create/dirty object
4637 bl
.append("hi there");
4638 ObjectWriteOperation op
;
4640 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4643 // try-flush + flush
4644 // (flush will not piggyback on try-flush)
4646 ObjectReadOperation op
;
4647 op
.cache_try_flush();
4648 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4649 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4650 "foo", completion
, &op
,
4651 librados::OPERATION_IGNORE_OVERLAY
|
4652 librados::OPERATION_SKIPRWLOCKS
, NULL
));
4654 ObjectReadOperation op2
;
4656 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
4657 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4658 "foo", completion2
, &op2
,
4659 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4661 completion
->wait_for_safe();
4662 completion2
->wait_for_safe();
4663 int r
= completion
->get_return_value();
4664 ASSERT_TRUE(r
== -EBUSY
|| r
== 0);
4665 ASSERT_EQ(0, completion2
->get_return_value());
4666 completion
->release();
4667 completion2
->release();
4670 cout
<< "didn't get EBUSY, trying again" << std::endl
;
4672 ASSERT_TRUE(--tries
);
4675 // create/dirty object
4678 bl
.append("hi there");
4679 ObjectWriteOperation op
;
4681 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4684 // try-flush + try-flush
4686 ObjectReadOperation op
;
4687 op
.cache_try_flush();
4688 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4689 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4690 "foo", completion
, &op
,
4691 librados::OPERATION_IGNORE_OVERLAY
|
4692 librados::OPERATION_SKIPRWLOCKS
, NULL
));
4694 ObjectReadOperation op2
;
4695 op2
.cache_try_flush();
4696 librados::AioCompletion
*completion2
= cluster
.aio_create_completion();
4697 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4698 "foo", completion2
, &op2
,
4699 librados::OPERATION_IGNORE_OVERLAY
|
4700 librados::OPERATION_SKIPRWLOCKS
, NULL
));
4702 completion
->wait_for_safe();
4703 completion2
->wait_for_safe();
4704 ASSERT_EQ(0, completion
->get_return_value());
4705 ASSERT_EQ(0, completion2
->get_return_value());
4706 completion
->release();
4707 completion2
->release();
4711 TEST_F(LibRadosTwoPoolsECPP
, TryFlushReadRace
) {
4714 ASSERT_EQ(0, cluster
.mon_command(
4715 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
4716 "\", \"tierpool\": \"" + cache_pool_name
+
4717 "\", \"force_nonempty\": \"--force-nonempty\" }",
4719 ASSERT_EQ(0, cluster
.mon_command(
4720 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
4721 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
4723 ASSERT_EQ(0, cluster
.mon_command(
4724 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
4725 "\", \"mode\": \"writeback\"}",
4728 // wait for maps to settle
4729 cluster
.wait_for_latest_osdmap();
4731 // create/dirty object
4734 bl
.append("hi there");
4735 bufferptr
bp(4000000); // make it big!
4738 ObjectWriteOperation op
;
4740 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4743 // start a continuous stream of reads
4744 read_ioctx
= &ioctx
;
4746 for (int i
= 0; i
< max_reads
; ++i
) {
4753 ObjectReadOperation op
;
4754 op
.cache_try_flush();
4755 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4756 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4757 "foo", completion
, &op
,
4758 librados::OPERATION_IGNORE_OVERLAY
|
4759 librados::OPERATION_SKIPRWLOCKS
, NULL
));
4761 completion
->wait_for_safe();
4762 ASSERT_EQ(0, completion
->get_return_value());
4763 completion
->release();
4768 while (num_reads
> 0)
4769 cond
.Wait(test_lock
);
4773 TEST_F(LibRadosTierECPP
, CallForcesPromote
) {
4775 std::string pool_name
= get_temp_pool_name();
4776 std::string cache_pool_name
= pool_name
+ "-cache";
4777 ASSERT_EQ("", create_one_ec_pool_pp(pool_name
, cluster
));
4778 ASSERT_EQ(0, cluster
.pool_create(cache_pool_name
.c_str()));
4780 ASSERT_EQ(0, cluster
.ioctx_create(cache_pool_name
.c_str(), cache_ioctx
));
4781 cache_ioctx
.application_enable("rados", true);
4783 ASSERT_EQ(0, cluster
.ioctx_create(pool_name
.c_str(), ioctx
));
4787 ASSERT_EQ(0, cluster
.mon_command(
4788 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
4789 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
4791 ASSERT_EQ(0, cluster
.mon_command(
4792 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
4793 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
4795 ASSERT_EQ(0, cluster
.mon_command(
4796 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
4797 "\", \"mode\": \"writeback\"}",
4800 // set things up such that the op would normally be proxied
4801 ASSERT_EQ(0, cluster
.mon_command(
4802 set_pool_str(cache_pool_name
, "hit_set_count", 2),
4804 ASSERT_EQ(0, cluster
.mon_command(
4805 set_pool_str(cache_pool_name
, "hit_set_period", 600),
4807 ASSERT_EQ(0, cluster
.mon_command(
4808 set_pool_str(cache_pool_name
, "hit_set_type",
4811 ASSERT_EQ(0, cluster
.mon_command(
4812 set_pool_str(cache_pool_name
, "min_read_recency_for_promote",
4816 // wait for maps to settle
4817 cluster
.wait_for_latest_osdmap();
4819 // create/dirty object
4821 bl
.append("hi there");
4823 ObjectWriteOperation op
;
4825 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
4830 ObjectReadOperation op
;
4832 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4833 ASSERT_EQ(0, cache_ioctx
.aio_operate(
4834 "foo", completion
, &op
,
4835 librados::OPERATION_IGNORE_OVERLAY
, NULL
));
4836 completion
->wait_for_safe();
4837 ASSERT_EQ(0, completion
->get_return_value());
4838 completion
->release();
4843 ObjectReadOperation op
;
4845 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
4846 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
,
4847 librados::OPERATION_IGNORE_CACHE
,
4849 completion
->wait_for_safe();
4850 ASSERT_EQ(0, completion
->get_return_value());
4851 completion
->release();
4856 ObjectReadOperation op
;
4858 op
.exec("rbd", "get_id", bl
);
4860 // should get EIO (not an rbd object), not -EOPNOTSUPP (we didn't promote)
4861 ASSERT_EQ(-5, ioctx
.operate("foo", &op
, &out
));
4864 // make sure foo is back in the cache tier
4866 NObjectIterator it
= cache_ioctx
.nobjects_begin();
4867 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
4868 ASSERT_TRUE(it
->get_oid() == string("foo"));
4870 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
4874 ASSERT_EQ(0, cluster
.mon_command(
4875 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
4878 ASSERT_EQ(0, cluster
.mon_command(
4879 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
4880 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
4883 // wait for maps to settle before next test
4884 cluster
.wait_for_latest_osdmap();
4886 ASSERT_EQ(0, cluster
.pool_delete(cache_pool_name
.c_str()));
4887 ASSERT_EQ(0, destroy_one_pool_pp(pool_name
, cluster
));
4890 TEST_F(LibRadosTierECPP
, HitSetNone
) {
4892 list
< pair
<time_t,time_t> > ls
;
4893 AioCompletion
*c
= librados::Rados::aio_create_completion();
4894 ASSERT_EQ(0, ioctx
.hit_set_list(123, c
, &ls
));
4895 c
->wait_for_complete();
4896 ASSERT_EQ(0, c
->get_return_value());
4897 ASSERT_TRUE(ls
.empty());
4902 AioCompletion
*c
= librados::Rados::aio_create_completion();
4903 ASSERT_EQ(0, ioctx
.hit_set_get(123, c
, 12345, &bl
));
4904 c
->wait_for_complete();
4905 ASSERT_EQ(-ENOENT
, c
->get_return_value());
4910 TEST_F(LibRadosTwoPoolsECPP
, HitSetRead
) {
4913 ASSERT_EQ(0, cluster
.mon_command(
4914 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
4915 "\", \"tierpool\": \"" + cache_pool_name
+
4916 "\", \"force_nonempty\": \"--force-nonempty\" }",
4919 // enable hitset tracking for this pool
4920 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", 2),
4922 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", 600),
4924 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type",
4928 // wait for maps to settle
4929 cluster
.wait_for_latest_osdmap();
4931 cache_ioctx
.set_namespace("");
4933 // keep reading until we see our object appear in the HitSet
4934 utime_t start
= ceph_clock_now();
4935 utime_t hard_stop
= start
+ utime_t(600, 0);
4938 utime_t now
= ceph_clock_now();
4939 ASSERT_TRUE(now
< hard_stop
);
4941 string name
= "foo";
4943 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(name
, &hash
));
4944 hobject_t
oid(sobject_t(name
, CEPH_NOSNAP
), "", hash
,
4945 cluster
.pool_lookup(cache_pool_name
.c_str()), "");
4948 ASSERT_EQ(-ENOENT
, cache_ioctx
.read("foo", bl
, 1, 0));
4951 AioCompletion
*c
= librados::Rados::aio_create_completion();
4952 ASSERT_EQ(0, cache_ioctx
.hit_set_get(hash
, c
, now
.sec(), &hbl
));
4953 c
->wait_for_complete();
4957 bufferlist::iterator p
= hbl
.begin();
4960 if (hs
.contains(oid
)) {
4961 cout
<< "ok, hit_set contains " << oid
<< std::endl
;
4964 cout
<< "hmm, not in HitSet yet" << std::endl
;
4966 cout
<< "hmm, no HitSet yet" << std::endl
;
4973 // disable this test until hitset-get reliably works on EC pools
4975 TEST_F(LibRadosTierECPP
, HitSetWrite
) {
4976 int num_pg
= _get_pg_num(cluster
, pool_name
);
4979 // enable hitset tracking for this pool
4981 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(pool_name
, "hit_set_count", 8),
4983 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(pool_name
, "hit_set_period", 600),
4985 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(pool_name
, "hit_set_type",
4989 // wait for maps to settle
4990 cluster
.wait_for_latest_osdmap();
4992 ioctx
.set_namespace("");
4994 // do a bunch of writes
4995 for (int i
=0; i
<1000; ++i
) {
4998 ASSERT_EQ(0, ioctx
.write(stringify(i
), bl
, 1, 0));
5002 std::map
<int,HitSet
> hitsets
;
5003 for (int i
=0; i
<num_pg
; ++i
) {
5004 list
< pair
<time_t,time_t> > ls
;
5005 AioCompletion
*c
= librados::Rados::aio_create_completion();
5006 ASSERT_EQ(0, ioctx
.hit_set_list(i
, c
, &ls
));
5007 c
->wait_for_complete();
5009 std::cout
<< "pg " << i
<< " ls " << ls
<< std::endl
;
5010 ASSERT_FALSE(ls
.empty());
5013 c
= librados::Rados::aio_create_completion();
5015 ASSERT_EQ(0, ioctx
.hit_set_get(i
, c
, ls
.back().first
, &bl
));
5016 c
->wait_for_complete();
5019 //std::cout << "bl len is " << bl.length() << "\n";
5020 //bl.hexdump(std::cout);
5021 //std::cout << std::endl;
5023 bufferlist::iterator p
= bl
.begin();
5024 ::decode(hitsets
[i
], p
);
5026 // cope with racing splits by refreshing pg_num
5027 if (i
== num_pg
- 1)
5028 num_pg
= _get_pg_num(cluster
, pool_name
);
5031 for (int i
=0; i
<1000; ++i
) {
5032 string n
= stringify(i
);
5033 uint32_t hash
= ioctx
.get_object_hash_position(n
);
5034 hobject_t
oid(sobject_t(n
, CEPH_NOSNAP
), "", hash
,
5035 cluster
.pool_lookup(pool_name
.c_str()), "");
5036 std::cout
<< "checking for " << oid
<< std::endl
;
5038 for (int p
=0; p
<num_pg
; ++p
) {
5039 if (hitsets
[p
].contains(oid
)) {
5049 TEST_F(LibRadosTwoPoolsECPP
, HitSetTrim
) {
5051 unsigned period
= 3;
5055 ASSERT_EQ(0, cluster
.mon_command(
5056 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
5057 "\", \"tierpool\": \"" + cache_pool_name
+
5058 "\", \"force_nonempty\": \"--force-nonempty\" }",
5061 // enable hitset tracking for this pool
5062 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_count", count
),
5064 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_period", period
),
5066 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
5068 ASSERT_EQ(0, cluster
.mon_command(set_pool_str(cache_pool_name
, "hit_set_fpp", ".01"),
5071 // wait for maps to settle
5072 cluster
.wait_for_latest_osdmap();
5074 cache_ioctx
.set_namespace("");
5076 // do a bunch of writes and make sure the hitsets rotate
5077 utime_t start
= ceph_clock_now();
5078 utime_t hard_stop
= start
+ utime_t(count
* period
* 50, 0);
5081 int bsize
= alignment
;
5082 char *buf
= (char *)new char[bsize
];
5083 memset(buf
, 'f', bsize
);
5086 string name
= "foo";
5088 ASSERT_EQ(0, cache_ioctx
.get_object_hash_position2(name
, &hash
));
5089 hobject_t
oid(sobject_t(name
, CEPH_NOSNAP
), "", hash
, -1, "");
5092 bl
.append(buf
, bsize
);
5093 ASSERT_EQ(0, cache_ioctx
.append("foo", bl
, bsize
));
5095 list
<pair
<time_t, time_t> > ls
;
5096 AioCompletion
*c
= librados::Rados::aio_create_completion();
5097 ASSERT_EQ(0, cache_ioctx
.hit_set_list(hash
, c
, &ls
));
5098 c
->wait_for_complete();
5101 cout
<< " got ls " << ls
<< std::endl
;
5104 first
= ls
.front().first
;
5105 cout
<< "first is " << first
<< std::endl
;
5107 if (ls
.front().first
!= first
) {
5108 cout
<< "first now " << ls
.front().first
<< ", trimmed" << std::endl
;
5114 utime_t now
= ceph_clock_now();
5115 ASSERT_TRUE(now
< hard_stop
);
5122 TEST_F(LibRadosTwoPoolsECPP
, PromoteOn2ndRead
) {
5124 for (int i
=0; i
<20; ++i
) {
5126 bl
.append("hi there");
5127 ObjectWriteOperation op
;
5129 ASSERT_EQ(0, ioctx
.operate("foo" + stringify(i
), &op
));
5134 ASSERT_EQ(0, cluster
.mon_command(
5135 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
5136 "\", \"tierpool\": \"" + cache_pool_name
+
5137 "\", \"force_nonempty\": \"--force-nonempty\" }",
5139 ASSERT_EQ(0, cluster
.mon_command(
5140 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
5141 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
5143 ASSERT_EQ(0, cluster
.mon_command(
5144 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
5145 "\", \"mode\": \"writeback\"}",
5148 // enable hitset tracking for this pool
5149 ASSERT_EQ(0, cluster
.mon_command(
5150 set_pool_str(cache_pool_name
, "hit_set_count", 2),
5152 ASSERT_EQ(0, cluster
.mon_command(
5153 set_pool_str(cache_pool_name
, "hit_set_period", 600),
5155 ASSERT_EQ(0, cluster
.mon_command(
5156 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
5158 ASSERT_EQ(0, cluster
.mon_command(
5159 set_pool_str(cache_pool_name
, "min_read_recency_for_promote", 1),
5161 ASSERT_EQ(0, cluster
.mon_command(
5162 set_pool_str(cache_pool_name
, "hit_set_grade_decay_rate", 20),
5164 ASSERT_EQ(0, cluster
.mon_command(
5165 set_pool_str(cache_pool_name
, "hit_set_search_last_n", 1),
5168 // wait for maps to settle
5169 cluster
.wait_for_latest_osdmap();
5171 int fake
= 0; // set this to non-zero to test spurious promotion,
5172 // e.g. from thrashing
5176 // 1st read, don't trigger a promote
5177 obj
= "foo" + stringify(attempt
);
5178 cout
<< obj
<< std::endl
;
5181 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
5184 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
5189 // verify the object is NOT present in the cache tier
5192 NObjectIterator it
= cache_ioctx
.nobjects_begin();
5193 while (it
!= cache_ioctx
.nobjects_end()) {
5194 cout
<< " see " << it
->get_oid() << std::endl
;
5195 if (it
->get_oid() == string(obj
.c_str())) {
5206 ASSERT_LE(attempt
, 20);
5207 cout
<< "hrm, object is present in cache on attempt " << attempt
5208 << ", retrying" << std::endl
;
5211 // Read until the object is present in the cache tier
5212 cout
<< "verifying " << obj
<< " is eventually promoted" << std::endl
;
5215 ASSERT_EQ(1, ioctx
.read(obj
.c_str(), bl
, 1, 0));
5218 NObjectIterator it
= cache_ioctx
.nobjects_begin();
5219 while (it
!= cache_ioctx
.nobjects_end()) {
5220 if (it
->get_oid() == string(obj
.c_str())) {
5233 ASSERT_EQ(0, cluster
.mon_command(
5234 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
5237 ASSERT_EQ(0, cluster
.mon_command(
5238 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
5239 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
5242 // wait for maps to settle before next test
5243 cluster
.wait_for_latest_osdmap();
5246 TEST_F(LibRadosTwoPoolsECPP
, ProxyRead
) {
5250 bl
.append("hi there");
5251 ObjectWriteOperation op
;
5253 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5258 ASSERT_EQ(0, cluster
.mon_command(
5259 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
5260 "\", \"tierpool\": \"" + cache_pool_name
+
5261 "\", \"force_nonempty\": \"--force-nonempty\" }",
5263 ASSERT_EQ(0, cluster
.mon_command(
5264 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
5265 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
5267 ASSERT_EQ(0, cluster
.mon_command(
5268 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
5269 "\", \"mode\": \"readproxy\"}",
5272 // wait for maps to settle
5273 cluster
.wait_for_latest_osdmap();
5275 // read and verify the object
5278 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
5279 ASSERT_EQ('h', bl
[0]);
5282 // Verify 10 times the object is NOT present in the cache tier
5285 NObjectIterator it
= cache_ioctx
.nobjects_begin();
5286 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
5291 ASSERT_EQ(0, cluster
.mon_command(
5292 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
5295 ASSERT_EQ(0, cluster
.mon_command(
5296 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
5297 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
5300 // wait for maps to settle before next test
5301 cluster
.wait_for_latest_osdmap();
5304 TEST_F(LibRadosTwoPoolsECPP
, CachePin
) {
5308 bl
.append("hi there");
5309 ObjectWriteOperation op
;
5311 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5315 bl
.append("hi there");
5316 ObjectWriteOperation op
;
5318 ASSERT_EQ(0, ioctx
.operate("bar", &op
));
5322 bl
.append("hi there");
5323 ObjectWriteOperation op
;
5325 ASSERT_EQ(0, ioctx
.operate("baz", &op
));
5329 bl
.append("hi there");
5330 ObjectWriteOperation op
;
5332 ASSERT_EQ(0, ioctx
.operate("bam", &op
));
5337 ASSERT_EQ(0, cluster
.mon_command(
5338 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
5339 "\", \"tierpool\": \"" + cache_pool_name
+
5340 "\", \"force_nonempty\": \"--force-nonempty\" }",
5342 ASSERT_EQ(0, cluster
.mon_command(
5343 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name
+
5344 "\", \"overlaypool\": \"" + cache_pool_name
+ "\"}",
5346 ASSERT_EQ(0, cluster
.mon_command(
5347 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name
+
5348 "\", \"mode\": \"writeback\"}",
5351 // wait for maps to settle
5352 cluster
.wait_for_latest_osdmap();
5354 // read, trigger promote
5357 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
5358 ASSERT_EQ(1, ioctx
.read("bar", bl
, 1, 0));
5359 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
5360 ASSERT_EQ(1, ioctx
.read("bam", bl
, 1, 0));
5363 // verify the objects are present in the cache tier
5365 NObjectIterator it
= cache_ioctx
.nobjects_begin();
5366 ASSERT_TRUE(it
!= cache_ioctx
.nobjects_end());
5367 for (uint32_t i
= 0; i
< 4; i
++) {
5368 ASSERT_TRUE(it
->get_oid() == string("foo") ||
5369 it
->get_oid() == string("bar") ||
5370 it
->get_oid() == string("baz") ||
5371 it
->get_oid() == string("bam"));
5374 ASSERT_TRUE(it
== cache_ioctx
.nobjects_end());
5379 ObjectWriteOperation op
;
5381 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5382 ASSERT_EQ(0, cache_ioctx
.aio_operate("foo", completion
, &op
));
5383 completion
->wait_for_safe();
5384 ASSERT_EQ(0, completion
->get_return_value());
5385 completion
->release();
5388 ObjectWriteOperation op
;
5390 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5391 ASSERT_EQ(0, cache_ioctx
.aio_operate("baz", completion
, &op
));
5392 completion
->wait_for_safe();
5393 ASSERT_EQ(0, completion
->get_return_value());
5394 completion
->release();
5398 ASSERT_EQ(0, cluster
.mon_command(
5399 set_pool_str(cache_pool_name
, "hit_set_count", 2),
5401 ASSERT_EQ(0, cluster
.mon_command(
5402 set_pool_str(cache_pool_name
, "hit_set_period", 600),
5404 ASSERT_EQ(0, cluster
.mon_command(
5405 set_pool_str(cache_pool_name
, "hit_set_type", "bloom"),
5407 ASSERT_EQ(0, cluster
.mon_command(
5408 set_pool_str(cache_pool_name
, "min_read_recency_for_promote", 1),
5410 ASSERT_EQ(0, cluster
.mon_command(
5411 set_pool_str(cache_pool_name
, "target_max_objects", 1),
5416 // Verify the pinned object 'foo' is not flushed/evicted
5420 ASSERT_EQ(1, ioctx
.read("baz", bl
, 1, 0));
5423 NObjectIterator it
= cache_ioctx
.nobjects_begin();
5424 while (it
!= cache_ioctx
.nobjects_end()) {
5425 ASSERT_TRUE(it
->get_oid() == string("foo") ||
5426 it
->get_oid() == string("bar") ||
5427 it
->get_oid() == string("baz") ||
5428 it
->get_oid() == string("bam"));
5433 ASSERT_TRUE(it
->get_oid() == string("foo") ||
5434 it
->get_oid() == string("baz"));
5442 ASSERT_EQ(0, cluster
.mon_command(
5443 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name
+
5446 ASSERT_EQ(0, cluster
.mon_command(
5447 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
5448 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
5451 // wait for maps to settle before next test
5452 cluster
.wait_for_latest_osdmap();
5454 TEST_F(LibRadosTwoPoolsECPP
, SetRedirectRead
) {
5455 // skip test if not yet luminous
5457 bufferlist inbl
, outbl
;
5458 ASSERT_EQ(0, cluster
.mon_command(
5459 "{\"prefix\": \"osd dump\"}",
5460 inbl
, &outbl
, NULL
));
5461 string
s(outbl
.c_str(), outbl
.length());
5462 if (s
.find("luminous") == std::string::npos
) {
5463 cout
<< "cluster is not yet luminous, skipping test" << std::endl
;
5471 bl
.append("hi there");
5472 ObjectWriteOperation op
;
5474 ASSERT_EQ(0, ioctx
.operate("foo", &op
));
5479 ObjectWriteOperation op
;
5481 ASSERT_EQ(0, cache_ioctx
.operate("bar", &op
));
5486 ASSERT_EQ(0, cluster
.mon_command(
5487 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name
+
5488 "\", \"tierpool\": \"" + cache_pool_name
+
5489 "\", \"force_nonempty\": \"--force-nonempty\" }",
5492 // wait for maps to settle
5493 cluster
.wait_for_latest_osdmap();
5496 ObjectWriteOperation op
;
5497 op
.set_redirect("bar", cache_ioctx
, 0);
5498 librados::AioCompletion
*completion
= cluster
.aio_create_completion();
5499 ASSERT_EQ(0, ioctx
.aio_operate("foo", completion
, &op
));
5500 completion
->wait_for_safe();
5501 ASSERT_EQ(0, completion
->get_return_value());
5502 completion
->release();
5504 // read and verify the object
5507 ASSERT_EQ(1, ioctx
.read("foo", bl
, 1, 0));
5508 ASSERT_EQ('t', bl
[0]);
5511 ASSERT_EQ(0, cluster
.mon_command(
5512 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name
+
5513 "\", \"tierpool\": \"" + cache_pool_name
+ "\"}",
5516 // wait for maps to settle before next test
5517 cluster
.wait_for_latest_osdmap();