]> git.proxmox.com Git - ceph.git/blame - ceph/src/test/librados/tier_cxx.cc
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
buildsys: switch source download to quincy
[ceph.git] / ceph / src / test / librados / tier_cxx.cc
CommitLineData
7c673cae
FG		 1// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2// vim: ts=8 sw=2 smarttab
3#include "gtest/gtest.h"
4
5#include "mds/mdstypes.h"
6#include "include/buffer.h"
7#include "include/rbd_types.h"
7c673cae
FG		 8#include "include/rados/librados.hpp"
9#include "include/stringify.h"
10#include "include/types.h"
11#include "global/global_context.h"
12#include "common/Cond.h"
11fdf7f2
TL		 13#include "test/librados/test_cxx.h"
14#include "test/librados/testcase_cxx.h"
7c673cae 15#include "json_spirit/json_spirit.h"
11fdf7f2 16#include "cls/cas/cls_cas_ops.h"
f67539c2 17#include "cls/cas/cls_cas_internal.h"
7c673cae
FG		 18
19#include "osd/HitSet.h"
20
21#include <errno.h>
22#include <map>
23#include <sstream>
24#include <string>
25
f67539c2
TL		 26#include "cls/cas/cls_cas_client.h"
27#include "cls/cas/cls_cas_internal.h"
28
7c673cae
FG		 29using namespace librados;
30using std::map;
31using std::ostringstream;
32using std::string;
33
34typedef RadosTestPP LibRadosTierPP;
35typedef RadosTestECPP LibRadosTierECPP;
36
37void flush_evict_all(librados::Rados& cluster, librados::IoCtx& cache_ioctx)
38{
39 bufferlist inbl;
40 cache_ioctx.set_namespace(all_nspaces);
41 for (NObjectIterator it = cache_ioctx.nobjects_begin();
42 it != cache_ioctx.nobjects_end(); ++it) {
43 cache_ioctx.locator_set_key(it->get_locator());
44 cache_ioctx.set_namespace(it->get_nspace());
45 {
46 ObjectReadOperation op;
47 op.cache_flush();
48 librados::AioCompletion *completion = cluster.aio_create_completion();
49 cache_ioctx.aio_operate(
50 it->get_oid(), completion, &op,
51 librados::OPERATION_IGNORE_OVERLAY, NULL);
9f95a23c 52 completion->wait_for_complete();
7c673cae
FG		 53 completion->get_return_value();
54 completion->release();
55 }
56 {
57 ObjectReadOperation op;
58 op.cache_evict();
59 librados::AioCompletion *completion = cluster.aio_create_completion();
60 cache_ioctx.aio_operate(
61 it->get_oid(), completion, &op,
62 librados::OPERATION_IGNORE_OVERLAY, NULL);
9f95a23c 63 completion->wait_for_complete();
7c673cae
FG		 64 completion->get_return_value();
65 completion->release();
66 }
67 }
68}
69
9f95a23c
TL		 70static string _get_required_osd_release(Rados& cluster)
71{
72 bufferlist inbl;
73 string cmd = string("{\"prefix\": \"osd dump\",\"format\":\"json\"}");
74 bufferlist outbl;
75 int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
76 ceph_assert(r >= 0);
77 string outstr(outbl.c_str(), outbl.length());
78 json_spirit::Value v;
79 if (!json_spirit::read(outstr, v)) {
80 cerr <<" unable to parse json " << outstr << std::endl;
81 return "";
82 }
83
84 json_spirit::Object& o = v.get_obj();
85 for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
86 json_spirit::Pair& p = o[i];
87 if (p.name_ == "require_osd_release") {
88 cout << "require_osd_release = " << p.value_.get_str() << std::endl;
89 return p.value_.get_str();
90 }
91 }
92 cerr << "didn't find require_osd_release in " << outstr << std::endl;
93 return "";
94}
95
f67539c2
TL		 96void manifest_set_chunk(Rados& cluster, librados::IoCtx& src_ioctx,
97 librados::IoCtx& tgt_ioctx,
98 uint64_t src_offset, uint64_t length,
99 std::string src_oid, std::string tgt_oid)
100{
101 ObjectReadOperation op;
102 op.set_chunk(src_offset, length, src_ioctx, src_oid, 0,
103 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
104 librados::AioCompletion *completion = cluster.aio_create_completion();
105 ASSERT_EQ(0, tgt_ioctx.aio_operate(tgt_oid, completion, &op,
106 librados::OPERATION_IGNORE_CACHE, NULL));
107 completion->wait_for_complete();
108 ASSERT_EQ(0, completion->get_return_value());
109 completion->release();
110}
111
112#include "common/ceph_crypto.h"
113using ceph::crypto::SHA1;
114#include "rgw/rgw_common.h"
115
116void check_fp_oid_refcount(librados::IoCtx& ioctx, std::string foid, uint64_t count,
117 std::string fp_algo = NULL)
118{
119 bufferlist t;
120 int size = foid.length();
121 if (fp_algo == "sha1") {
122 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
123 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
124 SHA1 sha1_gen;
125 sha1_gen.Update((const unsigned char *)foid.c_str(), size);
126 sha1_gen.Final(fingerprint);
127 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
128 ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
129 } else if (fp_algo.empty()) {
130 ioctx.getxattr(foid, CHUNK_REFCOUNT_ATTR, t);
131 } else if (!fp_algo.empty()) {
132 ceph_assert(0 == "unrecognized fingerprint algorithm");
133 }
134
135 chunk_refs_t refs;
136 try {
137 auto iter = t.cbegin();
138 decode(refs, iter);
139 } catch (buffer::error& err) {
140 ASSERT_TRUE(0);
141 }
a4b75251 142 ASSERT_LE(count, refs.count());
f67539c2
TL		 143}
144
145string get_fp_oid(string oid, std::string fp_algo = NULL)
146{
147 if (fp_algo == "sha1") {
148 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
149 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
150 SHA1 sha1_gen;
151 int size = oid.length();
152 sha1_gen.Update((const unsigned char *)oid.c_str(), size);
153 sha1_gen.Final(fingerprint);
154 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
155 return string(p_str);
156 }
157
158 return string();
159}
160
161void is_intended_refcount_state(librados::IoCtx& src_ioctx,
162 std::string src_oid,
163 librados::IoCtx& dst_ioctx,
164 std::string dst_oid,
165 int expected_refcount)
166{
167 int src_refcount = 0, dst_refcount = 0;
168 bufferlist t;
169 int r = dst_ioctx.getxattr(dst_oid, CHUNK_REFCOUNT_ATTR, t);
170 if (r == -ENOENT) {
171 dst_refcount = 0;
172 } else {
173 chunk_refs_t refs;
174 try {
175 auto iter = t.cbegin();
176 decode(refs, iter);
177 } catch (buffer::error& err) {
178 ceph_assert(0);
179 }
180 dst_refcount = refs.count();
181 }
182 for (int tries = 0; tries < 10; ++tries) {
183 r = cls_cas_references_chunk(src_ioctx, src_oid, dst_oid);
184 if (r == -ENOENT || r == -ENOLINK) {
185 src_refcount = 0;
186 } else if (r == -EBUSY) {
187 sleep(15);
188 continue;
189 } else {
190 src_refcount = r;
191 }
192 break;
193 }
194 ASSERT_TRUE(src_refcount >= 0);
195 ASSERT_TRUE(src_refcount == expected_refcount);
196 ASSERT_TRUE(src_refcount <= dst_refcount);
197}
198
7c673cae
FG		 199class LibRadosTwoPoolsPP : public RadosTestPP
200{
201public:
202 LibRadosTwoPoolsPP() {};
203 ~LibRadosTwoPoolsPP() override {};
204protected:
205 static void SetUpTestCase() {
206 pool_name = get_temp_pool_name();
207 ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
208 }
209 static void TearDownTestCase() {
210 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
211 }
212 static std::string cache_pool_name;
213
214 void SetUp() override {
215 cache_pool_name = get_temp_pool_name();
216 ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
217 RadosTestPP::SetUp();
c07f9fc5 218
7c673cae 219 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 220 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 221 cache_ioctx.set_namespace(nspace);
222 }
223 void TearDown() override {
224 // flush + evict cache
225 flush_evict_all(cluster, cache_ioctx);
226
227 bufferlist inbl;
228 // tear down tiers
229 ASSERT_EQ(0, cluster.mon_command(
230 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
231 "\"}",
232 inbl, NULL, NULL));
233 ASSERT_EQ(0, cluster.mon_command(
234 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
235 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
236 inbl, NULL, NULL));
237
238 // wait for maps to settle before next test
239 cluster.wait_for_latest_osdmap();
240
241 RadosTestPP::TearDown();
242
243 cleanup_default_namespace(cache_ioctx);
244 cleanup_namespace(cache_ioctx, nspace);
245
246 cache_ioctx.close();
247 ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
248 }
249 librados::IoCtx cache_ioctx;
250};
251
252class Completions
253{
254public:
255 Completions() = default;
256 librados::AioCompletion* getCompletion() {
257 librados::AioCompletion* comp = librados::Rados::aio_create_completion();
258 m_completions.push_back(comp);
259 return comp;
260 }
261
262 ~Completions() {
263 for (auto& comp : m_completions) {
264 comp->release();
265 }
266 }
267
268private:
269 vector<librados::AioCompletion *> m_completions;
270};
271
272Completions completions;
273
274std::string LibRadosTwoPoolsPP::cache_pool_name;
275
276TEST_F(LibRadosTierPP, Dirty) {
277 {
278 ObjectWriteOperation op;
279 op.undirty();
280 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
281 }
282 {
283 ObjectWriteOperation op;
284 op.create(true);
285 ASSERT_EQ(0, ioctx.operate("foo", &op));
286 }
287 {
288 bool dirty = false;
289 int r = -1;
290 ObjectReadOperation op;
291 op.is_dirty(&dirty, &r);
292 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
293 ASSERT_TRUE(dirty);
294 ASSERT_EQ(0, r);
295 }
296 {
297 ObjectWriteOperation op;
298 op.undirty();
299 ASSERT_EQ(0, ioctx.operate("foo", &op));
300 }
301 {
302 ObjectWriteOperation op;
303 op.undirty();
304 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
305 }
306 {
307 bool dirty = false;
308 int r = -1;
309 ObjectReadOperation op;
310 op.is_dirty(&dirty, &r);
311 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
312 ASSERT_FALSE(dirty);
313 ASSERT_EQ(0, r);
314 }
315 {
316 ObjectWriteOperation op;
317 op.truncate(0); // still a write even tho it is a no-op
318 ASSERT_EQ(0, ioctx.operate("foo", &op));
319 }
320 {
321 bool dirty = false;
322 int r = -1;
323 ObjectReadOperation op;
324 op.is_dirty(&dirty, &r);
325 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
326 ASSERT_TRUE(dirty);
327 ASSERT_EQ(0, r);
328 }
329}
330
331TEST_F(LibRadosTwoPoolsPP, Overlay) {
332 // create objects
333 {
334 bufferlist bl;
335 bl.append("base");
336 ObjectWriteOperation op;
337 op.write_full(bl);
338 ASSERT_EQ(0, ioctx.operate("foo", &op));
339 }
340 {
341 bufferlist bl;
342 bl.append("cache");
343 ObjectWriteOperation op;
344 op.write_full(bl);
345 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
346 }
347
348 // configure cache
349 bufferlist inbl;
350 ASSERT_EQ(0, cluster.mon_command(
351 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
352 "\", \"tierpool\": \"" + cache_pool_name +
353 "\", \"force_nonempty\": \"--force-nonempty\" }",
354 inbl, NULL, NULL));
355 ASSERT_EQ(0, cluster.mon_command(
356 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
357 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
358 inbl, NULL, NULL));
359
360 // wait for maps to settle
361 cluster.wait_for_latest_osdmap();
362
363 // by default, the overlay sends us to cache pool
364 {
365 bufferlist bl;
366 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
367 ASSERT_EQ('c', bl[0]);
368 }
369 {
370 bufferlist bl;
371 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
372 ASSERT_EQ('c', bl[0]);
373 }
374
375 // unless we say otherwise
376 {
377 bufferlist bl;
378 ObjectReadOperation op;
379 op.read(0, 1, &bl, NULL);
380 librados::AioCompletion *completion = cluster.aio_create_completion();
381 ASSERT_EQ(0, ioctx.aio_operate(
382 "foo", completion, &op,
383 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 384 completion->wait_for_complete();
7c673cae
FG		 385 ASSERT_EQ(0, completion->get_return_value());
386 completion->release();
387 ASSERT_EQ('b', bl[0]);
388 }
389}
390
391TEST_F(LibRadosTwoPoolsPP, Promote) {
392 // create object
393 {
394 bufferlist bl;
395 bl.append("hi there");
396 ObjectWriteOperation op;
397 op.write_full(bl);
398 ASSERT_EQ(0, ioctx.operate("foo", &op));
399 }
400
401 // configure cache
402 bufferlist inbl;
403 ASSERT_EQ(0, cluster.mon_command(
404 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
405 "\", \"tierpool\": \"" + cache_pool_name +
406 "\", \"force_nonempty\": \"--force-nonempty\" }",
407 inbl, NULL, NULL));
408 ASSERT_EQ(0, cluster.mon_command(
409 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
410 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
411 inbl, NULL, NULL));
412 ASSERT_EQ(0, cluster.mon_command(
413 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
414 "\", \"mode\": \"writeback\"}",
415 inbl, NULL, NULL));
416
417 // wait for maps to settle
418 cluster.wait_for_latest_osdmap();
419
420 // read, trigger a promote
421 {
422 bufferlist bl;
423 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
424 }
425
426 // read, trigger a whiteout
427 {
428 bufferlist bl;
429 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
430 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
431 }
432
433 // verify the object is present in the cache tier
434 {
435 NObjectIterator it = cache_ioctx.nobjects_begin();
436 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
437 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
438 ++it;
439 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
440 ++it;
441 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
442 }
443}
444
445TEST_F(LibRadosTwoPoolsPP, PromoteSnap) {
446 // create object
447 {
448 bufferlist bl;
449 bl.append("hi there");
450 ObjectWriteOperation op;
451 op.write_full(bl);
452 ASSERT_EQ(0, ioctx.operate("foo", &op));
453 }
454 {
455 bufferlist bl;
456 bl.append("hi there");
457 ObjectWriteOperation op;
458 op.write_full(bl);
459 ASSERT_EQ(0, ioctx.operate("bar", &op));
460 }
461 {
462 bufferlist bl;
463 bl.append("hi there");
464 ObjectWriteOperation op;
465 op.write_full(bl);
466 ASSERT_EQ(0, ioctx.operate("baz", &op));
467 }
468 {
469 bufferlist bl;
470 bl.append("hi there");
471 ObjectWriteOperation op;
472 op.write_full(bl);
473 ASSERT_EQ(0, ioctx.operate("bam", &op));
474 }
475
476 // create a snapshot, clone
477 vector<uint64_t> my_snaps(1);
478 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
479 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
480 my_snaps));
481 {
482 bufferlist bl;
483 bl.append("ciao!");
484 ObjectWriteOperation op;
485 op.write_full(bl);
486 ASSERT_EQ(0, ioctx.operate("foo", &op));
487 }
488 {
489 bufferlist bl;
490 bl.append("ciao!");
491 ObjectWriteOperation op;
492 op.write_full(bl);
493 ASSERT_EQ(0, ioctx.operate("bar", &op));
494 }
495 {
496 ObjectWriteOperation op;
497 op.remove();
498 ASSERT_EQ(0, ioctx.operate("baz", &op));
499 }
500 {
501 bufferlist bl;
502 bl.append("ciao!");
503 ObjectWriteOperation op;
504 op.write_full(bl);
505 ASSERT_EQ(0, ioctx.operate("bam", &op));
506 }
507
508 // configure cache
509 bufferlist inbl;
510 ASSERT_EQ(0, cluster.mon_command(
511 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
512 "\", \"tierpool\": \"" + cache_pool_name +
513 "\", \"force_nonempty\": \"--force-nonempty\" }",
514 inbl, NULL, NULL));
515 ASSERT_EQ(0, cluster.mon_command(
516 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
517 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
518 inbl, NULL, NULL));
519 ASSERT_EQ(0, cluster.mon_command(
520 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
521 "\", \"mode\": \"writeback\"}",
522 inbl, NULL, NULL));
523
524 // wait for maps to settle
525 cluster.wait_for_latest_osdmap();
526
527 // read, trigger a promote on the head
528 {
529 bufferlist bl;
530 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
531 ASSERT_EQ('c', bl[0]);
532 }
533 {
534 bufferlist bl;
535 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
536 ASSERT_EQ('c', bl[0]);
537 }
538
539 ioctx.snap_set_read(my_snaps[0]);
540
541 // read foo snap
542 {
543 bufferlist bl;
544 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
545 ASSERT_EQ('h', bl[0]);
546 }
547
548 // read bar snap
549 {
550 bufferlist bl;
551 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
552 ASSERT_EQ('h', bl[0]);
553 }
554
555 // read baz snap
556 {
557 bufferlist bl;
558 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
559 ASSERT_EQ('h', bl[0]);
560 }
561
562 ioctx.snap_set_read(librados::SNAP_HEAD);
563
564 // read foo
565 {
566 bufferlist bl;
567 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
568 ASSERT_EQ('c', bl[0]);
569 }
570
571 // read bar
572 {
573 bufferlist bl;
574 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
575 ASSERT_EQ('c', bl[0]);
576 }
577
578 // read baz
579 {
580 bufferlist bl;
581 ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
582 }
583
584 // cleanup
585 ioctx.selfmanaged_snap_remove(my_snaps[0]);
586}
587
588TEST_F(LibRadosTwoPoolsPP, PromoteSnapScrub) {
589 int num = 100;
590
591 // create objects
592 for (int i=0; i<num; ++i) {
593 bufferlist bl;
594 bl.append("hi there");
595 ObjectWriteOperation op;
596 op.write_full(bl);
597 ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
598 }
599
600 vector<uint64_t> my_snaps;
601 for (int snap=0; snap<4; ++snap) {
602 // create a snapshot, clone
603 vector<uint64_t> ns(1);
604 ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
605 my_snaps.swap(ns);
606 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
607 cout << "my_snaps " << my_snaps << std::endl;
608 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
609 my_snaps));
610 for (int i=0; i<num; ++i) {
611 bufferlist bl;
612 bl.append(string("ciao! snap") + stringify(snap));
613 ObjectWriteOperation op;
614 op.write_full(bl);
615 ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
616 }
617 }
618
619 // configure cache
620 bufferlist inbl;
621 ASSERT_EQ(0, cluster.mon_command(
622 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
623 "\", \"tierpool\": \"" + cache_pool_name +
624 "\", \"force_nonempty\": \"--force-nonempty\" }",
625 inbl, NULL, NULL));
626 ASSERT_EQ(0, cluster.mon_command(
627 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
628 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
629 inbl, NULL, NULL));
630 ASSERT_EQ(0, cluster.mon_command(
631 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
632 "\", \"mode\": \"writeback\"}",
633 inbl, NULL, NULL));
634
635 // wait for maps to settle
636 cluster.wait_for_latest_osdmap();
637
638 // read, trigger a promote on _some_ heads to make sure we handle cases
639 // where snaps are present and where they are not.
640 cout << "promoting some heads" << std::endl;
641 for (int i=0; i<num; ++i) {
642 if (i % 5 == 0 || i > num - 3) {
643 bufferlist bl;
644 ASSERT_EQ(1, ioctx.read(string("foo") + stringify(i), bl, 1, 0));
645 ASSERT_EQ('c', bl[0]);
646 }
647 }
648
649 for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
650 cout << "promoting from clones for snap " << my_snaps[snap] << std::endl;
651 ioctx.snap_set_read(my_snaps[snap]);
652
653 // read some snaps, semi-randomly
654 for (int i=0; i<50; ++i) {
655 bufferlist bl;
656 string o = string("foo") + stringify((snap * i * 137) % 80);
657 //cout << o << std::endl;
658 ASSERT_EQ(1, ioctx.read(o, bl, 1, 0));
659 }
660 }
661
662 // ok, stop and scrub this pool (to make sure scrub can handle
663 // missing clones in the cache tier).
664 {
665 IoCtx cache_ioctx;
666 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
667 for (int i=0; i<10; ++i) {
668 do {
669 ostringstream ss;
670 ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
671 << cache_ioctx.get_id() << "." << i
672 << "\"}";
673 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
674 if (r == -ENOENT || // in case mgr osdmap is stale
675 r == -EAGAIN) {
676 sleep(5);
677 continue;
678 }
679 } while (false);
680 }
681
682 // give it a few seconds to go. this is sloppy but is usually enough time
683 cout << "waiting for scrubs..." << std::endl;
684 sleep(30);
685 cout << "done waiting" << std::endl;
686 }
687
688 ioctx.snap_set_read(librados::SNAP_HEAD);
689
690 //cleanup
691 for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
692 ioctx.selfmanaged_snap_remove(my_snaps[snap]);
693 }
694}
695
696TEST_F(LibRadosTwoPoolsPP, PromoteSnapTrimRace) {
697 // create object
698 {
699 bufferlist bl;
700 bl.append("hi there");
701 ObjectWriteOperation op;
702 op.write_full(bl);
703 ASSERT_EQ(0, ioctx.operate("foo", &op));
704 }
705
706 // create a snapshot, clone
707 vector<uint64_t> my_snaps(1);
708 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
709 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
710 my_snaps));
711 {
712 bufferlist bl;
713 bl.append("ciao!");
714 ObjectWriteOperation op;
715 op.write_full(bl);
716 ASSERT_EQ(0, ioctx.operate("foo", &op));
717 }
718
719 // configure cache
720 bufferlist inbl;
721 ASSERT_EQ(0, cluster.mon_command(
722 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
723 "\", \"tierpool\": \"" + cache_pool_name +
724 "\", \"force_nonempty\": \"--force-nonempty\" }",
725 inbl, NULL, NULL));
726 ASSERT_EQ(0, cluster.mon_command(
727 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
728 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
729 inbl, NULL, NULL));
730 ASSERT_EQ(0, cluster.mon_command(
731 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
732 "\", \"mode\": \"writeback\"}",
733 inbl, NULL, NULL));
734
735 // wait for maps to settle
736 cluster.wait_for_latest_osdmap();
737
738 // delete the snap
739 ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
740
741 ioctx.snap_set_read(my_snaps[0]);
742
9f95a23c
TL		 743 // read foo snap. the OSD may or may not realize that this snap has
744 // been logically deleted; either response is valid.
7c673cae
FG		 745 {
746 bufferlist bl;
9f95a23c
TL		 747 int r = ioctx.read("foo", bl, 1, 0);
748 ASSERT_TRUE(r == 1 || r == -ENOENT);
7c673cae
FG		 749 }
750
751 // cleanup
752 ioctx.selfmanaged_snap_remove(my_snaps[0]);
753}
754
755TEST_F(LibRadosTwoPoolsPP, Whiteout) {
756 // create object
757 {
758 bufferlist bl;
759 bl.append("hi there");
760 ObjectWriteOperation op;
761 op.write_full(bl);
762 ASSERT_EQ(0, ioctx.operate("foo", &op));
763 }
764
765 // configure cache
766 bufferlist inbl;
767 ASSERT_EQ(0, cluster.mon_command(
768 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
769 "\", \"tierpool\": \"" + cache_pool_name +
770 "\", \"force_nonempty\": \"--force-nonempty\" }",
771 inbl, NULL, NULL));
772 ASSERT_EQ(0, cluster.mon_command(
773 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
774 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
775 inbl, NULL, NULL));
776 ASSERT_EQ(0, cluster.mon_command(
777 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
778 "\", \"mode\": \"writeback\"}",
779 inbl, NULL, NULL));
780
781 // wait for maps to settle
782 cluster.wait_for_latest_osdmap();
783
784 // create some whiteouts, verify they behave
785 {
786 ObjectWriteOperation op;
787 op.assert_exists();
788 op.remove();
789 ASSERT_EQ(0, ioctx.operate("foo", &op));
790 }
791
792 {
793 ObjectWriteOperation op;
794 op.assert_exists();
795 op.remove();
796 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
797 }
798 {
799 ObjectWriteOperation op;
800 op.assert_exists();
801 op.remove();
802 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
803 }
804
805 // verify the whiteouts are there in the cache tier
806 {
807 NObjectIterator it = cache_ioctx.nobjects_begin();
808 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
809 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
810 ++it;
811 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
812 ++it;
813 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
814 }
815
816 // delete a whiteout and verify it goes away
817 ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
818 {
819 ObjectWriteOperation op;
820 op.remove();
821 librados::AioCompletion *completion = cluster.aio_create_completion();
822 ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
823 librados::OPERATION_IGNORE_CACHE));
9f95a23c 824 completion->wait_for_complete();
7c673cae
FG		 825 ASSERT_EQ(0, completion->get_return_value());
826 completion->release();
827
828 NObjectIterator it = cache_ioctx.nobjects_begin();
829 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
830 ASSERT_TRUE(it->get_oid() == string("foo"));
831 ++it;
832 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
833 }
834
835 // recreate an object and verify we can read it
836 {
837 bufferlist bl;
838 bl.append("hi there");
839 ObjectWriteOperation op;
840 op.write_full(bl);
841 ASSERT_EQ(0, ioctx.operate("foo", &op));
842 }
843 {
844 bufferlist bl;
845 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
846 ASSERT_EQ('h', bl[0]);
847 }
848}
849
850TEST_F(LibRadosTwoPoolsPP, WhiteoutDeleteCreate) {
851 // configure cache
852 bufferlist inbl;
853 ASSERT_EQ(0, cluster.mon_command(
854 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
855 "\", \"tierpool\": \"" + cache_pool_name +
856 "\", \"force_nonempty\": \"--force-nonempty\" }",
857 inbl, NULL, NULL));
858 ASSERT_EQ(0, cluster.mon_command(
859 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
860 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
861 inbl, NULL, NULL));
862 ASSERT_EQ(0, cluster.mon_command(
863 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
864 "\", \"mode\": \"writeback\"}",
865 inbl, NULL, NULL));
866
867 // wait for maps to settle
868 cluster.wait_for_latest_osdmap();
869
870 // create an object
871 {
872 bufferlist bl;
873 bl.append("foo");
874 ASSERT_EQ(0, ioctx.write_full("foo", bl));
875 }
876
877 // do delete + create operation
878 {
879 ObjectWriteOperation op;
880 op.remove();
881 bufferlist bl;
882 bl.append("bar");
883 op.write_full(bl);
884 ASSERT_EQ(0, ioctx.operate("foo", &op));
885 }
886
887 // verify it still "exists" (w/ new content)
888 {
889 bufferlist bl;
890 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
891 ASSERT_EQ('b', bl[0]);
892 }
893}
894
895TEST_F(LibRadosTwoPoolsPP, Evict) {
896 // create object
897 {
898 bufferlist bl;
899 bl.append("hi there");
900 ObjectWriteOperation op;
901 op.write_full(bl);
902 ASSERT_EQ(0, ioctx.operate("foo", &op));
903 }
904
905 // configure cache
906 bufferlist inbl;
907 ASSERT_EQ(0, cluster.mon_command(
908 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
909 "\", \"tierpool\": \"" + cache_pool_name +
910 "\", \"force_nonempty\": \"--force-nonempty\" }",
911 inbl, NULL, NULL));
912 ASSERT_EQ(0, cluster.mon_command(
913 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
914 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
915 inbl, NULL, NULL));
916 ASSERT_EQ(0, cluster.mon_command(
917 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
918 "\", \"mode\": \"writeback\"}",
919 inbl, NULL, NULL));
920
921 // wait for maps to settle
922 cluster.wait_for_latest_osdmap();
923
924 // read, trigger a promote
925 {
926 bufferlist bl;
927 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
928 }
929
930 // read, trigger a whiteout, and a dirty object
931 {
932 bufferlist bl;
933 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
934 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
935 ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
936 }
937
938 // verify the object is present in the cache tier
939 {
940 NObjectIterator it = cache_ioctx.nobjects_begin();
941 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
942 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
943 ++it;
944 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
945 ++it;
946 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
947 }
948
949 // pin
950 {
951 ObjectWriteOperation op;
952 op.cache_pin();
953 librados::AioCompletion *completion = cluster.aio_create_completion();
954 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 955 completion->wait_for_complete();
7c673cae
FG		 956 ASSERT_EQ(0, completion->get_return_value());
957 completion->release();
958 }
959
960 // evict the pinned object with -EPERM
961 {
962 ObjectReadOperation op;
963 op.cache_evict();
964 librados::AioCompletion *completion = cluster.aio_create_completion();
965 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
966 librados::OPERATION_IGNORE_CACHE,
967 NULL));
9f95a23c 968 completion->wait_for_complete();
7c673cae
FG		 969 ASSERT_EQ(-EPERM, completion->get_return_value());
970 completion->release();
971 }
972
973 // unpin
974 {
975 ObjectWriteOperation op;
976 op.cache_unpin();
977 librados::AioCompletion *completion = cluster.aio_create_completion();
978 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 979 completion->wait_for_complete();
7c673cae
FG		 980 ASSERT_EQ(0, completion->get_return_value());
981 completion->release();
982 }
983
984 // flush
985 {
986 ObjectReadOperation op;
987 op.cache_flush();
988 librados::AioCompletion *completion = cluster.aio_create_completion();
989 ASSERT_EQ(0, cache_ioctx.aio_operate(
990 "foo", completion, &op,
991 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 992 completion->wait_for_complete();
7c673cae
FG		 993 ASSERT_EQ(0, completion->get_return_value());
994 completion->release();
995 }
996
997 // verify clean
998 {
999 bool dirty = false;
1000 int r = -1;
1001 ObjectReadOperation op;
1002 op.is_dirty(&dirty, &r);
1003 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1004 ASSERT_FALSE(dirty);
1005 ASSERT_EQ(0, r);
1006 }
1007
1008 // evict
1009 {
1010 ObjectReadOperation op;
1011 op.cache_evict();
1012 librados::AioCompletion *completion = cluster.aio_create_completion();
1013 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
1014 librados::OPERATION_IGNORE_CACHE,
1015 NULL));
9f95a23c 1016 completion->wait_for_complete();
7c673cae
FG		 1017 ASSERT_EQ(0, completion->get_return_value());
1018 completion->release();
1019 }
1020 {
1021 ObjectReadOperation op;
1022 op.cache_evict();
1023 librados::AioCompletion *completion = cluster.aio_create_completion();
1024 ASSERT_EQ(0, cache_ioctx.aio_operate(
1025 "foo", completion, &op,
1026 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1027 completion->wait_for_complete();
7c673cae
FG		 1028 ASSERT_EQ(0, completion->get_return_value());
1029 completion->release();
1030 }
1031 {
1032 ObjectReadOperation op;
1033 op.cache_evict();
1034 librados::AioCompletion *completion = cluster.aio_create_completion();
1035 ASSERT_EQ(0, cache_ioctx.aio_operate(
1036 "bar", completion, &op,
1037 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1038 completion->wait_for_complete();
7c673cae
FG		 1039 ASSERT_EQ(-EBUSY, completion->get_return_value());
1040 completion->release();
1041 }
1042}
1043
1044TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
1045 // create object
1046 {
1047 bufferlist bl;
1048 bl.append("hi there");
1049 ObjectWriteOperation op;
1050 op.write_full(bl);
1051 ASSERT_EQ(0, ioctx.operate("foo", &op));
1052 }
1053 {
1054 bufferlist bl;
1055 bl.append("hi there");
1056 ObjectWriteOperation op;
1057 op.write_full(bl);
1058 ASSERT_EQ(0, ioctx.operate("bar", &op));
1059 }
1060 {
1061 bufferlist bl;
1062 bl.append("hi there");
1063 ObjectWriteOperation op;
1064 op.write_full(bl);
1065 ASSERT_EQ(0, ioctx.operate("baz", &op));
1066 }
1067 {
1068 bufferlist bl;
1069 bl.append("hi there");
1070 ObjectWriteOperation op;
1071 op.write_full(bl);
1072 ASSERT_EQ(0, ioctx.operate("bam", &op));
1073 }
1074
1075 // create a snapshot, clone
1076 vector<uint64_t> my_snaps(1);
1077 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1078 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1079 my_snaps));
1080 {
1081 bufferlist bl;
1082 bl.append("ciao!");
1083 ObjectWriteOperation op;
1084 op.write_full(bl);
1085 ASSERT_EQ(0, ioctx.operate("foo", &op));
1086 }
1087 {
1088 bufferlist bl;
1089 bl.append("ciao!");
1090 ObjectWriteOperation op;
1091 op.write_full(bl);
1092 ASSERT_EQ(0, ioctx.operate("bar", &op));
1093 }
1094 {
1095 ObjectWriteOperation op;
1096 op.remove();
1097 ASSERT_EQ(0, ioctx.operate("baz", &op));
1098 }
1099 {
1100 bufferlist bl;
1101 bl.append("ciao!");
1102 ObjectWriteOperation op;
1103 op.write_full(bl);
1104 ASSERT_EQ(0, ioctx.operate("bam", &op));
1105 }
1106
1107 // configure cache
1108 bufferlist inbl;
1109 ASSERT_EQ(0, cluster.mon_command(
1110 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1111 "\", \"tierpool\": \"" + cache_pool_name +
1112 "\", \"force_nonempty\": \"--force-nonempty\" }",
1113 inbl, NULL, NULL));
1114 ASSERT_EQ(0, cluster.mon_command(
1115 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1116 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1117 inbl, NULL, NULL));
1118 ASSERT_EQ(0, cluster.mon_command(
1119 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1120 "\", \"mode\": \"writeback\"}",
1121 inbl, NULL, NULL));
1122
1123 // wait for maps to settle
1124 cluster.wait_for_latest_osdmap();
1125
1126 // read, trigger a promote on the head
1127 {
1128 bufferlist bl;
1129 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1130 ASSERT_EQ('c', bl[0]);
1131 }
1132 {
1133 bufferlist bl;
1134 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
1135 ASSERT_EQ('c', bl[0]);
1136 }
1137
1138 // evict bam
1139 {
1140 ObjectReadOperation op;
1141 op.cache_evict();
1142 librados::AioCompletion *completion = cluster.aio_create_completion();
1143 ASSERT_EQ(0, cache_ioctx.aio_operate(
1144 "bam", completion, &op,
1145 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1146 completion->wait_for_complete();
7c673cae
FG		 1147 ASSERT_EQ(0, completion->get_return_value());
1148 completion->release();
1149 }
1150 {
1151 bufferlist bl;
1152 ObjectReadOperation op;
1153 op.read(1, 0, &bl, NULL);
1154 librados::AioCompletion *completion = cluster.aio_create_completion();
1155 ASSERT_EQ(0, cache_ioctx.aio_operate(
1156 "bam", completion, &op,
1157 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1158 completion->wait_for_complete();
7c673cae
FG		 1159 ASSERT_EQ(-ENOENT, completion->get_return_value());
1160 completion->release();
1161 }
1162
1163 // read foo snap
1164 ioctx.snap_set_read(my_snaps[0]);
1165 {
1166 bufferlist bl;
1167 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1168 ASSERT_EQ('h', bl[0]);
1169 }
1170
1171 // evict foo snap
1172 {
1173 ObjectReadOperation op;
1174 op.cache_evict();
1175 librados::AioCompletion *completion = cluster.aio_create_completion();
1176 ASSERT_EQ(0, ioctx.aio_operate(
1177 "foo", completion, &op,
1178 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1179 completion->wait_for_complete();
7c673cae
FG		 1180 ASSERT_EQ(0, completion->get_return_value());
1181 completion->release();
1182 }
1183 // snap is gone...
1184 {
1185 bufferlist bl;
1186 ObjectReadOperation op;
1187 op.read(1, 0, &bl, NULL);
1188 librados::AioCompletion *completion = cluster.aio_create_completion();
1189 ASSERT_EQ(0, ioctx.aio_operate(
1190 "foo", completion, &op,
1191 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1192 completion->wait_for_complete();
7c673cae
FG		 1193 ASSERT_EQ(-ENOENT, completion->get_return_value());
1194 completion->release();
1195 }
1196 // head is still there...
1197 ioctx.snap_set_read(librados::SNAP_HEAD);
1198 {
1199 bufferlist bl;
1200 ObjectReadOperation op;
1201 op.read(1, 0, &bl, NULL);
1202 librados::AioCompletion *completion = cluster.aio_create_completion();
1203 ASSERT_EQ(0, ioctx.aio_operate(
1204 "foo", completion, &op,
1205 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1206 completion->wait_for_complete();
7c673cae
FG		 1207 ASSERT_EQ(0, completion->get_return_value());
1208 completion->release();
1209 }
1210
1211 // promote head + snap of bar
1212 ioctx.snap_set_read(librados::SNAP_HEAD);
1213 {
1214 bufferlist bl;
1215 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
1216 ASSERT_EQ('c', bl[0]);
1217 }
1218 ioctx.snap_set_read(my_snaps[0]);
1219 {
1220 bufferlist bl;
1221 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
1222 ASSERT_EQ('h', bl[0]);
1223 }
1224
1225 // evict bar head (fail)
1226 ioctx.snap_set_read(librados::SNAP_HEAD);
1227 {
1228 ObjectReadOperation op;
1229 op.cache_evict();
1230 librados::AioCompletion *completion = cluster.aio_create_completion();
1231 ASSERT_EQ(0, ioctx.aio_operate(
1232 "bar", completion, &op,
1233 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1234 completion->wait_for_complete();
7c673cae
FG		 1235 ASSERT_EQ(-EBUSY, completion->get_return_value());
1236 completion->release();
1237 }
1238
1239 // evict bar snap
1240 ioctx.snap_set_read(my_snaps[0]);
1241 {
1242 ObjectReadOperation op;
1243 op.cache_evict();
1244 librados::AioCompletion *completion = cluster.aio_create_completion();
1245 ASSERT_EQ(0, ioctx.aio_operate(
1246 "bar", completion, &op,
1247 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1248 completion->wait_for_complete();
7c673cae
FG		 1249 ASSERT_EQ(0, completion->get_return_value());
1250 completion->release();
1251 }
1252 // ...and then head
1253 ioctx.snap_set_read(librados::SNAP_HEAD);
1254 {
1255 bufferlist bl;
1256 ObjectReadOperation op;
1257 op.read(1, 0, &bl, NULL);
1258 librados::AioCompletion *completion = cluster.aio_create_completion();
1259 ASSERT_EQ(0, ioctx.aio_operate(
1260 "bar", completion, &op,
1261 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1262 completion->wait_for_complete();
7c673cae
FG		 1263 ASSERT_EQ(0, completion->get_return_value());
1264 completion->release();
1265 }
1266 {
1267 ObjectReadOperation op;
1268 op.cache_evict();
1269 librados::AioCompletion *completion = cluster.aio_create_completion();
1270 ASSERT_EQ(0, ioctx.aio_operate(
1271 "bar", completion, &op,
1272 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1273 completion->wait_for_complete();
7c673cae
FG		 1274 ASSERT_EQ(0, completion->get_return_value());
1275 completion->release();
1276 }
1277
1278 // cleanup
1279 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1280}
1281
1282// this test case reproduces http://tracker.ceph.com/issues/8629
1283TEST_F(LibRadosTwoPoolsPP, EvictSnap2) {
1284 // create object
1285 {
1286 bufferlist bl;
1287 bl.append("hi there");
1288 ObjectWriteOperation op;
1289 op.write_full(bl);
1290 ASSERT_EQ(0, ioctx.operate("foo", &op));
1291 }
1292 // create a snapshot, clone
1293 vector<uint64_t> my_snaps(1);
1294 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1295 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1296 my_snaps));
1297 {
1298 bufferlist bl;
1299 bl.append("ciao!");
1300 ObjectWriteOperation op;
1301 op.write_full(bl);
1302 ASSERT_EQ(0, ioctx.operate("foo", &op));
1303 }
1304 // configure cache
1305 bufferlist inbl;
1306 ASSERT_EQ(0, cluster.mon_command(
1307 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1308 "\", \"tierpool\": \"" + cache_pool_name +
1309 "\", \"force_nonempty\": \"--force-nonempty\" }",
1310 inbl, NULL, NULL));
1311 ASSERT_EQ(0, cluster.mon_command(
1312 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1313 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1314 inbl, NULL, NULL));
1315 ASSERT_EQ(0, cluster.mon_command(
1316 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1317 "\", \"mode\": \"writeback\"}",
1318 inbl, NULL, NULL));
1319
1320 // wait for maps to settle
1321 cluster.wait_for_latest_osdmap();
1322
1323 // read, trigger a promote on the head
1324 {
1325 bufferlist bl;
1326 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1327 ASSERT_EQ('c', bl[0]);
1328 }
1329
1330 // evict
1331 {
1332 ObjectReadOperation op;
1333 op.cache_evict();
1334 librados::AioCompletion *completion = cluster.aio_create_completion();
1335 ASSERT_EQ(0, cache_ioctx.aio_operate(
1336 "foo", completion, &op,
1337 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1338 completion->wait_for_complete();
7c673cae
FG		 1339 ASSERT_EQ(0, completion->get_return_value());
1340 completion->release();
1341 }
1342
1343 // verify the snapdir is not present in the cache pool
1344 {
1345 ObjectReadOperation op;
1346 librados::snap_set_t snapset;
1347 op.list_snaps(&snapset, NULL);
1348 ioctx.snap_set_read(librados::SNAP_DIR);
1349 librados::AioCompletion *completion = cluster.aio_create_completion();
1350 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op,
1351 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1352 completion->wait_for_complete();
7c673cae
FG		 1353 ASSERT_EQ(-ENOENT, completion->get_return_value());
1354 completion->release();
1355 }
1356}
1357
11fdf7f2
TL		 1358//This test case reproduces http://tracker.ceph.com/issues/17445
1359TEST_F(LibRadosTwoPoolsPP, ListSnap){
1360 // Create object
1361 {
1362 bufferlist bl;
1363 bl.append("hi there");
1364 ObjectWriteOperation op;
1365 op.write_full(bl);
1366 ASSERT_EQ(0, ioctx.operate("foo", &op));
1367 }
1368 {
1369 bufferlist bl;
1370 bl.append("hi there");
1371 ObjectWriteOperation op;
1372 op.write_full(bl);
1373 ASSERT_EQ(0, ioctx.operate("bar", &op));
1374 }
1375 {
1376 bufferlist bl;
1377 bl.append("hi there");
1378 ObjectWriteOperation op;
1379 op.write_full(bl);
1380 ASSERT_EQ(0, ioctx.operate("baz", &op));
1381 }
1382 {
1383 bufferlist bl;
1384 bl.append("hi there");
1385 ObjectWriteOperation op;
1386 op.write_full(bl);
1387 ASSERT_EQ(0, ioctx.operate("bam", &op));
1388 }
1389
1390 // Create a snapshot, clone
1391 vector<uint64_t> my_snaps(1);
1392 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1393 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1394 my_snaps));
1395 {
1396 bufferlist bl;
1397 bl.append("ciao!");
1398 ObjectWriteOperation op;
1399 op.write_full(bl);
1400 ASSERT_EQ(0, ioctx.operate("foo", &op));
1401 }
1402 {
1403 bufferlist bl;
1404 bl.append("ciao!");
1405 ObjectWriteOperation op;
1406 op.write_full(bl);
1407 ASSERT_EQ(0, ioctx.operate("bar", &op));
1408 }
1409 {
1410 ObjectWriteOperation op;
1411 op.remove();
1412 ASSERT_EQ(0, ioctx.operate("baz", &op));
1413 }
1414 {
1415 bufferlist bl;
1416 bl.append("ciao!");
1417 ObjectWriteOperation op;
1418 op.write_full(bl);
1419 ASSERT_EQ(0, ioctx.operate("bam", &op));
1420 }
1421
1422 // Configure cache
1423 bufferlist inbl;
1424 ASSERT_EQ(0, cluster.mon_command(
1425 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1426 "\", \"tierpool\": \"" + cache_pool_name +
1427 "\", \"force_nonempty\": \"--force-nonempty\" }",
1428 inbl, NULL, NULL));
1429 ASSERT_EQ(0, cluster.mon_command(
1430 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1431 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1432 inbl, NULL, NULL));
1433 ASSERT_EQ(0, cluster.mon_command(
1434 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1435 "\", \"mode\": \"writeback\"}",
1436 inbl, NULL, NULL));
1437
1438 // Wait for maps to settle
1439 cluster.wait_for_latest_osdmap();
1440
1441 // Read, trigger a promote on the head
1442 {
1443 bufferlist bl;
1444 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1445 ASSERT_EQ('c', bl[0]);
1446 }
1447
1448 // Read foo snap
1449 ioctx.snap_set_read(my_snaps[0]);
1450 {
1451 bufferlist bl;
1452 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1453 ASSERT_EQ('h', bl[0]);
1454 }
1455
1456 // Evict foo snap
1457 {
1458 ObjectReadOperation op;
1459 op.cache_evict();
1460 librados::AioCompletion *completion = cluster.aio_create_completion();
1461 ASSERT_EQ(0, ioctx.aio_operate(
1462 "foo", completion, &op,
1463 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1464 completion->wait_for_complete();
11fdf7f2
TL		 1465 ASSERT_EQ(0, completion->get_return_value());
1466 completion->release();
1467 }
1468 // Snap is gone...
1469 {
1470 bufferlist bl;
1471 ObjectReadOperation op;
1472 op.read(1, 0, &bl, NULL);
1473 librados::AioCompletion *completion = cluster.aio_create_completion();
1474 ASSERT_EQ(0, ioctx.aio_operate(
1475 "foo", completion, &op,
1476 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1477 completion->wait_for_complete();
11fdf7f2
TL		 1478 ASSERT_EQ(-ENOENT, completion->get_return_value());
1479 completion->release();
1480 }
1481
1482 // Do list-snaps
1483 ioctx.snap_set_read(CEPH_SNAPDIR);
1484 {
1485 snap_set_t snap_set;
1486 int snap_ret;
1487 ObjectReadOperation op;
1488 op.list_snaps(&snap_set, &snap_ret);
1489 librados::AioCompletion *completion = cluster.aio_create_completion();
1490 ASSERT_EQ(0, ioctx.aio_operate(
1491 "foo", completion, &op,
1492 0, NULL));
9f95a23c 1493 completion->wait_for_complete();
11fdf7f2
TL		 1494 ASSERT_EQ(0, snap_ret);
1495 ASSERT_LT(0u, snap_set.clones.size());
1496 for (vector<librados::clone_info_t>::const_iterator r = snap_set.clones.begin();
1497 r != snap_set.clones.end();
1498 ++r) {
1499 if (r->cloneid != librados::SNAP_HEAD) {
1500 ASSERT_LT(0u, r->snaps.size());
1501 }
1502 }
1503 }
1504
1505 // Cleanup
1506 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1507}
1508
7c673cae
FG		 1509TEST_F(LibRadosTwoPoolsPP, TryFlush) {
1510 // configure cache
1511 bufferlist inbl;
1512 ASSERT_EQ(0, cluster.mon_command(
1513 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1514 "\", \"tierpool\": \"" + cache_pool_name +
1515 "\", \"force_nonempty\": \"--force-nonempty\" }",
1516 inbl, NULL, NULL));
1517 ASSERT_EQ(0, cluster.mon_command(
1518 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1519 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1520 inbl, NULL, NULL));
1521 ASSERT_EQ(0, cluster.mon_command(
1522 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1523 "\", \"mode\": \"writeback\"}",
1524 inbl, NULL, NULL));
1525
1526 // wait for maps to settle
1527 cluster.wait_for_latest_osdmap();
1528
1529 // create object
1530 {
1531 bufferlist bl;
1532 bl.append("hi there");
1533 ObjectWriteOperation op;
1534 op.write_full(bl);
1535 ASSERT_EQ(0, ioctx.operate("foo", &op));
1536 }
1537
1538 // verify the object is present in the cache tier
1539 {
1540 NObjectIterator it = cache_ioctx.nobjects_begin();
1541 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1542 ASSERT_TRUE(it->get_oid() == string("foo"));
1543 ++it;
1544 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1545 }
1546
1547 // verify the object is NOT present in the base tier
1548 {
1549 NObjectIterator it = ioctx.nobjects_begin();
1550 ASSERT_TRUE(it == ioctx.nobjects_end());
1551 }
1552
1553 // verify dirty
1554 {
1555 bool dirty = false;
1556 int r = -1;
1557 ObjectReadOperation op;
1558 op.is_dirty(&dirty, &r);
1559 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1560 ASSERT_TRUE(dirty);
1561 ASSERT_EQ(0, r);
1562 }
1563
1564 // pin
1565 {
1566 ObjectWriteOperation op;
1567 op.cache_pin();
1568 librados::AioCompletion *completion = cluster.aio_create_completion();
1569 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1570 completion->wait_for_complete();
7c673cae
FG		 1571 ASSERT_EQ(0, completion->get_return_value());
1572 completion->release();
1573 }
1574
1575 // flush the pinned object with -EPERM
1576 {
1577 ObjectReadOperation op;
1578 op.cache_try_flush();
1579 librados::AioCompletion *completion = cluster.aio_create_completion();
1580 ASSERT_EQ(0, cache_ioctx.aio_operate(
1581 "foo", completion, &op,
1582 librados::OPERATION_IGNORE_OVERLAY |
1583 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1584 completion->wait_for_complete();
7c673cae
FG		 1585 ASSERT_EQ(-EPERM, completion->get_return_value());
1586 completion->release();
1587 }
1588
1589 // unpin
1590 {
1591 ObjectWriteOperation op;
1592 op.cache_unpin();
1593 librados::AioCompletion *completion = cluster.aio_create_completion();
1594 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1595 completion->wait_for_complete();
7c673cae
FG		 1596 ASSERT_EQ(0, completion->get_return_value());
1597 completion->release();
1598 }
1599
1600 // flush
1601 {
1602 ObjectReadOperation op;
1603 op.cache_try_flush();
1604 librados::AioCompletion *completion = cluster.aio_create_completion();
1605 ASSERT_EQ(0, cache_ioctx.aio_operate(
1606 "foo", completion, &op,
1607 librados::OPERATION_IGNORE_OVERLAY |
1608 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1609 completion->wait_for_complete();
7c673cae
FG		 1610 ASSERT_EQ(0, completion->get_return_value());
1611 completion->release();
1612 }
1613
1614 // verify clean
1615 {
1616 bool dirty = false;
1617 int r = -1;
1618 ObjectReadOperation op;
1619 op.is_dirty(&dirty, &r);
1620 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1621 ASSERT_FALSE(dirty);
1622 ASSERT_EQ(0, r);
1623 }
1624
1625 // verify in base tier
1626 {
1627 NObjectIterator it = ioctx.nobjects_begin();
1628 ASSERT_TRUE(it != ioctx.nobjects_end());
1629 ASSERT_TRUE(it->get_oid() == string("foo"));
1630 ++it;
1631 ASSERT_TRUE(it == ioctx.nobjects_end());
1632 }
1633
1634 // evict it
1635 {
1636 ObjectReadOperation op;
1637 op.cache_evict();
1638 librados::AioCompletion *completion = cluster.aio_create_completion();
1639 ASSERT_EQ(0, cache_ioctx.aio_operate(
1640 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1641 completion->wait_for_complete();
7c673cae
FG		 1642 ASSERT_EQ(0, completion->get_return_value());
1643 completion->release();
1644 }
1645
1646 // verify no longer in cache tier
1647 {
1648 NObjectIterator it = cache_ioctx.nobjects_begin();
1649 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1650 }
1651}
1652
1653TEST_F(LibRadosTwoPoolsPP, Flush) {
1654 // configure cache
1655 bufferlist inbl;
1656 ASSERT_EQ(0, cluster.mon_command(
1657 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1658 "\", \"tierpool\": \"" + cache_pool_name +
1659 "\", \"force_nonempty\": \"--force-nonempty\" }",
1660 inbl, NULL, NULL));
1661 ASSERT_EQ(0, cluster.mon_command(
1662 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1663 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1664 inbl, NULL, NULL));
1665 ASSERT_EQ(0, cluster.mon_command(
1666 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1667 "\", \"mode\": \"writeback\"}",
1668 inbl, NULL, NULL));
1669
1670 // wait for maps to settle
1671 cluster.wait_for_latest_osdmap();
1672
1673 uint64_t user_version = 0;
1674
1675 // create object
1676 {
1677 bufferlist bl;
1678 bl.append("hi there");
1679 ObjectWriteOperation op;
1680 op.write_full(bl);
1681 ASSERT_EQ(0, ioctx.operate("foo", &op));
1682 }
1683
1684 // verify the object is present in the cache tier
1685 {
1686 NObjectIterator it = cache_ioctx.nobjects_begin();
1687 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1688 ASSERT_TRUE(it->get_oid() == string("foo"));
1689 ++it;
1690 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1691 }
1692
1693 // verify the object is NOT present in the base tier
1694 {
1695 NObjectIterator it = ioctx.nobjects_begin();
1696 ASSERT_TRUE(it == ioctx.nobjects_end());
1697 }
1698
1699 // verify dirty
1700 {
1701 bool dirty = false;
1702 int r = -1;
1703 ObjectReadOperation op;
1704 op.is_dirty(&dirty, &r);
1705 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1706 ASSERT_TRUE(dirty);
1707 ASSERT_EQ(0, r);
1708 user_version = cache_ioctx.get_last_version();
1709 }
1710
1711 // pin
1712 {
1713 ObjectWriteOperation op;
1714 op.cache_pin();
1715 librados::AioCompletion *completion = cluster.aio_create_completion();
1716 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1717 completion->wait_for_complete();
7c673cae
FG		 1718 ASSERT_EQ(0, completion->get_return_value());
1719 completion->release();
1720 }
1721
1722 // flush the pinned object with -EPERM
1723 {
1724 ObjectReadOperation op;
1725 op.cache_try_flush();
1726 librados::AioCompletion *completion = cluster.aio_create_completion();
1727 ASSERT_EQ(0, cache_ioctx.aio_operate(
1728 "foo", completion, &op,
1729 librados::OPERATION_IGNORE_OVERLAY |
1730 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1731 completion->wait_for_complete();
7c673cae
FG		 1732 ASSERT_EQ(-EPERM, completion->get_return_value());
1733 completion->release();
1734 }
1735
1736 // unpin
1737 {
1738 ObjectWriteOperation op;
1739 op.cache_unpin();
1740 librados::AioCompletion *completion = cluster.aio_create_completion();
1741 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1742 completion->wait_for_complete();
7c673cae
FG		 1743 ASSERT_EQ(0, completion->get_return_value());
1744 completion->release();
1745 }
1746
1747 // flush
1748 {
1749 ObjectReadOperation op;
1750 op.cache_flush();
1751 librados::AioCompletion *completion = cluster.aio_create_completion();
1752 ASSERT_EQ(0, cache_ioctx.aio_operate(
1753 "foo", completion, &op,
1754 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 1755 completion->wait_for_complete();
7c673cae
FG		 1756 ASSERT_EQ(0, completion->get_return_value());
1757 completion->release();
1758 }
1759
1760 // verify clean
1761 {
1762 bool dirty = false;
1763 int r = -1;
1764 ObjectReadOperation op;
1765 op.is_dirty(&dirty, &r);
1766 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1767 ASSERT_FALSE(dirty);
1768 ASSERT_EQ(0, r);
1769 }
1770
1771 // verify in base tier
1772 {
1773 NObjectIterator it = ioctx.nobjects_begin();
1774 ASSERT_TRUE(it != ioctx.nobjects_end());
1775 ASSERT_TRUE(it->get_oid() == string("foo"));
1776 ++it;
1777 ASSERT_TRUE(it == ioctx.nobjects_end());
1778 }
1779
1780 // evict it
1781 {
1782 ObjectReadOperation op;
1783 op.cache_evict();
1784 librados::AioCompletion *completion = cluster.aio_create_completion();
1785 ASSERT_EQ(0, cache_ioctx.aio_operate(
1786 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1787 completion->wait_for_complete();
7c673cae
FG		 1788 ASSERT_EQ(0, completion->get_return_value());
1789 completion->release();
1790 }
1791
1792 // verify no longer in cache tier
1793 {
1794 NObjectIterator it = cache_ioctx.nobjects_begin();
1795 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1796 }
1797
1798 // read it again and verify the version is consistent
1799 {
1800 bufferlist bl;
1801 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
1802 ASSERT_EQ(user_version, cache_ioctx.get_last_version());
1803 }
1804
1805 // erase it
1806 {
1807 ObjectWriteOperation op;
1808 op.remove();
1809 ASSERT_EQ(0, ioctx.operate("foo", &op));
1810 }
1811
1812 // flush whiteout
1813 {
1814 ObjectReadOperation op;
1815 op.cache_flush();
1816 librados::AioCompletion *completion = cluster.aio_create_completion();
1817 ASSERT_EQ(0, cache_ioctx.aio_operate(
1818 "foo", completion, &op,
1819 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 1820 completion->wait_for_complete();
7c673cae
FG		 1821 ASSERT_EQ(0, completion->get_return_value());
1822 completion->release();
1823 }
1824
1825 // evict
1826 {
1827 ObjectReadOperation op;
1828 op.cache_evict();
1829 librados::AioCompletion *completion = cluster.aio_create_completion();
1830 ASSERT_EQ(0, cache_ioctx.aio_operate(
1831 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1832 completion->wait_for_complete();
7c673cae
FG		 1833 ASSERT_EQ(0, completion->get_return_value());
1834 completion->release();
1835 }
1836
1837 // verify no longer in cache tier
1838 {
1839 NObjectIterator it = cache_ioctx.nobjects_begin();
1840 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1841 }
1842 // or base tier
1843 {
1844 NObjectIterator it = ioctx.nobjects_begin();
1845 ASSERT_TRUE(it == ioctx.nobjects_end());
1846 }
1847}
1848
1849TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
1850 // configure cache
1851 bufferlist inbl;
1852 ASSERT_EQ(0, cluster.mon_command(
1853 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1854 "\", \"tierpool\": \"" + cache_pool_name +
1855 "\", \"force_nonempty\": \"--force-nonempty\" }",
1856 inbl, NULL, NULL));
1857 ASSERT_EQ(0, cluster.mon_command(
1858 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1859 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1860 inbl, NULL, NULL));
1861 ASSERT_EQ(0, cluster.mon_command(
1862 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1863 "\", \"mode\": \"writeback\"}",
1864 inbl, NULL, NULL));
1865
1866 // wait for maps to settle
1867 cluster.wait_for_latest_osdmap();
1868
1869 // create object
1870 {
1871 bufferlist bl;
1872 bl.append("a");
1873 ObjectWriteOperation op;
1874 op.write_full(bl);
1875 ASSERT_EQ(0, ioctx.operate("foo", &op));
1876 }
1877
1878 // create a snapshot, clone
1879 vector<uint64_t> my_snaps(1);
1880 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1881 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1882 my_snaps));
1883 {
1884 bufferlist bl;
1885 bl.append("b");
1886 ObjectWriteOperation op;
1887 op.write_full(bl);
1888 ASSERT_EQ(0, ioctx.operate("foo", &op));
1889 }
1890
1891 // and another
1892 my_snaps.resize(2);
1893 my_snaps[1] = my_snaps[0];
1894 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1895 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1896 my_snaps));
1897 {
1898 bufferlist bl;
1899 bl.append("c");
1900 ObjectWriteOperation op;
1901 op.write_full(bl);
1902 ASSERT_EQ(0, ioctx.operate("foo", &op));
1903 }
1904
1905 // verify the object is present in the cache tier
1906 {
1907 NObjectIterator it = cache_ioctx.nobjects_begin();
1908 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1909 ASSERT_TRUE(it->get_oid() == string("foo"));
1910 ++it;
1911 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1912 }
1913
1914 // verify the object is NOT present in the base tier
1915 {
1916 NObjectIterator it = ioctx.nobjects_begin();
1917 ASSERT_TRUE(it == ioctx.nobjects_end());
1918 }
1919
1920 // flush on head (should fail)
1921 ioctx.snap_set_read(librados::SNAP_HEAD);
1922 {
1923 ObjectReadOperation op;
1924 op.cache_flush();
1925 librados::AioCompletion *completion = cluster.aio_create_completion();
1926 ASSERT_EQ(0, ioctx.aio_operate(
1927 "foo", completion, &op,
1928 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1929 completion->wait_for_complete();
7c673cae
FG		 1930 ASSERT_EQ(-EBUSY, completion->get_return_value());
1931 completion->release();
1932 }
1933 // flush on recent snap (should fail)
1934 ioctx.snap_set_read(my_snaps[0]);
1935 {
1936 ObjectReadOperation op;
1937 op.cache_flush();
1938 librados::AioCompletion *completion = cluster.aio_create_completion();
1939 ASSERT_EQ(0, ioctx.aio_operate(
1940 "foo", completion, &op,
1941 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1942 completion->wait_for_complete();
7c673cae
FG		 1943 ASSERT_EQ(-EBUSY, completion->get_return_value());
1944 completion->release();
1945 }
1946 // flush on oldest snap
1947 ioctx.snap_set_read(my_snaps[1]);
1948 {
1949 ObjectReadOperation op;
1950 op.cache_flush();
1951 librados::AioCompletion *completion = cluster.aio_create_completion();
1952 ASSERT_EQ(0, ioctx.aio_operate(
1953 "foo", completion, &op,
1954 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1955 completion->wait_for_complete();
7c673cae
FG		 1956 ASSERT_EQ(0, completion->get_return_value());
1957 completion->release();
1958 }
1959 // flush on next oldest snap
1960 ioctx.snap_set_read(my_snaps[0]);
1961 {
1962 ObjectReadOperation op;
1963 op.cache_flush();
1964 librados::AioCompletion *completion = cluster.aio_create_completion();
1965 ASSERT_EQ(0, ioctx.aio_operate(
1966 "foo", completion, &op,
1967 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1968 completion->wait_for_complete();
7c673cae
FG		 1969 ASSERT_EQ(0, completion->get_return_value());
1970 completion->release();
1971 }
1972 // flush on head
1973 ioctx.snap_set_read(librados::SNAP_HEAD);
1974 {
1975 ObjectReadOperation op;
1976 op.cache_flush();
1977 librados::AioCompletion *completion = cluster.aio_create_completion();
1978 ASSERT_EQ(0, ioctx.aio_operate(
1979 "foo", completion, &op,
1980 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1981 completion->wait_for_complete();
7c673cae
FG		 1982 ASSERT_EQ(0, completion->get_return_value());
1983 completion->release();
1984 }
1985
1986 // verify i can read the snaps from the cache pool
1987 ioctx.snap_set_read(librados::SNAP_HEAD);
1988 {
1989 bufferlist bl;
1990 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1991 ASSERT_EQ('c', bl[0]);
1992 }
1993 ioctx.snap_set_read(my_snaps[0]);
1994 {
1995 bufferlist bl;
1996 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1997 ASSERT_EQ('b', bl[0]);
1998 }
1999 ioctx.snap_set_read(my_snaps[1]);
2000 {
2001 bufferlist bl;
2002 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2003 ASSERT_EQ('a', bl[0]);
2004 }
2005
2006 // remove overlay
2007 ASSERT_EQ(0, cluster.mon_command(
2008 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2009 "\"}",
2010 inbl, NULL, NULL));
2011
2012 // wait for maps to settle
2013 cluster.wait_for_latest_osdmap();
2014
2015 // verify i can read the snaps from the base pool
2016 ioctx.snap_set_read(librados::SNAP_HEAD);
2017 {
2018 bufferlist bl;
2019 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2020 ASSERT_EQ('c', bl[0]);
2021 }
2022 ioctx.snap_set_read(my_snaps[0]);
2023 {
2024 bufferlist bl;
2025 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2026 ASSERT_EQ('b', bl[0]);
2027 }
2028 ioctx.snap_set_read(my_snaps[1]);
2029 {
2030 bufferlist bl;
2031 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2032 ASSERT_EQ('a', bl[0]);
2033 }
2034
2035 ASSERT_EQ(0, cluster.mon_command(
2036 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2037 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2038 inbl, NULL, NULL));
2039
2040 // cleanup
2041 ioctx.selfmanaged_snap_remove(my_snaps[0]);
2042}
2043
2044TEST_F(LibRadosTierPP, FlushWriteRaces) {
2045 Rados cluster;
2046 std::string pool_name = get_temp_pool_name();
2047 std::string cache_pool_name = pool_name + "-cache";
2048 ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
2049 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
2050 IoCtx cache_ioctx;
2051 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 2052 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 2053 IoCtx ioctx;
2054 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
2055
2056 // configure cache
2057 bufferlist inbl;
2058 ASSERT_EQ(0, cluster.mon_command(
2059 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2060 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2061 inbl, NULL, NULL));
2062 ASSERT_EQ(0, cluster.mon_command(
2063 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2064 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2065 inbl, NULL, NULL));
2066 ASSERT_EQ(0, cluster.mon_command(
2067 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2068 "\", \"mode\": \"writeback\"}",
2069 inbl, NULL, NULL));
2070
2071 // wait for maps to settle
2072 cluster.wait_for_latest_osdmap();
2073
2074 // create/dirty object
2075 bufferlist bl;
2076 bl.append("hi there");
2077 {
2078 ObjectWriteOperation op;
2079 op.write_full(bl);
2080 ASSERT_EQ(0, ioctx.operate("foo", &op));
2081 }
2082
2083 // flush + write
2084 {
2085 ObjectReadOperation op;
2086 op.cache_flush();
2087 librados::AioCompletion *completion = cluster.aio_create_completion();
2088 ASSERT_EQ(0, cache_ioctx.aio_operate(
2089 "foo", completion, &op,
2090 librados::OPERATION_IGNORE_OVERLAY, NULL));
2091
2092 ObjectWriteOperation op2;
2093 op2.write_full(bl);
2094 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2095 ASSERT_EQ(0, ioctx.aio_operate(
2096 "foo", completion2, &op2, 0));
2097
9f95a23c
TL		 2098 completion->wait_for_complete();
2099 completion2->wait_for_complete();
7c673cae
FG		 2100 ASSERT_EQ(0, completion->get_return_value());
2101 ASSERT_EQ(0, completion2->get_return_value());
2102 completion->release();
2103 completion2->release();
2104 }
2105
2106 int tries = 1000;
2107 do {
2108 // create/dirty object
2109 {
2110 bufferlist bl;
2111 bl.append("hi there");
2112 ObjectWriteOperation op;
2113 op.write_full(bl);
2114 ASSERT_EQ(0, ioctx.operate("foo", &op));
2115 }
2116
2117 // try-flush + write
2118 {
2119 ObjectReadOperation op;
2120 op.cache_try_flush();
2121 librados::AioCompletion *completion = cluster.aio_create_completion();
2122 ASSERT_EQ(0, cache_ioctx.aio_operate(
2123 "foo", completion, &op,
2124 librados::OPERATION_IGNORE_OVERLAY |
2125 librados::OPERATION_SKIPRWLOCKS, NULL));
2126
2127 ObjectWriteOperation op2;
2128 op2.write_full(bl);
2129 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2130 ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
2131
9f95a23c
TL		 2132 completion->wait_for_complete();
2133 completion2->wait_for_complete();
7c673cae
FG		 2134 int r = completion->get_return_value();
2135 ASSERT_TRUE(r == -EBUSY || r == 0);
2136 ASSERT_EQ(0, completion2->get_return_value());
2137 completion->release();
2138 completion2->release();
2139 if (r == -EBUSY)
2140 break;
2141 cout << "didn't get EBUSY, trying again" << std::endl;
2142 }
2143 ASSERT_TRUE(--tries);
2144 } while (true);
2145
2146 // tear down tiers
2147 ASSERT_EQ(0, cluster.mon_command(
2148 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2149 "\"}",
2150 inbl, NULL, NULL));
2151 ASSERT_EQ(0, cluster.mon_command(
2152 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2153 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2154 inbl, NULL, NULL));
2155
2156 // wait for maps to settle before next test
2157 cluster.wait_for_latest_osdmap();
2158
2159 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
2160 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
2161}
2162
2163TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
2164 // configure cache
2165 bufferlist inbl;
2166 ASSERT_EQ(0, cluster.mon_command(
2167 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2168 "\", \"tierpool\": \"" + cache_pool_name +
2169 "\", \"force_nonempty\": \"--force-nonempty\" }",
2170 inbl, NULL, NULL));
2171 ASSERT_EQ(0, cluster.mon_command(
2172 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2173 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2174 inbl, NULL, NULL));
2175 ASSERT_EQ(0, cluster.mon_command(
2176 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2177 "\", \"mode\": \"writeback\"}",
2178 inbl, NULL, NULL));
2179
2180 // wait for maps to settle
2181 cluster.wait_for_latest_osdmap();
2182
2183 // create/dirty object
2184 {
2185 bufferlist bl;
2186 bl.append("hi there");
2187 ObjectWriteOperation op;
2188 op.write_full(bl);
2189 ASSERT_EQ(0, ioctx.operate("foo", &op));
2190 }
2191
2192 // flush + flush
2193 {
2194 ObjectReadOperation op;
2195 op.cache_flush();
2196 librados::AioCompletion *completion = cluster.aio_create_completion();
2197 ASSERT_EQ(0, cache_ioctx.aio_operate(
2198 "foo", completion, &op,
2199 librados::OPERATION_IGNORE_OVERLAY, NULL));
2200
2201 ObjectReadOperation op2;
2202 op2.cache_flush();
2203 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2204 ASSERT_EQ(0, cache_ioctx.aio_operate(
2205 "foo", completion2, &op2,
2206 librados::OPERATION_IGNORE_OVERLAY, NULL));
2207
9f95a23c
TL		 2208 completion->wait_for_complete();
2209 completion2->wait_for_complete();
7c673cae
FG		 2210 ASSERT_EQ(0, completion->get_return_value());
2211 ASSERT_EQ(0, completion2->get_return_value());
2212 completion->release();
2213 completion2->release();
2214 }
2215
2216 // create/dirty object
2217 {
2218 bufferlist bl;
2219 bl.append("hi there");
2220 ObjectWriteOperation op;
2221 op.write_full(bl);
2222 ASSERT_EQ(0, ioctx.operate("foo", &op));
2223 }
2224
2225 // flush + try-flush
2226 {
2227 ObjectReadOperation op;
2228 op.cache_flush();
2229 librados::AioCompletion *completion = cluster.aio_create_completion();
2230 ASSERT_EQ(0, cache_ioctx.aio_operate(
2231 "foo", completion, &op,
2232 librados::OPERATION_IGNORE_OVERLAY, NULL));
2233
2234 ObjectReadOperation op2;
2235 op2.cache_try_flush();
2236 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2237 ASSERT_EQ(0, cache_ioctx.aio_operate(
2238 "foo", completion2, &op2,
2239 librados::OPERATION_IGNORE_OVERLAY |
2240 librados::OPERATION_SKIPRWLOCKS, NULL));
2241
9f95a23c
TL		 2242 completion->wait_for_complete();
2243 completion2->wait_for_complete();
7c673cae
FG		 2244 ASSERT_EQ(0, completion->get_return_value());
2245 ASSERT_EQ(0, completion2->get_return_value());
2246 completion->release();
2247 completion2->release();
2248 }
2249
2250 // create/dirty object
2251 int tries = 1000;
2252 do {
2253 {
2254 bufferlist bl;
2255 bl.append("hi there");
2256 ObjectWriteOperation op;
2257 op.write_full(bl);
2258 ASSERT_EQ(0, ioctx.operate("foo", &op));
2259 }
2260
2261 // try-flush + flush
2262 // (flush will not piggyback on try-flush)
2263 {
2264 ObjectReadOperation op;
2265 op.cache_try_flush();
2266 librados::AioCompletion *completion = cluster.aio_create_completion();
2267 ASSERT_EQ(0, cache_ioctx.aio_operate(
2268 "foo", completion, &op,
2269 librados::OPERATION_IGNORE_OVERLAY |
2270 librados::OPERATION_SKIPRWLOCKS, NULL));
2271
2272 ObjectReadOperation op2;
2273 op2.cache_flush();
2274 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2275 ASSERT_EQ(0, cache_ioctx.aio_operate(
2276 "foo", completion2, &op2,
2277 librados::OPERATION_IGNORE_OVERLAY, NULL));
2278
9f95a23c
TL		 2279 completion->wait_for_complete();
2280 completion2->wait_for_complete();
7c673cae
FG		 2281 int r = completion->get_return_value();
2282 ASSERT_TRUE(r == -EBUSY || r == 0);
2283 ASSERT_EQ(0, completion2->get_return_value());
2284 completion->release();
2285 completion2->release();
2286 if (r == -EBUSY)
2287 break;
2288 cout << "didn't get EBUSY, trying again" << std::endl;
2289 }
2290 ASSERT_TRUE(--tries);
2291 } while (true);
2292
2293 // create/dirty object
2294 {
2295 bufferlist bl;
2296 bl.append("hi there");
2297 ObjectWriteOperation op;
2298 op.write_full(bl);
2299 ASSERT_EQ(0, ioctx.operate("foo", &op));
2300 }
2301
2302 // try-flush + try-flush
2303 {
2304 ObjectReadOperation op;
2305 op.cache_try_flush();
2306 librados::AioCompletion *completion = cluster.aio_create_completion();
2307 ASSERT_EQ(0, cache_ioctx.aio_operate(
2308 "foo", completion, &op,
2309 librados::OPERATION_IGNORE_OVERLAY |
2310 librados::OPERATION_SKIPRWLOCKS, NULL));
2311
2312 ObjectReadOperation op2;
2313 op2.cache_try_flush();
2314 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2315 ASSERT_EQ(0, cache_ioctx.aio_operate(
2316 "foo", completion2, &op2,
2317 librados::OPERATION_IGNORE_OVERLAY |
2318 librados::OPERATION_SKIPRWLOCKS, NULL));
2319
9f95a23c
TL		 2320 completion->wait_for_complete();
2321 completion2->wait_for_complete();
7c673cae
FG		 2322 ASSERT_EQ(0, completion->get_return_value());
2323 ASSERT_EQ(0, completion2->get_return_value());
2324 completion->release();
2325 completion2->release();
2326 }
2327}
2328
2329
2330IoCtx *read_ioctx = 0;
9f95a23c
TL		 2331ceph::mutex test_lock = ceph::make_mutex("FlushReadRaces::lock");
2332ceph::condition_variable cond;
7c673cae
FG		 2333int max_reads = 100;
2334int num_reads = 0; // in progress
2335
2336void flush_read_race_cb(completion_t cb, void *arg);
2337
2338void start_flush_read()
2339{
2340 //cout << " starting read" << std::endl;
2341 ObjectReadOperation op;
2342 op.stat(NULL, NULL, NULL);
2343 librados::AioCompletion *completion = completions.getCompletion();
2344 completion->set_complete_callback(0, flush_read_race_cb);
2345 read_ioctx->aio_operate("foo", completion, &op, NULL);
2346}
2347
2348void flush_read_race_cb(completion_t cb, void *arg)
2349{
2350 //cout << " finished read" << std::endl;
9f95a23c 2351 std::lock_guard l{test_lock};
7c673cae
FG		 2352 if (num_reads > max_reads) {
2353 num_reads--;
9f95a23c 2354 cond.notify_all();
7c673cae
FG		 2355 } else {
2356 start_flush_read();
2357 }
7c673cae
FG		 2358}
2359
2360TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
2361 // configure cache
2362 bufferlist inbl;
2363 ASSERT_EQ(0, cluster.mon_command(
2364 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2365 "\", \"tierpool\": \"" + cache_pool_name +
2366 "\", \"force_nonempty\": \"--force-nonempty\" }",
2367 inbl, NULL, NULL));
2368 ASSERT_EQ(0, cluster.mon_command(
2369 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2370 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2371 inbl, NULL, NULL));
2372 ASSERT_EQ(0, cluster.mon_command(
2373 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2374 "\", \"mode\": \"writeback\"}",
2375 inbl, NULL, NULL));
2376
2377 // wait for maps to settle
2378 cluster.wait_for_latest_osdmap();
2379
2380 // create/dirty object
2381 {
2382 bufferlist bl;
2383 bl.append("hi there");
2384 bufferptr bp(4000000); // make it big!
2385 bp.zero();
2386 bl.append(bp);
2387 ObjectWriteOperation op;
2388 op.write_full(bl);
2389 ASSERT_EQ(0, ioctx.operate("foo", &op));
2390 }
2391
2392 // start a continuous stream of reads
2393 read_ioctx = &ioctx;
9f95a23c 2394 test_lock.lock();
7c673cae
FG		 2395 for (int i = 0; i < max_reads; ++i) {
2396 start_flush_read();
2397 num_reads++;
2398 }
9f95a23c 2399 test_lock.unlock();
7c673cae
FG		 2400
2401 // try-flush
2402 ObjectReadOperation op;
2403 op.cache_try_flush();
2404 librados::AioCompletion *completion = cluster.aio_create_completion();
2405 ASSERT_EQ(0, cache_ioctx.aio_operate(
2406 "foo", completion, &op,
2407 librados::OPERATION_IGNORE_OVERLAY |
2408 librados::OPERATION_SKIPRWLOCKS, NULL));
2409
9f95a23c 2410 completion->wait_for_complete();
7c673cae
FG		 2411 ASSERT_EQ(0, completion->get_return_value());
2412 completion->release();
2413
2414 // stop reads
9f95a23c
TL		 2415 std::unique_lock locker{test_lock};
2416 max_reads = 0;
2417 cond.wait(locker, [] { return num_reads == 0;});
7c673cae
FG		 2418}
2419
2420TEST_F(LibRadosTierPP, HitSetNone) {
2421 {
2422 list< pair<time_t,time_t> > ls;
2423 AioCompletion *c = librados::Rados::aio_create_completion();
2424 ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
2425 c->wait_for_complete();
2426 ASSERT_EQ(0, c->get_return_value());
2427 ASSERT_TRUE(ls.empty());
2428 c->release();
2429 }
2430 {
2431 bufferlist bl;
2432 AioCompletion *c = librados::Rados::aio_create_completion();
2433 ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
2434 c->wait_for_complete();
2435 ASSERT_EQ(-ENOENT, c->get_return_value());
2436 c->release();
2437 }
2438}
2439
2440string set_pool_str(string pool, string var, string val)
2441{
2442 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2443 + string("\",\"var\": \"") + var + string("\",\"val\": \"")
2444 + val + string("\"}");
2445}
2446
2447string set_pool_str(string pool, string var, int val)
2448{
2449 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2450 + string("\",\"var\": \"") + var + string("\",\"val\": \"")
2451 + stringify(val) + string("\"}");
2452}
2453
2454TEST_F(LibRadosTwoPoolsPP, HitSetRead) {
2455 // make it a tier
2456 bufferlist inbl;
2457 ASSERT_EQ(0, cluster.mon_command(
2458 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2459 "\", \"tierpool\": \"" + cache_pool_name +
2460 "\", \"force_nonempty\": \"--force-nonempty\" }",
2461 inbl, NULL, NULL));
2462
2463 // enable hitset tracking for this pool
2464 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
2465 inbl, NULL, NULL));
2466 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
2467 inbl, NULL, NULL));
2468 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
2469 "explicit_object"),
2470 inbl, NULL, NULL));
2471
2472 // wait for maps to settle
2473 cluster.wait_for_latest_osdmap();
2474
2475 cache_ioctx.set_namespace("");
2476
2477 // keep reading until we see our object appear in the HitSet
2478 utime_t start = ceph_clock_now();
2479 utime_t hard_stop = start + utime_t(600, 0);
2480
2481 while (true) {
2482 utime_t now = ceph_clock_now();
2483 ASSERT_TRUE(now < hard_stop);
2484
2485 string name = "foo";
2486 uint32_t hash;
2487 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
2488 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
2489 cluster.pool_lookup(cache_pool_name.c_str()), "");
2490
2491 bufferlist bl;
2492 ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
2493
2494 bufferlist hbl;
2495 AioCompletion *c = librados::Rados::aio_create_completion();
2496 ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
2497 c->wait_for_complete();
2498 c->release();
2499
2500 if (hbl.length()) {
11fdf7f2 2501 auto p = hbl.cbegin();
7c673cae 2502 HitSet hs;
11fdf7f2 2503 decode(hs, p);
7c673cae
FG		 2504 if (hs.contains(oid)) {
2505 cout << "ok, hit_set contains " << oid << std::endl;
2506 break;
2507 }
2508 cout << "hmm, not in HitSet yet" << std::endl;
2509 } else {
2510 cout << "hmm, no HitSet yet" << std::endl;
2511 }
2512
2513 sleep(1);
2514 }
2515}
2516
2517static int _get_pg_num(Rados& cluster, string pool_name)
2518{
2519 bufferlist inbl;
2520 string cmd = string("{\"prefix\": \"osd pool get\",\"pool\":\"")
2521 + pool_name
2522 + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
2523 bufferlist outbl;
2524 int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
11fdf7f2 2525 ceph_assert(r >= 0);
7c673cae
FG		 2526 string outstr(outbl.c_str(), outbl.length());
2527 json_spirit::Value v;
2528 if (!json_spirit::read(outstr, v)) {
2529 cerr <<" unable to parse json " << outstr << std::endl;
2530 return -1;
2531 }
2532
2533 json_spirit::Object& o = v.get_obj();
2534 for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
2535 json_spirit::Pair& p = o[i];
2536 if (p.name_ == "pg_num") {
2537 cout << "pg_num = " << p.value_.get_int() << std::endl;
2538 return p.value_.get_int();
2539 }
2540 }
2541 cerr << "didn't find pg_num in " << outstr << std::endl;
2542 return -1;
2543}
2544
7c673cae
FG		 2545TEST_F(LibRadosTwoPoolsPP, HitSetWrite) {
2546 int num_pg = _get_pg_num(cluster, pool_name);
11fdf7f2 2547 ceph_assert(num_pg > 0);
7c673cae
FG		 2548
2549 // make it a tier
2550 bufferlist inbl;
2551 ASSERT_EQ(0, cluster.mon_command(
2552 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2553 "\", \"tierpool\": \"" + cache_pool_name +
2554 "\", \"force_nonempty\": \"--force-nonempty\" }",
2555 inbl, NULL, NULL));
2556
2557 // enable hitset tracking for this pool
2558 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 8),
2559 inbl, NULL, NULL));
2560 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
2561 inbl, NULL, NULL));
2562 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
2563 "explicit_hash"),
2564 inbl, NULL, NULL));
2565
2566 // wait for maps to settle
2567 cluster.wait_for_latest_osdmap();
2568
2569 cache_ioctx.set_namespace("");
2570
2571 int num = 200;
2572
2573 // do a bunch of writes
2574 for (int i=0; i<num; ++i) {
2575 bufferlist bl;
2576 bl.append("a");
2577 ASSERT_EQ(0, cache_ioctx.write(stringify(i), bl, 1, 0));
2578 }
2579
2580 // get HitSets
2581 std::map<int,HitSet> hitsets;
2582 for (int i=0; i<num_pg; ++i) {
2583 list< pair<time_t,time_t> > ls;
2584 AioCompletion *c = librados::Rados::aio_create_completion();
2585 ASSERT_EQ(0, cache_ioctx.hit_set_list(i, c, &ls));
2586 c->wait_for_complete();
2587 c->release();
2588 std::cout << "pg " << i << " ls " << ls << std::endl;
2589 ASSERT_FALSE(ls.empty());
2590
2591 // get the latest
2592 c = librados::Rados::aio_create_completion();
2593 bufferlist bl;
2594 ASSERT_EQ(0, cache_ioctx.hit_set_get(i, c, ls.back().first, &bl));
2595 c->wait_for_complete();
2596 c->release();
2597
2598 try {
11fdf7f2
TL		 2599 auto p = bl.cbegin();
2600 decode(hitsets[i], p);
7c673cae
FG		 2601 }
2602 catch (buffer::error& e) {
2603 std::cout << "failed to decode hit set; bl len is " << bl.length() << "\n";
2604 bl.hexdump(std::cout);
2605 std::cout << std::endl;
2606 throw e;
2607 }
2608
2609 // cope with racing splits by refreshing pg_num
2610 if (i == num_pg - 1)
2611 num_pg = _get_pg_num(cluster, cache_pool_name);
2612 }
2613
2614 for (int i=0; i<num; ++i) {
2615 string n = stringify(i);
2616 uint32_t hash;
2617 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(n, &hash));
2618 hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
2619 cluster.pool_lookup(cache_pool_name.c_str()), "");
2620 std::cout << "checking for " << oid << std::endl;
2621 bool found = false;
2622 for (int p=0; p<num_pg; ++p) {
2623 if (hitsets[p].contains(oid)) {
2624 found = true;
2625 break;
2626 }
2627 }
2628 ASSERT_TRUE(found);
2629 }
2630}
2631
2632TEST_F(LibRadosTwoPoolsPP, HitSetTrim) {
2633 unsigned count = 3;
2634 unsigned period = 3;
2635
2636 // make it a tier
2637 bufferlist inbl;
2638 ASSERT_EQ(0, cluster.mon_command(
2639 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2640 "\", \"tierpool\": \"" + cache_pool_name +
2641 "\", \"force_nonempty\": \"--force-nonempty\" }",
2642 inbl, NULL, NULL));
2643
2644 // enable hitset tracking for this pool
2645 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
2646 inbl, NULL, NULL));
2647 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
2648 inbl, NULL, NULL));
2649 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2650 inbl, NULL, NULL));
2651 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
2652 inbl, NULL, NULL));
2653
2654 // wait for maps to settle
2655 cluster.wait_for_latest_osdmap();
2656
2657 cache_ioctx.set_namespace("");
2658
2659 // do a bunch of writes and make sure the hitsets rotate
2660 utime_t start = ceph_clock_now();
2661 utime_t hard_stop = start + utime_t(count * period * 50, 0);
2662
2663 time_t first = 0;
2664 while (true) {
2665 string name = "foo";
2666 uint32_t hash;
2667 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
2668 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
2669
2670 bufferlist bl;
2671 bl.append("f");
2672 ASSERT_EQ(0, cache_ioctx.write("foo", bl, 1, 0));
2673
2674 list<pair<time_t, time_t> > ls;
2675 AioCompletion *c = librados::Rados::aio_create_completion();
2676 ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
2677 c->wait_for_complete();
2678 c->release();
2679
2680 cout << " got ls " << ls << std::endl;
2681 if (!ls.empty()) {
2682 if (!first) {
2683 first = ls.front().first;
2684 cout << "first is " << first << std::endl;
2685 } else {
2686 if (ls.front().first != first) {
2687 cout << "first now " << ls.front().first << ", trimmed" << std::endl;
2688 break;
2689 }
2690 }
2691 }
2692
2693 utime_t now = ceph_clock_now();
2694 ASSERT_TRUE(now < hard_stop);
2695
2696 sleep(1);
2697 }
2698}
2699
2700TEST_F(LibRadosTwoPoolsPP, PromoteOn2ndRead) {
2701 // create object
2702 for (int i=0; i<20; ++i) {
2703 bufferlist bl;
2704 bl.append("hi there");
2705 ObjectWriteOperation op;
2706 op.write_full(bl);
2707 ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
2708 }
2709
2710 // configure cache
2711 bufferlist inbl;
2712 ASSERT_EQ(0, cluster.mon_command(
2713 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2714 "\", \"tierpool\": \"" + cache_pool_name +
2715 "\", \"force_nonempty\": \"--force-nonempty\" }",
2716 inbl, NULL, NULL));
2717 ASSERT_EQ(0, cluster.mon_command(
2718 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2719 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2720 inbl, NULL, NULL));
2721 ASSERT_EQ(0, cluster.mon_command(
2722 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2723 "\", \"mode\": \"writeback\"}",
2724 inbl, NULL, NULL));
2725
2726 // enable hitset tracking for this pool
2727 ASSERT_EQ(0, cluster.mon_command(
2728 set_pool_str(cache_pool_name, "hit_set_count", 2),
2729 inbl, NULL, NULL));
2730 ASSERT_EQ(0, cluster.mon_command(
2731 set_pool_str(cache_pool_name, "hit_set_period", 600),
2732 inbl, NULL, NULL));
2733 ASSERT_EQ(0, cluster.mon_command(
2734 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2735 inbl, NULL, NULL));
2736 ASSERT_EQ(0, cluster.mon_command(
2737 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
2738 inbl, NULL, NULL));
2739 ASSERT_EQ(0, cluster.mon_command(
2740 set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
2741 inbl, NULL, NULL));
2742 ASSERT_EQ(0, cluster.mon_command(
2743 set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
2744 inbl, NULL, NULL));
2745
2746 // wait for maps to settle
2747 cluster.wait_for_latest_osdmap();
2748
2749 int fake = 0; // set this to non-zero to test spurious promotion,
2750 // e.g. from thrashing
2751 int attempt = 0;
2752 string obj;
2753 while (true) {
2754 // 1st read, don't trigger a promote
2755 obj = "foo" + stringify(attempt);
2756 cout << obj << std::endl;
2757 {
2758 bufferlist bl;
2759 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2760 if (--fake >= 0) {
2761 sleep(1);
2762 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2763 sleep(1);
2764 }
2765 }
2766
2767 // verify the object is NOT present in the cache tier
2768 {
2769 bool found = false;
2770 NObjectIterator it = cache_ioctx.nobjects_begin();
2771 while (it != cache_ioctx.nobjects_end()) {
2772 cout << " see " << it->get_oid() << std::endl;
2773 if (it->get_oid() == string(obj.c_str())) {
2774 found = true;
2775 break;
2776 }
2777 ++it;
2778 }
2779 if (!found)
2780 break;
2781 }
2782
2783 ++attempt;
2784 ASSERT_LE(attempt, 20);
2785 cout << "hrm, object is present in cache on attempt " << attempt
2786 << ", retrying" << std::endl;
2787 }
2788
2789 // Read until the object is present in the cache tier
2790 cout << "verifying " << obj << " is eventually promoted" << std::endl;
2791 while (true) {
2792 bufferlist bl;
2793 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2794
2795 bool there = false;
2796 NObjectIterator it = cache_ioctx.nobjects_begin();
2797 while (it != cache_ioctx.nobjects_end()) {
2798 if (it->get_oid() == string(obj.c_str())) {
2799 there = true;
2800 break;
2801 }
2802 ++it;
2803 }
2804 if (there)
2805 break;
2806
2807 sleep(1);
2808 }
2809
2810 // tear down tiers
2811 ASSERT_EQ(0, cluster.mon_command(
2812 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2813 "\"}",
2814 inbl, NULL, NULL));
2815 ASSERT_EQ(0, cluster.mon_command(
2816 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2817 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2818 inbl, NULL, NULL));
2819
2820 // wait for maps to settle before next test
2821 cluster.wait_for_latest_osdmap();
2822}
2823
2824TEST_F(LibRadosTwoPoolsPP, ProxyRead) {
2825 // create object
2826 {
2827 bufferlist bl;
2828 bl.append("hi there");
2829 ObjectWriteOperation op;
2830 op.write_full(bl);
2831 ASSERT_EQ(0, ioctx.operate("foo", &op));
2832 }
2833
2834 // configure cache
2835 bufferlist inbl;
2836 ASSERT_EQ(0, cluster.mon_command(
2837 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2838 "\", \"tierpool\": \"" + cache_pool_name +
2839 "\", \"force_nonempty\": \"--force-nonempty\" }",
2840 inbl, NULL, NULL));
2841 ASSERT_EQ(0, cluster.mon_command(
2842 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2843 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2844 inbl, NULL, NULL));
2845 ASSERT_EQ(0, cluster.mon_command(
2846 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2847 "\", \"mode\": \"readproxy\"}",
2848 inbl, NULL, NULL));
2849
2850 // wait for maps to settle
2851 cluster.wait_for_latest_osdmap();
2852
2853 // read and verify the object
2854 {
2855 bufferlist bl;
2856 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2857 ASSERT_EQ('h', bl[0]);
2858 }
2859
2860 // Verify 10 times the object is NOT present in the cache tier
2861 uint32_t i = 0;
2862 while (i++ < 10) {
2863 NObjectIterator it = cache_ioctx.nobjects_begin();
2864 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
2865 sleep(1);
2866 }
2867
2868 // tear down tiers
2869 ASSERT_EQ(0, cluster.mon_command(
2870 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2871 "\"}",
2872 inbl, NULL, NULL));
2873 ASSERT_EQ(0, cluster.mon_command(
2874 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2875 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2876 inbl, NULL, NULL));
2877
2878 // wait for maps to settle before next test
2879 cluster.wait_for_latest_osdmap();
2880}
2881
2882TEST_F(LibRadosTwoPoolsPP, CachePin) {
2883 // create object
2884 {
2885 bufferlist bl;
2886 bl.append("hi there");
2887 ObjectWriteOperation op;
2888 op.write_full(bl);
2889 ASSERT_EQ(0, ioctx.operate("foo", &op));
2890 }
2891 {
2892 bufferlist bl;
2893 bl.append("hi there");
2894 ObjectWriteOperation op;
2895 op.write_full(bl);
2896 ASSERT_EQ(0, ioctx.operate("bar", &op));
2897 }
2898 {
2899 bufferlist bl;
2900 bl.append("hi there");
2901 ObjectWriteOperation op;
2902 op.write_full(bl);
2903 ASSERT_EQ(0, ioctx.operate("baz", &op));
2904 }
2905 {
2906 bufferlist bl;
2907 bl.append("hi there");
2908 ObjectWriteOperation op;
2909 op.write_full(bl);
2910 ASSERT_EQ(0, ioctx.operate("bam", &op));
2911 }
2912
2913 // configure cache
2914 bufferlist inbl;
2915 ASSERT_EQ(0, cluster.mon_command(
2916 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2917 "\", \"tierpool\": \"" + cache_pool_name +
2918 "\", \"force_nonempty\": \"--force-nonempty\" }",
2919 inbl, NULL, NULL));
2920 ASSERT_EQ(0, cluster.mon_command(
2921 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2922 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2923 inbl, NULL, NULL));
2924 ASSERT_EQ(0, cluster.mon_command(
2925 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2926 "\", \"mode\": \"writeback\"}",
2927 inbl, NULL, NULL));
2928
2929 // wait for maps to settle
2930 cluster.wait_for_latest_osdmap();
2931
2932 // read, trigger promote
2933 {
2934 bufferlist bl;
2935 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2936 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
2937 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
2938 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
2939 }
2940
2941 // verify the objects are present in the cache tier
2942 {
2943 NObjectIterator it = cache_ioctx.nobjects_begin();
2944 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
2945 for (uint32_t i = 0; i < 4; i++) {
2946 ASSERT_TRUE(it->get_oid() == string("foo") ||
2947 it->get_oid() == string("bar") ||
2948 it->get_oid() == string("baz") ||
2949 it->get_oid() == string("bam"));
2950 ++it;
2951 }
2952 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
2953 }
2954
2955 // pin objects
2956 {
2957 ObjectWriteOperation op;
2958 op.cache_pin();
2959 librados::AioCompletion *completion = cluster.aio_create_completion();
2960 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 2961 completion->wait_for_complete();
7c673cae
FG		 2962 ASSERT_EQ(0, completion->get_return_value());
2963 completion->release();
2964 }
2965 {
2966 ObjectWriteOperation op;
2967 op.cache_pin();
2968 librados::AioCompletion *completion = cluster.aio_create_completion();
2969 ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
9f95a23c 2970 completion->wait_for_complete();
7c673cae
FG		 2971 ASSERT_EQ(0, completion->get_return_value());
2972 completion->release();
2973 }
2974
2975 // enable agent
2976 ASSERT_EQ(0, cluster.mon_command(
2977 set_pool_str(cache_pool_name, "hit_set_count", 2),
2978 inbl, NULL, NULL));
2979 ASSERT_EQ(0, cluster.mon_command(
2980 set_pool_str(cache_pool_name, "hit_set_period", 600),
2981 inbl, NULL, NULL));
2982 ASSERT_EQ(0, cluster.mon_command(
2983 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2984 inbl, NULL, NULL));
2985 ASSERT_EQ(0, cluster.mon_command(
2986 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
2987 inbl, NULL, NULL));
2988 ASSERT_EQ(0, cluster.mon_command(
2989 set_pool_str(cache_pool_name, "target_max_objects", 1),
2990 inbl, NULL, NULL));
2991
2992 sleep(10);
2993
2994 // Verify the pinned object 'foo' is not flushed/evicted
2995 uint32_t count = 0;
2996 while (true) {
2997 bufferlist bl;
2998 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
2999
3000 count = 0;
3001 NObjectIterator it = cache_ioctx.nobjects_begin();
3002 while (it != cache_ioctx.nobjects_end()) {
3003 ASSERT_TRUE(it->get_oid() == string("foo") ||
3004 it->get_oid() == string("bar") ||
3005 it->get_oid() == string("baz") ||
3006 it->get_oid() == string("bam"));
3007 ++count;
3008 ++it;
3009 }
3010 if (count == 2) {
3011 ASSERT_TRUE(it->get_oid() == string("foo") ||
3012 it->get_oid() == string("baz"));
3013 break;
3014 }
3015
3016 sleep(1);
3017 }
3018
3019 // tear down tiers
3020 ASSERT_EQ(0, cluster.mon_command(
3021 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
3022 "\"}",
3023 inbl, NULL, NULL));
3024 ASSERT_EQ(0, cluster.mon_command(
3025 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
3026 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
3027 inbl, NULL, NULL));
3028
3029 // wait for maps to settle before next test
3030 cluster.wait_for_latest_osdmap();
3031}
3032
11fdf7f2
TL		 3033TEST_F(LibRadosTwoPoolsPP, SetRedirectRead) {
3034 // create object
3035 {
3036 bufferlist bl;
3037 bl.append("hi there");
3038 ObjectWriteOperation op;
3039 op.write_full(bl);
3040 ASSERT_EQ(0, ioctx.operate("foo", &op));
3041 }
3042 {
3043 bufferlist bl;
3044 bl.append("there");
3045 ObjectWriteOperation op;
3046 op.write_full(bl);
3047 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3048 }
3049
11fdf7f2
TL		 3050 // wait for maps to settle
3051 cluster.wait_for_latest_osdmap();
3052
3053 {
3054 ObjectWriteOperation op;
3055 op.set_redirect("bar", cache_ioctx, 0);
3056 librados::AioCompletion *completion = cluster.aio_create_completion();
3057 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3058 completion->wait_for_complete();
11fdf7f2
TL		 3059 ASSERT_EQ(0, completion->get_return_value());
3060 completion->release();
3061 }
3062 // read and verify the object
3063 {
3064 bufferlist bl;
3065 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3066 ASSERT_EQ('t', bl[0]);
3067 }
3068
11fdf7f2
TL		 3069 // wait for maps to settle before next test
3070 cluster.wait_for_latest_osdmap();
3071}
3072
3073TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
3074 // skip test if not yet mimic
9f95a23c
TL		 3075 if (_get_required_osd_release(cluster) < "mimic") {
3076 GTEST_SKIP() << "cluster is not yet mimic, skipping test";
11fdf7f2
TL		 3077 }
3078
3079 // create object
3080 {
3081 bufferlist bl;
3082 bl.append("hi there");
3083 ObjectWriteOperation op;
3084 op.write_full(bl);
3085 ASSERT_EQ(0, ioctx.operate("foo", &op));
3086 }
3087 {
3088 bufferlist bl;
3089 bl.append("base chunk");
3090 ObjectWriteOperation op;
3091 op.write_full(bl);
3092 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3093 }
3094 {
3095 bufferlist bl;
3096 bl.append("there");
3097 ObjectWriteOperation op;
3098 op.write_full(bl);
3099 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3100 }
3101 {
3102 bufferlist bl;
3103 bl.append("CHUNK");
3104 ObjectWriteOperation op;
3105 op.write_full(bl);
3106 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3107 }
3108
11fdf7f2
TL		 3109 // wait for maps to settle
3110 cluster.wait_for_latest_osdmap();
3111
3112 // set-redirect
3113 {
3114 ObjectWriteOperation op;
3115 op.set_redirect("bar", cache_ioctx, 0);
3116 librados::AioCompletion *completion = cluster.aio_create_completion();
3117 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3118 completion->wait_for_complete();
11fdf7f2
TL		 3119 ASSERT_EQ(0, completion->get_return_value());
3120 completion->release();
3121 }
3122 // set-chunk
f67539c2
TL		 3123 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, "bar-chunk", "foo-chunk");
3124
11fdf7f2
TL		 3125 // promote
3126 {
3127 ObjectWriteOperation op;
3128 op.tier_promote();
3129 librados::AioCompletion *completion = cluster.aio_create_completion();
3130 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3131 completion->wait_for_complete();
11fdf7f2
TL		 3132 ASSERT_EQ(0, completion->get_return_value());
3133 completion->release();
3134 }
3135 // read and verify the object (redirect)
3136 {
3137 bufferlist bl;
3138 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3139 ASSERT_EQ('t', bl[0]);
3140 }
3141 // promote
3142 {
3143 ObjectWriteOperation op;
3144 op.tier_promote();
3145 librados::AioCompletion *completion = cluster.aio_create_completion();
3146 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3147 completion->wait_for_complete();
11fdf7f2
TL		 3148 ASSERT_EQ(0, completion->get_return_value());
3149 completion->release();
3150 }
3151 // read and verify the object
3152 {
3153 bufferlist bl;
3154 ASSERT_EQ(1, ioctx.read("foo-chunk", bl, 1, 0));
3155 ASSERT_EQ('C', bl[0]);
3156 }
3157
11fdf7f2
TL		 3158 // wait for maps to settle before next test
3159 cluster.wait_for_latest_osdmap();
3160}
3161
3162TEST_F(LibRadosTwoPoolsPP, ManifestRefRead) {
3163 // note: require >= mimic
3164
3165 // create object
3166 {
3167 bufferlist bl;
3168 bl.append("hi there");
3169 ObjectWriteOperation op;
3170 op.write_full(bl);
3171 ASSERT_EQ(0, ioctx.operate("foo", &op));
3172 }
3173 {
3174 bufferlist bl;
3175 bl.append("base chunk");
3176 ObjectWriteOperation op;
3177 op.write_full(bl);
3178 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3179 }
3180 {
3181 bufferlist bl;
3182 bl.append("there");
3183 ObjectWriteOperation op;
3184 op.write_full(bl);
3185 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3186 }
3187 {
3188 bufferlist bl;
3189 bl.append("CHUNK");
3190 ObjectWriteOperation op;
3191 op.write_full(bl);
3192 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3193 }
3194
3195 // wait for maps to settle
3196 cluster.wait_for_latest_osdmap();
3197
3198 // set-redirect
3199 {
3200 ObjectWriteOperation op;
3201 op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3202 librados::AioCompletion *completion = cluster.aio_create_completion();
3203 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3204 completion->wait_for_complete();
11fdf7f2
TL		 3205 ASSERT_EQ(0, completion->get_return_value());
3206 completion->release();
3207 }
3208 // set-chunk
3209 {
f67539c2 3210 ObjectReadOperation op;
11fdf7f2
TL		 3211 op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3212 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2
TL		 3213 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op,
3214 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 3215 completion->wait_for_complete();
11fdf7f2
TL		 3216 ASSERT_EQ(0, completion->get_return_value());
3217 completion->release();
3218 }
3219 // redirect's refcount
3220 {
f67539c2
TL		 3221 bufferlist t;
3222 cache_ioctx.getxattr("bar", CHUNK_REFCOUNT_ATTR, t);
3223 chunk_refs_t refs;
11fdf7f2 3224 try {
f67539c2
TL		 3225 auto iter = t.cbegin();
3226 decode(refs, iter);
11fdf7f2
TL		 3227 } catch (buffer::error& err) {
3228 ASSERT_TRUE(0);
3229 }
f67539c2 3230 ASSERT_EQ(1U, refs.count());
11fdf7f2
TL		 3231 }
3232 // chunk's refcount
3233 {
f67539c2
TL		 3234 bufferlist t;
3235 cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
3236 chunk_refs_t refs;
11fdf7f2 3237 try {
f67539c2
TL		 3238 auto iter = t.cbegin();
3239 decode(refs, iter);
11fdf7f2
TL		 3240 } catch (buffer::error& err) {
3241 ASSERT_TRUE(0);
3242 }
f67539c2 3243 ASSERT_EQ(1u, refs.count());
11fdf7f2
TL		 3244 }
3245
3246 // wait for maps to settle before next test
3247 cluster.wait_for_latest_osdmap();
3248}
3249
3250TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
3251 // skip test if not yet nautilus
9f95a23c
TL		 3252 if (_get_required_osd_release(cluster) < "nautilus") {
3253 GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
11fdf7f2
TL		 3254 }
3255
3256 // create object
3257 {
3258 bufferlist bl;
3259 bl.append("hi there");
3260 ObjectWriteOperation op;
3261 op.write_full(bl);
3262 ASSERT_EQ(0, ioctx.operate("foo", &op));
3263 }
3264 {
3265 bufferlist bl;
3266 bl.append("base chunk");
3267 ObjectWriteOperation op;
3268 op.write_full(bl);
3269 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3270 }
3271 {
3272 bufferlist bl;
3273 bl.append("there");
3274 ObjectWriteOperation op;
3275 op.write_full(bl);
3276 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3277 }
3278 {
3279 bufferlist bl;
3280 bl.append("CHUNK");
3281 ObjectWriteOperation op;
3282 op.write_full(bl);
3283 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3284 }
3285
3286 // wait for maps to settle
3287 cluster.wait_for_latest_osdmap();
3288
3289 // set-redirect
3290 {
3291 ObjectWriteOperation op;
3292 op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3293 librados::AioCompletion *completion = cluster.aio_create_completion();
3294 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3295 completion->wait_for_complete();
11fdf7f2
TL		 3296 ASSERT_EQ(0, completion->get_return_value());
3297 completion->release();
3298 }
3299 // set-chunk
3300 {
f67539c2 3301 ObjectReadOperation op;
11fdf7f2
TL		 3302 op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3303 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2
TL		 3304 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op,
3305 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 3306 completion->wait_for_complete();
11fdf7f2
TL		 3307 ASSERT_EQ(0, completion->get_return_value());
3308 completion->release();
3309 }
3310 // redirect's refcount
3311 {
f67539c2
TL		 3312 bufferlist t;
3313 cache_ioctx.getxattr("bar", CHUNK_REFCOUNT_ATTR, t);
3314 chunk_refs_t refs;
11fdf7f2 3315 try {
f67539c2
TL		 3316 auto iter = t.cbegin();
3317 decode(refs, iter);
11fdf7f2
TL		 3318 } catch (buffer::error& err) {
3319 ASSERT_TRUE(0);
3320 }
f67539c2 3321 ASSERT_EQ(1u, refs.count());
11fdf7f2
TL		 3322 }
3323 // chunk's refcount
3324 {
f67539c2
TL		 3325 bufferlist t;
3326 cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
3327 chunk_refs_t refs;
11fdf7f2 3328 try {
f67539c2
TL		 3329 auto iter = t.cbegin();
3330 decode(refs, iter);
11fdf7f2
TL		 3331 } catch (buffer::error& err) {
3332 ASSERT_TRUE(0);
3333 }
f67539c2 3334 ASSERT_EQ(1u, refs.count());
11fdf7f2
TL		 3335 }
3336
3337 // unset-manifest for set-redirect
3338 {
3339 ObjectWriteOperation op;
3340 op.unset_manifest();
3341 librados::AioCompletion *completion = cluster.aio_create_completion();
3342 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3343 completion->wait_for_complete();
11fdf7f2
TL		 3344 ASSERT_EQ(0, completion->get_return_value());
3345 completion->release();
3346 }
3347
3348 // unset-manifest for set-chunk
3349 {
3350 ObjectWriteOperation op;
3351 op.unset_manifest();
3352 librados::AioCompletion *completion = cluster.aio_create_completion();
3353 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3354 completion->wait_for_complete();
11fdf7f2
TL		 3355 ASSERT_EQ(0, completion->get_return_value());
3356 completion->release();
3357 }
3358 // redirect's refcount
3359 {
f67539c2
TL		 3360 bufferlist t;
3361 cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
3362 if (t.length() != 0U) {
11fdf7f2
TL		 3363 ObjectWriteOperation op;
3364 op.unset_manifest();
3365 librados::AioCompletion *completion = cluster.aio_create_completion();
3366 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3367 completion->wait_for_complete();
11fdf7f2
TL		 3368 ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
3369 completion->release();
3370 }
3371 }
3372 // chunk's refcount
3373 {
f67539c2
TL		 3374 bufferlist t;
3375 cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
3376 if (t.length() != 0U) {
11fdf7f2
TL		 3377 ObjectWriteOperation op;
3378 op.unset_manifest();
3379 librados::AioCompletion *completion = cluster.aio_create_completion();
3380 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3381 completion->wait_for_complete();
11fdf7f2
TL		 3382 ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
3383 completion->release();
3384 }
3385 }
3386
3387 // wait for maps to settle before next test
3388 cluster.wait_for_latest_osdmap();
3389}
3390
3391#include "common/ceph_crypto.h"
3392using ceph::crypto::SHA1;
3393#include "rgw/rgw_common.h"
3394TEST_F(LibRadosTwoPoolsPP, ManifestDedupRefRead) {
3395 // skip test if not yet nautilus
9f95a23c
TL		 3396 if (_get_required_osd_release(cluster) < "nautilus") {
3397 GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
31f18b77 3398 }
9f95a23c 3399
11fdf7f2
TL		 3400 bufferlist inbl;
3401 ASSERT_EQ(0, cluster.mon_command(
3402 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
3403 inbl, NULL, NULL));
3404 cluster.wait_for_latest_osdmap();
f67539c2
TL		 3405 string tgt_oid;
3406
3407 // get fp_oid
3408 tgt_oid = get_fp_oid("There hi", "sha1");
31f18b77
FG		 3409
3410 // create object
3411 {
3412 bufferlist bl;
f67539c2 3413 bl.append("There hi");
31f18b77
FG		 3414 ObjectWriteOperation op;
3415 op.write_full(bl);
3416 ASSERT_EQ(0, ioctx.operate("foo", &op));
3417 }
11fdf7f2
TL		 3418 {
3419 bufferlist bl;
f67539c2 3420 bl.append("There hi");
11fdf7f2
TL		 3421 ObjectWriteOperation op;
3422 op.write_full(bl);
3423 ASSERT_EQ(0, ioctx.operate("foo-dedup", &op));
3424 }
f67539c2
TL		 3425
3426 // write
31f18b77 3427 {
31f18b77 3428 ObjectWriteOperation op;
f67539c2
TL		 3429 bufferlist bl;
3430 bl.append("There hi");
31f18b77 3431 op.write_full(bl);
f67539c2 3432 ASSERT_EQ(0, cache_ioctx.operate(tgt_oid, &op));
31f18b77 3433 }
f67539c2
TL		 3434
3435 // set-chunk (dedup)
3436 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 8, tgt_oid, "foo-dedup");
3437 // set-chunk (dedup)
3438 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 8, tgt_oid, "foo");
3439 // chunk's refcount
11fdf7f2 3440 {
f67539c2
TL		 3441 bufferlist t;
3442 cache_ioctx.getxattr(tgt_oid, CHUNK_REFCOUNT_ATTR, t);
3443 chunk_refs_t refs;
3444 try {
3445 auto iter = t.cbegin();
3446 decode(refs, iter);
3447 } catch (buffer::error& err) {
3448 ASSERT_TRUE(0);
3449 }
a4b75251 3450 ASSERT_LE(2u, refs.count());
11fdf7f2 3451 }
31f18b77 3452
f67539c2 3453 // wait for maps to settle before next test
31f18b77 3454 cluster.wait_for_latest_osdmap();
f67539c2 3455}
31f18b77 3456
f67539c2
TL		 3457TEST_F(LibRadosTwoPoolsPP, ManifestSnapRefcount) {
3458 // skip test if not yet octopus
3459 if (_get_required_osd_release(cluster) < "octopus") {
3460 cout << "cluster is not yet octopus, skipping test" << std::endl;
3461 return;
11fdf7f2 3462 }
f67539c2
TL		 3463
3464 bufferlist inbl;
3465 ASSERT_EQ(0, cluster.mon_command(
3466 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
3467 inbl, NULL, NULL));
3468 cluster.wait_for_latest_osdmap();
3469
3470 // create object
11fdf7f2 3471 {
f67539c2
TL		 3472 bufferlist bl;
3473 bl.append("there hi");
11fdf7f2 3474 ObjectWriteOperation op;
f67539c2
TL		 3475 op.write_full(bl);
3476 ASSERT_EQ(0, ioctx.operate("foo", &op));
31f18b77 3477 }
31f18b77
FG		 3478 {
3479 bufferlist bl;
f67539c2 3480 bl.append("there hi");
11fdf7f2
TL		 3481 ObjectWriteOperation op;
3482 op.write_full(bl);
f67539c2 3483 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
11fdf7f2 3484 }
f67539c2
TL		 3485
3486 // wait for maps to settle
3487 cluster.wait_for_latest_osdmap();
3488
3489 string er_fp_oid, hi_fp_oid, bb_fp_oid;
3490
3491 // get fp_oid
3492 er_fp_oid = get_fp_oid("er", "sha1");
3493 hi_fp_oid = get_fp_oid("hi", "sha1");
3494 bb_fp_oid = get_fp_oid("bb", "sha1");
3495
3496 // write
11fdf7f2 3497 {
11fdf7f2 3498 ObjectWriteOperation op;
f67539c2
TL		 3499 bufferlist bl;
3500 bl.append("er");
11fdf7f2 3501 op.write_full(bl);
f67539c2 3502 ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
11fdf7f2 3503 }
f67539c2 3504 // write
11fdf7f2 3505 {
11fdf7f2 3506 ObjectWriteOperation op;
f67539c2
TL		 3507 bufferlist bl;
3508 bl.append("hi");
11fdf7f2 3509 op.write_full(bl);
f67539c2 3510 ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
11fdf7f2 3511 }
f67539c2 3512 // write
11fdf7f2 3513 {
11fdf7f2 3514 ObjectWriteOperation op;
f67539c2
TL		 3515 bufferlist bl;
3516 bl.append("bb");
11fdf7f2 3517 op.write_full(bl);
f67539c2 3518 ASSERT_EQ(0, cache_ioctx.operate(bb_fp_oid, &op));
11fdf7f2 3519 }
f67539c2
TL		 3520
3521 // set-chunk (dedup)
3522 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
3523 // set-chunk (dedup)
3524 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, hi_fp_oid, "foo");
3525
3526 // make all chunks dirty --> flush
3527 // foo: [er] [hi]
3528
3529 // check chunk's refcount
11fdf7f2 3530 {
f67539c2 3531 bufferlist t;
11fdf7f2 3532 SHA1 sha1_gen;
f67539c2 3533 int size = strlen("er");
11fdf7f2
TL		 3534 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3535 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
f67539c2 3536 sha1_gen.Update((const unsigned char *)"er", size);
11fdf7f2
TL		 3537 sha1_gen.Final(fingerprint);
3538 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
f67539c2
TL		 3539 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3540 chunk_refs_t refs;
11fdf7f2 3541 try {
f67539c2
TL		 3542 auto iter = t.cbegin();
3543 decode(refs, iter);
11fdf7f2
TL		 3544 } catch (buffer::error& err) {
3545 ASSERT_TRUE(0);
3546 }
a4b75251 3547 ASSERT_LE(1u, refs.count());
31f18b77 3548 }
31f18b77 3549
f67539c2
TL		 3550 // create a snapshot, clone
3551 vector<uint64_t> my_snaps(1);
3552 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3553 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3554 my_snaps));
9f95a23c 3555
f67539c2
TL		 3556 // foo: [bb] [hi]
3557 // create a clone
9f95a23c
TL		 3558 {
3559 bufferlist bl;
f67539c2
TL		 3560 bl.append("Thbbe");
3561 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
9f95a23c 3562 }
f67539c2 3563 // make clean
9f95a23c
TL		 3564 {
3565 bufferlist bl;
f67539c2
TL		 3566 bl.append("Thbbe");
3567 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3568 }
3569 // set-chunk (dedup)
3570 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, bb_fp_oid, "foo");
3571
3572 // and another
3573 my_snaps.resize(2);
3574 my_snaps[1] = my_snaps[0];
3575 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3576 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3577 my_snaps));
3578
3579 // foo: [er] [hi]
3580 // create a clone
3581 {
3582 bufferlist bl;
3583 bl.append("There");
3584 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3585 }
3586 // make clean
3587 {
3588 bufferlist bl;
3589 bl.append("There");
3590 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3591 }
3592 // set-chunk (dedup)
3593 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
3594
3595 // check chunk's refcount
3596 {
3597 bufferlist t;
3598 SHA1 sha1_gen;
3599 int size = strlen("er");
3600 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3601 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3602 sha1_gen.Update((const unsigned char *)"er", size);
3603 sha1_gen.Final(fingerprint);
3604 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3605 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3606 chunk_refs_t refs;
3607 try {
3608 auto iter = t.cbegin();
3609 decode(refs, iter);
3610 } catch (buffer::error& err) {
3611 ASSERT_TRUE(0);
3612 }
a4b75251 3613 ASSERT_LE(2u, refs.count());
f67539c2
TL		 3614 }
3615
3616 // and another
3617 my_snaps.resize(3);
3618 my_snaps[2] = my_snaps[1];
3619 my_snaps[1] = my_snaps[0];
3620 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3621 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3622 my_snaps));
3623
3624 // foo: [bb] [hi]
3625 // create a clone
3626 {
3627 bufferlist bl;
3628 bl.append("Thbbe");
3629 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3630 }
3631 // make clean
3632 {
3633 bufferlist bl;
3634 bl.append("Thbbe");
3635 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3636 }
3637 // set-chunk (dedup)
3638 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, bb_fp_oid, "foo");
3639
3640 /*
3641 * snap[2]: [er] [hi]
3642 * snap[1]: [bb] [hi]
3643 * snap[0]: [er] [hi]
3644 * head: [bb] [hi]
3645 */
3646
3647 // check chunk's refcount
3648 {
3649 bufferlist t;
3650 SHA1 sha1_gen;
3651 int size = strlen("hi");
3652 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3653 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3654 sha1_gen.Update((const unsigned char *)"hi", size);
3655 sha1_gen.Final(fingerprint);
3656 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3657 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3658 }
3659
3660 // check chunk's refcount
3661 {
3662 bufferlist t;
3663 SHA1 sha1_gen;
3664 int size = strlen("er");
3665 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3666 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3667 sha1_gen.Update((const unsigned char *)"er", size);
3668 sha1_gen.Final(fingerprint);
3669 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3670 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3671 chunk_refs_t refs;
3672 try {
3673 auto iter = t.cbegin();
3674 decode(refs, iter);
3675 } catch (buffer::error& err) {
3676 ASSERT_TRUE(0);
3677 }
a4b75251 3678 ASSERT_LE(2u, refs.count());
f67539c2
TL		 3679 }
3680
3681 // remove snap
3682 ioctx.selfmanaged_snap_remove(my_snaps[2]);
3683
3684 /*
3685 * snap[1]: [bb] [hi]
3686 * snap[0]: [er] [hi]
3687 * head: [bb] [hi]
3688 */
3689
3690 sleep(10);
3691
3692 // check chunk's refcount
3693 {
3694 bufferlist t;
3695 SHA1 sha1_gen;
3696 int size = strlen("hi");
3697 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3698 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3699 sha1_gen.Update((const unsigned char *)"hi", size);
3700 sha1_gen.Final(fingerprint);
3701 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3702 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3703 }
3704
3705 // remove snap
3706 ioctx.selfmanaged_snap_remove(my_snaps[0]);
3707
3708 /*
3709 * snap[1]: [bb] [hi]
3710 * head: [bb] [hi]
3711 */
3712
3713 sleep(10);
3714
3715 // check chunk's refcount
3716 {
3717 bufferlist t;
3718 SHA1 sha1_gen;
3719 int size = strlen("bb");
3720 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3721 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3722 sha1_gen.Update((const unsigned char *)"bb", size);
3723 sha1_gen.Final(fingerprint);
3724 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3725 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3726 }
3727
3728 // remove snap
3729 ioctx.selfmanaged_snap_remove(my_snaps[1]);
3730
3731 /*
3732 * snap[1]: [bb] [hi]
3733 */
3734
3735 sleep(10);
3736
3737 // check chunk's refcount
3738 {
3739 bufferlist t;
3740 SHA1 sha1_gen;
3741 int size = strlen("bb");
3742 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3743 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3744 sha1_gen.Update((const unsigned char *)"bb", size);
3745 sha1_gen.Final(fingerprint);
3746 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3747 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3748 }
3749
3750 // check chunk's refcount
3751 {
3752 bufferlist t;
3753 SHA1 sha1_gen;
3754 int size = strlen("hi");
3755 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3756 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3757 sha1_gen.Update((const unsigned char *)"hi", size);
3758 sha1_gen.Final(fingerprint);
3759 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3760 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3761 }
3762}
3763
3764TEST_F(LibRadosTwoPoolsPP, ManifestSnapRefcount2) {
3765 // skip test if not yet octopus
3766 if (_get_required_osd_release(cluster) < "octopus") {
3767 cout << "cluster is not yet octopus, skipping test" << std::endl;
3768 return;
3769 }
3770
3771 bufferlist inbl;
3772 ASSERT_EQ(0, cluster.mon_command(
3773 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
3774 inbl, NULL, NULL));
3775 cluster.wait_for_latest_osdmap();
3776
3777 // create object
3778 {
3779 bufferlist bl;
3780 bl.append("Thabe cdHI");
3781 ObjectWriteOperation op;
3782 op.write_full(bl);
3783 ASSERT_EQ(0, ioctx.operate("foo", &op));
3784 }
3785 {
3786 bufferlist bl;
3787 bl.append("there hiHI");
3788 ObjectWriteOperation op;
3789 op.write_full(bl);
3790 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3791 }
3792
3793 string ab_fp_oid, cd_fp_oid, ef_fp_oid, BB_fp_oid;
3794
3795 // get fp_oid
3796 ab_fp_oid = get_fp_oid("ab", "sha1");
3797 cd_fp_oid = get_fp_oid("cd", "sha1");
3798 ef_fp_oid = get_fp_oid("ef", "sha1");
3799 BB_fp_oid = get_fp_oid("BB", "sha1");
3800
3801 // write
3802 {
3803 ObjectWriteOperation op;
3804 bufferlist bl;
3805 bl.append("ab");
3806 op.write_full(bl);
3807 ASSERT_EQ(0, cache_ioctx.operate(ab_fp_oid, &op));
3808 }
3809 // write
3810 {
3811 ObjectWriteOperation op;
3812 bufferlist bl;
3813 bl.append("cd");
3814 op.write_full(bl);
3815 ASSERT_EQ(0, cache_ioctx.operate(cd_fp_oid, &op));
3816 }
3817 // write
3818 {
3819 ObjectWriteOperation op;
3820 bufferlist bl;
3821 bl.append("ef");
3822 op.write_full(bl);
3823 ASSERT_EQ(0, cache_ioctx.operate(ef_fp_oid, &op));
3824 }
3825 // write
3826 {
3827 ObjectWriteOperation op;
3828 bufferlist bl;
3829 bl.append("BB");
3830 op.write_full(bl);
3831 ASSERT_EQ(0, cache_ioctx.operate(BB_fp_oid, &op));
3832 }
3833
3834 // set-chunk (dedup)
3835 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, ab_fp_oid, "foo");
3836 // set-chunk (dedup)
3837 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, cd_fp_oid, "foo");
3838 // set-chunk (dedup)
3839 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, ef_fp_oid, "foo");
3840
3841
3842 // make all chunks dirty --> flush
3843 // foo: [ab] [cd] [ef]
3844
3845 // create a snapshot, clone
3846 vector<uint64_t> my_snaps(1);
3847 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3848 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3849 my_snaps));
3850
3851 // foo: [BB] [BB] [ef]
3852 // create a clone
3853 {
3854 bufferlist bl;
3855 bl.append("ThBBe BB");
3856 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3857 }
3858 // make clean
3859 {
3860 bufferlist bl;
3861 bl.append("ThBBe BB");
3862 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3863 }
3864 // set-chunk (dedup)
3865 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, BB_fp_oid, "foo");
3866 // set-chunk (dedup)
3867 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, BB_fp_oid, "foo");
3868
3869 // and another
3870 my_snaps.resize(2);
3871 my_snaps[1] = my_snaps[0];
3872 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3873 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3874 my_snaps));
3875
3876 // foo: [ab] [cd] [ef]
3877 // create a clone
3878 {
3879 bufferlist bl;
3880 bl.append("Thabe cd");
3881 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3882 }
3883 // make clean
3884 {
3885 bufferlist bl;
3886 bl.append("Thabe cd");
3887 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3888 }
3889 // set-chunk (dedup)
3890 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, ab_fp_oid, "foo");
3891 // set-chunk (dedup)
3892 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, cd_fp_oid, "foo");
3893
3894 /*
3895 * snap[1]: [ab] [cd] [ef]
3896 * snap[0]: [BB] [BB] [ef]
3897 * head: [ab] [cd] [ef]
3898 */
3899
3900 // check chunk's refcount
3901 {
3902 bufferlist t;
3903 SHA1 sha1_gen;
3904 int size = strlen("ab");
3905 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3906 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3907 sha1_gen.Update((const unsigned char *)"ab", size);
3908 sha1_gen.Final(fingerprint);
3909 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3910 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3911 chunk_refs_t refs;
3912 try {
3913 auto iter = t.cbegin();
3914 decode(refs, iter);
3915 } catch (buffer::error& err) {
3916 ASSERT_TRUE(0);
3917 }
a4b75251 3918 ASSERT_LE(2u, refs.count());
f67539c2
TL		 3919 }
3920
3921 // check chunk's refcount
3922 {
3923 bufferlist t;
3924 SHA1 sha1_gen;
3925 int size = strlen("cd");
3926 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3927 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3928 sha1_gen.Update((const unsigned char *)"cd", size);
3929 sha1_gen.Final(fingerprint);
3930 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3931 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3932 chunk_refs_t refs;
3933 try {
3934 auto iter = t.cbegin();
3935 decode(refs, iter);
3936 } catch (buffer::error& err) {
3937 ASSERT_TRUE(0);
3938 }
a4b75251 3939 ASSERT_LE(2u, refs.count());
f67539c2
TL		 3940 }
3941
3942 // check chunk's refcount
3943 {
3944 bufferlist t;
3945 SHA1 sha1_gen;
3946 int size = strlen("BB");
3947 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3948 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3949 sha1_gen.Update((const unsigned char *)"BB", size);
3950 sha1_gen.Final(fingerprint);
3951 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3952 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3953 chunk_refs_t refs;
3954 try {
3955 auto iter = t.cbegin();
3956 decode(refs, iter);
3957 } catch (buffer::error& err) {
3958 ASSERT_TRUE(0);
3959 }
a4b75251 3960 ASSERT_LE(2u, refs.count());
f67539c2
TL		 3961 }
3962
3963 // remove snap
3964 ioctx.selfmanaged_snap_remove(my_snaps[0]);
3965
3966 /*
3967 * snap[1]: [ab] [cd] [ef]
3968 * head: [ab] [cd] [ef]
3969 */
3970
3971 sleep(10);
3972
3973 // check chunk's refcount
3974 {
3975 bufferlist t;
3976 SHA1 sha1_gen;
3977 int size = strlen("BB");
3978 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3979 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3980 sha1_gen.Update((const unsigned char *)"BB", size);
3981 sha1_gen.Final(fingerprint);
3982 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3983 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
3984 }
3985}
3986
3987TEST_F(LibRadosTwoPoolsPP, ManifestTestSnapCreate) {
3988 // skip test if not yet octopus
3989 if (_get_required_osd_release(cluster) < "octopus") {
3990 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
3991 }
3992
3993 // create object
3994 {
3995 bufferlist bl;
3996 bl.append("base chunk");
3997 ObjectWriteOperation op;
3998 op.write_full(bl);
3999 ASSERT_EQ(0, ioctx.operate("foo", &op));
4000 }
4001 {
4002 bufferlist bl;
4003 bl.append("CHUNKS CHUNKS");
4004 ObjectWriteOperation op;
4005 op.write_full(bl);
4006 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
4007 }
4008
4009 string ba_fp_oid, se_fp_oid, ch_fp_oid;
4010
4011 // get fp_oid
4012 ba_fp_oid = get_fp_oid("ba", "sha1");
4013 se_fp_oid = get_fp_oid("se", "sha1");
4014 ch_fp_oid = get_fp_oid("ch", "sha1");
4015
4016 // write
4017 {
4018 ObjectWriteOperation op;
4019 bufferlist bl;
4020 bl.append("ba");
4021 op.write_full(bl);
4022 ASSERT_EQ(0, cache_ioctx.operate(ba_fp_oid, &op));
4023 }
4024 // write
4025 {
4026 ObjectWriteOperation op;
4027 bufferlist bl;
4028 bl.append("se");
4029 op.write_full(bl);
4030 ASSERT_EQ(0, cache_ioctx.operate(se_fp_oid, &op));
4031 }
4032 // write
4033 {
4034 ObjectWriteOperation op;
4035 bufferlist bl;
4036 bl.append("ch");
4037 op.write_full(bl);
4038 ASSERT_EQ(0, cache_ioctx.operate(ch_fp_oid, &op));
4039 }
4040
4041 // set-chunk (dedup)
4042 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, ba_fp_oid, "foo");
4043
4044 // try to create a snapshot, clone
4045 vector<uint64_t> my_snaps(1);
4046 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4047 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4048 my_snaps));
4049
4050 // set-chunk (dedup)
4051 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, se_fp_oid, "foo");
4052
4053 // check whether clone is created
4054 ioctx.snap_set_read(librados::SNAP_DIR);
4055 {
4056 snap_set_t snap_set;
4057 int snap_ret;
4058 ObjectReadOperation op;
4059 op.list_snaps(&snap_set, &snap_ret);
4060 librados::AioCompletion *completion = cluster.aio_create_completion();
4061 ASSERT_EQ(0, ioctx.aio_operate(
4062 "foo", completion, &op,
4063 0, NULL));
4064 completion->wait_for_complete();
4065 ASSERT_EQ(0, snap_ret);
4066 ASSERT_LT(0u, snap_set.clones.size());
4067 ASSERT_EQ(1, snap_set.clones.size());
4068 }
4069
4070 // create a clone
4071 ioctx.snap_set_read(librados::SNAP_HEAD);
4072 {
4073 bufferlist bl;
4074 bl.append("B");
4075 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 0));
4076 }
4077
4078 ioctx.snap_set_read(my_snaps[0]);
4079 // set-chunk to clone
4080 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ch_fp_oid, "foo");
4081}
4082
4083TEST_F(LibRadosTwoPoolsPP, ManifestRedirectAfterPromote) {
4084 // skip test if not yet octopus
4085 if (_get_required_osd_release(cluster) < "octopus") {
4086 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4087 }
4088
4089 // create object
4090 {
4091 bufferlist bl;
4092 bl.append("base chunk");
4093 ObjectWriteOperation op;
4094 op.write_full(bl);
4095 ASSERT_EQ(0, ioctx.operate("foo", &op));
4096 }
4097 {
4098 bufferlist bl;
4099 bl.append("BASE CHUNK");
4100 ObjectWriteOperation op;
4101 op.write_full(bl);
4102 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
4103 }
4104
4105 // set-redirect
4106 {
4107 ObjectWriteOperation op;
4108 op.set_redirect("bar", cache_ioctx, 0);
4109 librados::AioCompletion *completion = cluster.aio_create_completion();
4110 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
4111 completion->wait_for_complete();
4112 ASSERT_EQ(0, completion->get_return_value());
4113 completion->release();
4114 }
4115
4116 // promote
4117 {
4118 ObjectWriteOperation op;
4119 op.tier_promote();
4120 librados::AioCompletion *completion = cluster.aio_create_completion();
4121 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
4122 completion->wait_for_complete();
4123 ASSERT_EQ(0, completion->get_return_value());
4124 completion->release();
4125 }
4126
4127 // write
4128 {
4129 bufferlist bl;
4130 bl.append("a");
4131 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 0));
4132 }
4133
4134 // read and verify the object (redirect)
4135 {
4136 bufferlist bl;
4137 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4138 ASSERT_EQ('a', bl[0]);
4139 }
4140
4141 // read and verify the object (redirect)
4142 {
4143 bufferlist bl;
4144 ASSERT_EQ(1, cache_ioctx.read("bar", bl, 1, 0));
4145 ASSERT_EQ('B', bl[0]);
4146 }
4147}
4148
4149TEST_F(LibRadosTwoPoolsPP, ManifestCheckRefcountWhenModification) {
4150 // skip test if not yet octopus
4151 if (_get_required_osd_release(cluster) < "octopus") {
4152 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4153 }
4154
4155 bufferlist inbl;
4156 ASSERT_EQ(0, cluster.mon_command(
4157 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
4158 inbl, NULL, NULL));
4159 cluster.wait_for_latest_osdmap();
4160
4161 // create object
4162 {
4163 bufferlist bl;
4164 bl.append("there hiHI");
4165 ObjectWriteOperation op;
4166 op.write_full(bl);
4167 ASSERT_EQ(0, ioctx.operate("foo", &op));
4168 }
4169
4170 string er_fp_oid, hi_fp_oid, HI_fp_oid, ai_fp_oid, bi_fp_oid,
4171 Er_fp_oid, Hi_fp_oid, Si_fp_oid;
4172
4173 // get fp_oid
4174 er_fp_oid = get_fp_oid("er", "sha1");
4175 hi_fp_oid = get_fp_oid("hi", "sha1");
4176 HI_fp_oid = get_fp_oid("HI", "sha1");
4177 ai_fp_oid = get_fp_oid("ai", "sha1");
4178 bi_fp_oid = get_fp_oid("bi", "sha1");
4179 Er_fp_oid = get_fp_oid("Er", "sha1");
4180 Hi_fp_oid = get_fp_oid("Hi", "sha1");
4181 Si_fp_oid = get_fp_oid("Si", "sha1");
4182
4183 // write
4184 {
4185 ObjectWriteOperation op;
4186 bufferlist bl;
4187 bl.append("er");
4188 op.write_full(bl);
4189 ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
4190 }
4191 // write
4192 {
4193 ObjectWriteOperation op;
4194 bufferlist bl;
4195 bl.append("hi");
4196 op.write_full(bl);
4197 ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
4198 }
4199 // write
4200 {
4201 ObjectWriteOperation op;
4202 bufferlist bl;
4203 bl.append("HI");
4204 op.write_full(bl);
4205 ASSERT_EQ(0, cache_ioctx.operate(HI_fp_oid, &op));
4206 }
4207 // write
4208 {
4209 ObjectWriteOperation op;
4210 bufferlist bl;
4211 bl.append("ai");
4212 op.write_full(bl);
4213 ASSERT_EQ(0, cache_ioctx.operate(ai_fp_oid, &op));
4214 }
4215 // write
4216 {
4217 ObjectWriteOperation op;
4218 bufferlist bl;
4219 bl.append("bi");
4220 op.write_full(bl);
4221 ASSERT_EQ(0, cache_ioctx.operate(bi_fp_oid, &op));
4222 }
4223 // write
4224 {
4225 ObjectWriteOperation op;
4226 bufferlist bl;
4227 bl.append("Er");
4228 op.write_full(bl);
4229 ASSERT_EQ(0, cache_ioctx.operate(Er_fp_oid, &op));
4230 }
4231 // write
4232 {
4233 ObjectWriteOperation op;
4234 bufferlist bl;
4235 bl.append("Hi");
4236 op.write_full(bl);
4237 ASSERT_EQ(0, cache_ioctx.operate(Hi_fp_oid, &op));
4238 }
4239 // write
4240 {
4241 ObjectWriteOperation op;
4242 bufferlist bl;
4243 bl.append("Si");
4244 op.write_full(bl);
4245 ASSERT_EQ(0, cache_ioctx.operate(Si_fp_oid, &op));
4246 }
4247
4248 // set-chunk (dedup)
4249 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
4250 // set-chunk (dedup)
4251 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, hi_fp_oid, "foo");
4252 // set-chunk (dedup)
4253 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, HI_fp_oid, "foo");
4254
4255 // foo head: [er] [hi] [HI]
4256
4257 // create a snapshot, clone
4258 vector<uint64_t> my_snaps(1);
4259 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4260 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4261 my_snaps));
4262
4263
4264 // foo snap[0]: [er] [hi] [HI]
4265 // foo head : [er] [ai] [HI]
4266 // create a clone
4267 {
4268 bufferlist bl;
4269 bl.append("a");
4270 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
4271 }
4272 // write
4273 {
4274 bufferlist bl;
4275 bl.append("a");
4276 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
4277 }
4278
4279 // set-chunk (dedup)
4280 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ai_fp_oid, "foo");
4281
4282 // foo snap[0]: [er] [hi] [HI]
4283 // foo head : [er] [bi] [HI]
4284 // create a clone
4285 {
4286 bufferlist bl;
4287 bl.append("b");
4288 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
4289 }
4290 // write
4291 {
4292 bufferlist bl;
4293 bl.append("b");
4294 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
4295 }
4296
4297 // set-chunk (dedup)
4298 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, bi_fp_oid, "foo");
4299
4300 sleep(10);
4301
4302 // check chunk's refcount
4303 // [ai]'s refcount should be 0
4304 {
4305 bufferlist t;
4306 SHA1 sha1_gen;
4307 int size = strlen("ai");
4308 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
4309 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4310 sha1_gen.Update((const unsigned char *)"ai", size);
4311 sha1_gen.Final(fingerprint);
4312 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4313 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
4314 }
4315
4316 // foo snap[0]: [er] [hi] [HI]
4317 // foo head : [Er] [Hi] [Si]
4318 // create a clone
4319 {
4320 bufferlist bl;
4321 bl.append("thEre HiSi");
4322 ObjectWriteOperation op;
4323 op.write_full(bl);
4324 ASSERT_EQ(0, ioctx.operate("foo", &op));
4325 }
4326 // write
4327 {
4328 bufferlist bl;
4329 bl.append("thEre HiSi");
4330 ObjectWriteOperation op;
4331 op.write_full(bl);
4332 ASSERT_EQ(0, ioctx.operate("foo", &op));
4333 }
4334
4335 // set-chunk (dedup)
4336 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, Er_fp_oid, "foo");
4337 // set-chunk (dedup)
4338 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, Hi_fp_oid, "foo");
4339 // set-chunk (dedup)
4340 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, Si_fp_oid, "foo");
4341
4342 // foo snap[0]: [er] [hi] [HI]
4343 // foo head : [ER] [HI] [SI]
4344 // write
4345 {
4346 bufferlist bl;
4347 bl.append("thERe HISI");
4348 ObjectWriteOperation op;
4349 op.write_full(bl);
4350 ASSERT_EQ(0, ioctx.operate("foo", &op));
4351 }
4352
4353 sleep(10);
4354
4355 // check chunk's refcount
4356 // [Er]'s refcount should be 0
4357 {
4358 bufferlist t;
4359 SHA1 sha1_gen;
4360 int size = strlen("Er");
4361 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
4362 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4363 sha1_gen.Update((const unsigned char *)"Er", size);
4364 sha1_gen.Final(fingerprint);
4365 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4366 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
4367 }
4368}
4369
4370TEST_F(LibRadosTwoPoolsPP, ManifestSnapIncCount) {
4371 // skip test if not yet octopus
4372 if (_get_required_osd_release(cluster) < "octopus") {
4373 cout << "cluster is not yet octopus, skipping test" << std::endl;
4374 return;
4375 }
4376
4377 // create object
4378 {
4379 bufferlist bl;
4380 bl.append("there hiHI");
4381 ObjectWriteOperation op;
4382 op.write_full(bl);
4383 ASSERT_EQ(0, ioctx.operate("foo", &op));
4384 }
4385 {
4386 bufferlist bl;
4387 bl.append("there hiHI");
4388 ObjectWriteOperation op;
4389 op.write_full(bl);
4390 ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
4391 }
4392 {
4393 bufferlist bl;
4394 bl.append("there hiHI");
4395 ObjectWriteOperation op;
4396 op.write_full(bl);
4397 ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
4398 }
4399 {
4400 bufferlist bl;
4401 bl.append("there hiHI");
4402 ObjectWriteOperation op;
4403 op.write_full(bl);
4404 ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
4405 }
4406 {
4407 bufferlist bl;
4408 bl.append("there hiHI");
4409 ObjectWriteOperation op;
4410 op.write_full(bl);
4411 ASSERT_EQ(0, cache_ioctx.operate("chunk4", &op));
4412 }
4413
4414 // wait for maps to settle
4415 cluster.wait_for_latest_osdmap();
4416
4417 // create a snapshot, clone
4418 vector<uint64_t> my_snaps(1);
4419 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4420 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4421 my_snaps));
4422
4423 {
4424 bufferlist bl;
4425 bl.append("there hiHI");
4426 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4427 }
4428
4429 my_snaps.resize(2);
4430 my_snaps[1] = my_snaps[0];
4431 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4432 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4433 my_snaps));
4434
4435 {
4436 bufferlist bl;
4437 bl.append("there hiHI");
4438 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4439 }
4440
4441 // set-chunk
4442 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
4443 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4444 // foo snap[1]:
4445 // foo snap[0]:
4446 // foo head : [chunk1] [chunk4]
4447
4448 ioctx.snap_set_read(my_snaps[1]);
4449 // set-chunk
4450 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
4451 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4452 // foo snap[1]: [chunk2] [chunk4]
4453 // foo snap[0]:
4454 // foo head : [chunk1] [chunk4]
4455
4456 ioctx.snap_set_read(my_snaps[0]);
4457 // set-chunk
4458 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
4459 // foo snap[1]: [chunk2] [chunk4]
4460 // foo snap[0]: [chunk2]
4461 // foo head : [chunk1] [chunk4]
4462
4463 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk3", "foo");
4464 // foo snap[1]: [chunk2] [chunk4]
4465 // foo snap[0]: [chunk3] [chunk2]
4466 // foo head : [chunk1] [chunk4]
4467 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4468 // foo snap[1]: [chunk2] [chunk4]
4469 // foo snap[0]: [chunk3] [chunk2] [chunk4]
4470 // foo head : [chunk1] [chunk4]
4471
4472 // check chunk's refcount
4473 check_fp_oid_refcount(cache_ioctx, "chunk1", 1u, "");
4474
4475 // check chunk's refcount
4476 check_fp_oid_refcount(cache_ioctx, "chunk2", 1u, "");
4477
4478 // check chunk's refcount
4479 check_fp_oid_refcount(cache_ioctx, "chunk3", 1u, "");
4480 sleep(10);
4481
4482 // check chunk's refcount
4483 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk4", 1);
4484}
4485
4486TEST_F(LibRadosTwoPoolsPP, ManifestEvict) {
4487 // skip test if not yet octopus
4488 if (_get_required_osd_release(cluster) < "octopus") {
4489 cout << "cluster is not yet octopus, skipping test" << std::endl;
4490 return;
4491 }
4492
4493 // create object
4494 {
4495 bufferlist bl;
4496 bl.append("there hiHI");
4497 ObjectWriteOperation op;
4498 op.write_full(bl);
4499 ASSERT_EQ(0, ioctx.operate("foo", &op));
4500 }
4501 {
4502 bufferlist bl;
4503 bl.append("there hiHI");
4504 ObjectWriteOperation op;
4505 op.write_full(bl);
4506 ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
4507 }
4508 {
4509 bufferlist bl;
4510 bl.append("there hiHI");
4511 ObjectWriteOperation op;
4512 op.write_full(bl);
4513 ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
4514 }
4515 {
4516 bufferlist bl;
4517 bl.append("there hiHI");
4518 ObjectWriteOperation op;
4519 op.write_full(bl);
4520 ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
4521 }
4522 {
4523 bufferlist bl;
4524 bl.append("there hiHI");
4525 ObjectWriteOperation op;
4526 op.write_full(bl);
4527 ASSERT_EQ(0, cache_ioctx.operate("chunk4", &op));
4528 }
4529
4530 // wait for maps to settle
4531 cluster.wait_for_latest_osdmap();
4532
4533 // create a snapshot, clone
4534 vector<uint64_t> my_snaps(1);
4535 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4536 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4537 my_snaps));
4538
4539 {
4540 bufferlist bl;
4541 bl.append("there hiHI");
4542 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4543 }
4544
4545 my_snaps.resize(2);
4546 my_snaps[1] = my_snaps[0];
4547 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4548 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4549 my_snaps));
4550
4551 {
4552 bufferlist bl;
4553 bl.append("there hiHI");
4554 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4555 }
4556
4557 // set-chunk
4558 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
4559 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4560 // foo snap[1]:
4561 // foo snap[0]:
4562 // foo head : [chunk1] [chunk4]
4563
4564 ioctx.snap_set_read(my_snaps[1]);
4565 // set-chunk
4566 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
4567 // foo snap[1]: [ chunk2 ]
4568 // foo snap[0]:
4569 // foo head : [chunk1] [chunk4]
4570
4571 ioctx.snap_set_read(my_snaps[0]);
4572 // set-chunk
4573 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
4574 // foo snap[1]: [ chunk2 ]
4575 // foo snap[0]: [chunk2]
4576 // foo head : [chunk1] [chunk4]
4577
4578 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk3", "foo");
4579 // foo snap[1]: [ chunk2 ]
4580 // foo snap[0]: [chunk3] [chunk2]
4581 // foo head : [chunk1] [chunk4]
4582 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4583 // foo snap[1]: [ chunk2 ]
4584 // foo snap[0]: [chunk3] [chunk2] [chunk4]
4585 // foo head : [chunk1] [chunk4]
4586 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, "chunk4", "foo");
4587 // foo snap[1]: [ chunk2 ]
4588 // foo snap[0]: [chunk4] [chunk3] [chunk2] [chunk4]
4589 // foo head : [chunk1] [chunk4]
4590 manifest_set_chunk(cluster, cache_ioctx, ioctx, 4, 2, "chunk1", "foo");
4591 // foo snap[1]: [ chunk2 ]
4592 // foo snap[0]: [chunk4] [chunk3] [chunk1] [chunk2] [chunk4]
4593 // foo head : [chunk1] [chunk4]
4594
4595 {
4596 ObjectReadOperation op, stat_op;
4597 uint64_t size;
4598 op.tier_evict();
4599 librados::AioCompletion *completion = cluster.aio_create_completion();
4600 ASSERT_EQ(0, ioctx.aio_operate(
4601 "foo", completion, &op,
4602 librados::OPERATION_IGNORE_OVERLAY, NULL));
4603 completion->wait_for_complete();
4604 ASSERT_EQ(0, completion->get_return_value());
4605
4606 stat_op.stat(&size, NULL, NULL);
4607 ASSERT_EQ(0, ioctx.operate("foo", &stat_op, NULL));
4608 ASSERT_EQ(10, size);
4609 }
4610
4611 ioctx.snap_set_read(librados::SNAP_HEAD);
4612 {
4613 ObjectReadOperation op, stat_op;
4614 uint64_t size;
4615 op.tier_evict();
4616 librados::AioCompletion *completion = cluster.aio_create_completion();
4617 ASSERT_EQ(0, ioctx.aio_operate(
4618 "foo", completion, &op,
4619 librados::OPERATION_IGNORE_OVERLAY, NULL));
4620 completion->wait_for_complete();
4621 ASSERT_EQ(0, completion->get_return_value());
4622
4623 stat_op.stat(&size, NULL, NULL);
4624 ASSERT_EQ(0, ioctx.operate("foo", &stat_op, NULL));
4625 ASSERT_EQ(strlen("there hiHI"), size);
4626 }
4627
4628}
4629
4630
4631TEST_F(LibRadosTwoPoolsPP, ManifestSnapSizeMismatch) {
4632 // skip test if not yet octopus
4633 if (_get_required_osd_release(cluster) < "octopus") {
4634 cout << "cluster is not yet octopus, skipping test" << std::endl;
4635 return;
4636 }
4637
4638 // create object
4639 {
4640 bufferlist bl;
4641 bl.append("there hiHI");
4642 ObjectWriteOperation op;
4643 op.write_full(bl);
4644 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
4645 }
4646 {
4647 bufferlist bl;
4648 bl.append("there hiHI");
4649 ObjectWriteOperation op;
4650 op.write_full(bl);
4651 ASSERT_EQ(0, ioctx.operate("chunk1", &op));
4652 }
4653 {
4654 bufferlist bl;
4655 bl.append("there HIHI");
4656 ObjectWriteOperation op;
4657 op.write_full(bl);
4658 ASSERT_EQ(0, ioctx.operate("chunk2", &op));
4659 }
4660
4661 // wait for maps to settle
4662 cluster.wait_for_latest_osdmap();
4663
4664 // create a snapshot, clone
4665 vector<uint64_t> my_snaps(1);
4666 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
4667 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4668 my_snaps));
4669
4670 {
4671 bufferlist bl;
4672 bl.append("There hiHI");
4673 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
4674 }
4675
4676 my_snaps.resize(2);
4677 my_snaps[1] = my_snaps[0];
4678 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
4679 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4680 my_snaps));
4681
4682 {
4683 bufferlist bl;
4684 bl.append("tHere hiHI");
4685 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
4686 }
4687
4688 // set-chunk
4689 manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "chunk1", "foo");
4690
4691 cache_ioctx.snap_set_read(my_snaps[1]);
4692
4693 // set-chunk
4694 manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "chunk2", "foo");
4695
4696 // evict
4697 {
4698 ObjectReadOperation op, stat_op;
4699 op.tier_evict();
4700 librados::AioCompletion *completion = cluster.aio_create_completion();
4701 ASSERT_EQ(0, cache_ioctx.aio_operate(
4702 "foo", completion, &op,
4703 librados::OPERATION_IGNORE_OVERLAY, NULL));
4704 completion->wait_for_complete();
4705 ASSERT_EQ(0, completion->get_return_value());
4706 }
4707
4708 uint32_t hash;
4709 ASSERT_EQ(0, cache_ioctx.get_object_pg_hash_position2("foo", &hash));
4710
4711 // scrub
4712 {
4713 for (int tries = 0; tries < 5; ++tries) {
4714 bufferlist inbl;
4715 ostringstream ss;
4716 ss << "{\"prefix\": \"pg deep-scrub\", \"pgid\": \""
4717 << cache_ioctx.get_id() << "."
4718 << std::hex << hash
4719 << "\"}";
4720 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
4721 if (r == -ENOENT ||
4722 r == -EAGAIN) {
4723 sleep(5);
4724 continue;
4725 }
4726 ASSERT_EQ(0, r);
4727 break;
4728 }
4729 cout << "waiting for scrubs..." << std::endl;
4730 sleep(20);
4731 cout << "done waiting" << std::endl;
4732 }
4733
4734 {
4735 bufferlist bl;
4736 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
4737 ASSERT_EQ('t', bl[0]);
4738 }
4739}
4740
4741#include <common/CDC.h>
4742TEST_F(LibRadosTwoPoolsPP, DedupFlushRead) {
4743 // skip test if not yet octopus
4744 if (_get_required_osd_release(cluster) < "octopus") {
4745 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4746 }
4747
4748 bufferlist inbl;
4749 ASSERT_EQ(0, cluster.mon_command(
4750 set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
4751 inbl, NULL, NULL));
4752 ASSERT_EQ(0, cluster.mon_command(
4753 set_pool_str(cache_pool_name, "dedup_tier", pool_name),
4754 inbl, NULL, NULL));
4755 ASSERT_EQ(0, cluster.mon_command(
4756 set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
4757 inbl, NULL, NULL));
4758 ASSERT_EQ(0, cluster.mon_command(
4759 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
4760 inbl, NULL, NULL));
4761
4762 // wait for maps to settle
4763 cluster.wait_for_latest_osdmap();
4764
4765 // create object
4766 bufferlist gbl;
4767 {
4768 generate_buffer(1024*8, &gbl);
4769 ObjectWriteOperation op;
4770 op.write_full(gbl);
4771 ASSERT_EQ(0, cache_ioctx.operate("foo-chunk", &op));
4772 }
4773 {
4774 bufferlist bl;
4775 bl.append("DDse chunk");
4776 ObjectWriteOperation op;
4777 op.write_full(bl);
4778 ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
4779 }
4780
4781 // set-chunk to set manifest object
4782 {
4783 ObjectReadOperation op;
4784 op.set_chunk(0, 2, ioctx, "bar-chunk", 0,
4785 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
4786 librados::AioCompletion *completion = cluster.aio_create_completion();
4787 ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op,
4788 librados::OPERATION_IGNORE_CACHE, NULL));
4789 completion->wait_for_complete();
4790 ASSERT_EQ(0, completion->get_return_value());
4791 completion->release();
4792 }
4793 // flush
4794 {
4795 ObjectReadOperation op;
4796 op.tier_flush();
4797 librados::AioCompletion *completion = cluster.aio_create_completion();
4798 ASSERT_EQ(0, cache_ioctx.aio_operate(
4799 "foo-chunk", completion, &op,
4800 librados::OPERATION_IGNORE_CACHE, NULL));
4801 completion->wait_for_complete();
4802 ASSERT_EQ(0, completion->get_return_value());
4803 completion->release();
4804 }
4805
4806 std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
4807 vector<pair<uint64_t, uint64_t>> chunks;
4808 bufferlist chunk;
4809 cdc->calc_chunks(gbl, &chunks);
4810 chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
4811 string tgt_oid;
4812 {
4813 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
4814 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4815 SHA1 sha1_gen;
4816 int size = chunk.length();
4817 sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
4818 sha1_gen.Final(fingerprint);
4819 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4820 tgt_oid = string(p_str);
4821 }
4822
4823 // read and verify the chunked object
4824 {
4825 bufferlist test_bl;
4826 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
4827 ASSERT_EQ(test_bl[1], chunk[1]);
4828 }
4829
4830 ASSERT_EQ(0, cluster.mon_command(
4831 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 512),
4832 inbl, NULL, NULL));
4833 cluster.wait_for_latest_osdmap();
4834
4835 // make a dirty chunks
4836 {
4837 bufferlist bl;
4838 bl.append("hi");
4839 ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
4840 }
4841
4842 // flush
4843 {
4844 ObjectReadOperation op;
4845 op.tier_flush();
4846 librados::AioCompletion *completion = cluster.aio_create_completion();
4847 ASSERT_EQ(0, cache_ioctx.aio_operate(
4848 "foo-chunk", completion, &op,
4849 librados::OPERATION_IGNORE_CACHE, NULL));
4850 completion->wait_for_complete();
4851 ASSERT_EQ(0, completion->get_return_value());
4852 completion->release();
4853 }
4854
4855 cdc = CDC::create("fastcdc", cbits(512)-1);
4856 chunks.clear();
4857 cdc->calc_chunks(gbl, &chunks);
4858 bufferlist chunk_512;
4859 chunk_512.substr_of(gbl, chunks[3].first, chunks[3].second);
4860 {
4861 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
4862 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4863 SHA1 sha1_gen;
4864 int size = chunk_512.length();
4865 sha1_gen.Update((const unsigned char *)chunk_512.c_str(), size);
4866 sha1_gen.Final(fingerprint);
4867 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4868 tgt_oid = string(p_str);
4869 }
4870
4871 // read and verify the chunked object
4872 {
4873 bufferlist test_bl;
4874 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
4875 ASSERT_EQ(test_bl[1], chunk_512[1]);
4876 }
4877
4878 ASSERT_EQ(0, cluster.mon_command(
4879 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 16384),
4880 inbl, NULL, NULL));
4881 cluster.wait_for_latest_osdmap();
4882
4883 // make a dirty chunks
4884 {
4885 bufferlist bl;
4886 bl.append("hi");
4887 ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
4888 gbl.begin(0).copy_in(bl.length(), bl);
4889 }
4890 // flush
4891 {
4892 ObjectReadOperation op;
4893 op.tier_flush();
4894 librados::AioCompletion *completion = cluster.aio_create_completion();
4895 ASSERT_EQ(0, cache_ioctx.aio_operate(
4896 "foo-chunk", completion, &op,
4897 librados::OPERATION_IGNORE_CACHE, NULL));
4898 completion->wait_for_complete();
4899 ASSERT_EQ(0, completion->get_return_value());
4900 completion->release();
4901 }
4902
4903 cdc = CDC::create("fastcdc", cbits(16384)-1);
4904 chunks.clear();
4905 cdc->calc_chunks(gbl, &chunks);
4906 bufferlist chunk_16384;
4907 chunk_16384.substr_of(gbl, chunks[0].first, chunks[0].second);
4908 {
4909 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
4910 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4911 SHA1 sha1_gen;
4912 int size = chunk_16384.length();
4913 sha1_gen.Update((const unsigned char *)chunk_16384.c_str(), size);
4914 sha1_gen.Final(fingerprint);
4915 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4916 tgt_oid = string(p_str);
4917 }
4918 // read and verify the chunked object
4919 {
4920 bufferlist test_bl;
4921 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
4922 ASSERT_EQ(test_bl[0], chunk_16384[0]);
4923 }
4924
4925 // less than object size
4926 ASSERT_EQ(0, cluster.mon_command(
4927 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
4928 inbl, NULL, NULL));
4929 cluster.wait_for_latest_osdmap();
4930
4931 // make a dirty chunks
4932 // a chunk_info is deleted by write, which converts the manifest object to non-manifest object
4933 {
4934 bufferlist bl;
4935 bl.append("hi");
4936 ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
4937 }
4938
4939 // reset set-chunk
4940 {
4941 bufferlist bl;
4942 bl.append("DDse chunk");
4943 ObjectWriteOperation op;
4944 op.write_full(bl);
4945 ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
4946 }
4947 // set-chunk to set manifest object
4948 {
4949 ObjectReadOperation op;
4950 op.set_chunk(0, 2, ioctx, "bar-chunk", 0,
4951 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
4952 librados::AioCompletion *completion = cluster.aio_create_completion();
4953 ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op,
4954 librados::OPERATION_IGNORE_CACHE, NULL));
4955 completion->wait_for_complete();
4956 ASSERT_EQ(0, completion->get_return_value());
4957 completion->release();
4958 }
4959 // flush
4960 {
4961 ObjectReadOperation op;
4962 op.tier_flush();
4963 librados::AioCompletion *completion = cluster.aio_create_completion();
4964 ASSERT_EQ(0, cache_ioctx.aio_operate(
4965 "foo-chunk", completion, &op,
4966 librados::OPERATION_IGNORE_CACHE, NULL));
4967 completion->wait_for_complete();
4968 ASSERT_EQ(0, completion->get_return_value());
4969 completion->release();
4970 }
4971
4972 cdc = CDC::create("fastcdc", cbits(1024)-1);
4973 chunks.clear();
4974 cdc->calc_chunks(gbl, &chunks);
4975 bufferlist small_chunk;
4976 small_chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
4977 {
4978 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
4979 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4980 SHA1 sha1_gen;
4981 int size = small_chunk.length();
4982 sha1_gen.Update((const unsigned char *)small_chunk.c_str(), size);
4983 sha1_gen.Final(fingerprint);
4984 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4985 tgt_oid = string(p_str);
4986 }
4987 // read and verify the chunked object
4988 {
4989 bufferlist test_bl;
4990 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
4991 ASSERT_EQ(test_bl[0], small_chunk[0]);
4992 }
4993
4994}
4995
4996TEST_F(LibRadosTwoPoolsPP, ManifestFlushSnap) {
4997 // skip test if not yet octopus
4998 if (_get_required_osd_release(cluster) < "octopus") {
4999 cout << "cluster is not yet octopus, skipping test" << std::endl;
5000 return;
5001 }
5002
5003 bufferlist inbl;
5004 ASSERT_EQ(0, cluster.mon_command(
5005 set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
5006 inbl, NULL, NULL));
5007 ASSERT_EQ(0, cluster.mon_command(
5008 set_pool_str(cache_pool_name, "dedup_tier", pool_name),
5009 inbl, NULL, NULL));
5010 ASSERT_EQ(0, cluster.mon_command(
5011 set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
5012 inbl, NULL, NULL));
5013 ASSERT_EQ(0, cluster.mon_command(
5014 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
5015 inbl, NULL, NULL));
5016
5017 // wait for maps to settle
5018 cluster.wait_for_latest_osdmap();
5019
5020 // create object
5021 bufferlist gbl;
5022 {
5023 //bufferlist bl;
5024 //bl.append("there hi");
5025 generate_buffer(1024*8, &gbl);
5026 ObjectWriteOperation op;
5027 op.write_full(gbl);
5028 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
5029 }
5030 {
5031 bufferlist bl;
5032 bl.append("there hi");
5033 ObjectWriteOperation op;
5034 op.write_full(bl);
5035 ASSERT_EQ(0, ioctx.operate("bar", &op));
5036 }
5037
5038 // set-chunk (dedup)
5039 manifest_set_chunk(cluster, ioctx, cache_ioctx, 2, 2, "bar", "foo");
5040
5041 // create a snapshot, clone
5042 vector<uint64_t> my_snaps(1);
5043 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
5044 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5045 my_snaps));
5046
5047 // make a dirty chunks
5048 {
5049 bufferlist bl;
5050 bl.append("Thbbe");
5051 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5052 }
5053
5054 // and another
5055 my_snaps.resize(2);
5056 my_snaps[1] = my_snaps[0];
5057 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
5058 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5059 my_snaps));
5060
5061 // make a dirty chunks
5062 {
5063 bufferlist bl;
5064 bl.append("Thcce");
5065 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5066 }
5067
5068 // flush on head (should fail)
5069 cache_ioctx.snap_set_read(librados::SNAP_HEAD);
5070 // flush
5071 {
5072 ObjectReadOperation op;
5073 op.tier_flush();
5074 librados::AioCompletion *completion = cluster.aio_create_completion();
5075 ASSERT_EQ(0, cache_ioctx.aio_operate(
5076 "foo", completion, &op,
5077 librados::OPERATION_IGNORE_CACHE, NULL));
5078 completion->wait_for_complete();
5079 ASSERT_EQ(-EBUSY, completion->get_return_value());
5080 completion->release();
5081 }
5082
5083 // flush on recent snap (should fail)
5084 cache_ioctx.snap_set_read(my_snaps[0]);
5085 {
5086 ObjectReadOperation op;
5087 op.tier_flush();
5088 librados::AioCompletion *completion = cluster.aio_create_completion();
5089 ASSERT_EQ(0, cache_ioctx.aio_operate(
5090 "foo", completion, &op,
5091 librados::OPERATION_IGNORE_CACHE, NULL));
5092 completion->wait_for_complete();
5093 ASSERT_EQ(-EBUSY, completion->get_return_value());
5094 completion->release();
5095 }
5096
5097 // flush on oldest snap
5098 cache_ioctx.snap_set_read(my_snaps[1]);
5099 {
5100 ObjectReadOperation op;
5101 op.tier_flush();
5102 librados::AioCompletion *completion = cluster.aio_create_completion();
5103 ASSERT_EQ(0, cache_ioctx.aio_operate(
5104 "foo", completion, &op,
5105 librados::OPERATION_IGNORE_CACHE, NULL));
5106 completion->wait_for_complete();
5107 ASSERT_EQ(0, completion->get_return_value());
5108 completion->release();
5109 }
5110
5111 // flush on oldest snap
5112 cache_ioctx.snap_set_read(my_snaps[0]);
5113 {
5114 ObjectReadOperation op;
5115 op.tier_flush();
5116 librados::AioCompletion *completion = cluster.aio_create_completion();
5117 ASSERT_EQ(0, cache_ioctx.aio_operate(
5118 "foo", completion, &op,
5119 librados::OPERATION_IGNORE_CACHE, NULL));
5120 completion->wait_for_complete();
5121 ASSERT_EQ(0, completion->get_return_value());
5122 completion->release();
5123 }
5124
5125 // flush on oldest snap
5126 cache_ioctx.snap_set_read(librados::SNAP_HEAD);
5127 {
5128 ObjectReadOperation op;
5129 op.tier_flush();
5130 librados::AioCompletion *completion = cluster.aio_create_completion();
5131 ASSERT_EQ(0, cache_ioctx.aio_operate(
5132 "foo", completion, &op,
5133 librados::OPERATION_IGNORE_CACHE, NULL));
5134 completion->wait_for_complete();
5135 ASSERT_EQ(0, completion->get_return_value());
5136 completion->release();
5137 }
5138
5139 // check chunk's refcount
5140 std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
5141 vector<pair<uint64_t, uint64_t>> chunks;
5142 bufferlist chunk;
5143 cdc->calc_chunks(gbl, &chunks);
5144 chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
5145 string tgt_oid;
5146 {
5147 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5148 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5149 SHA1 sha1_gen;
5150 int size = chunk.length();
5151 sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
5152 sha1_gen.Final(fingerprint);
5153 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5154 tgt_oid = string(p_str);
5155 }
5156 // read and verify the chunked object
5157 {
5158 bufferlist test_bl;
5159 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
5160 ASSERT_EQ(test_bl[1], chunk[1]);
5161 }
5162
5163 cache_ioctx.snap_set_read(librados::SNAP_HEAD);
5164 {
5165 bufferlist bl;
5166 ASSERT_EQ(4, cache_ioctx.read("foo", bl, 4, 0));
5167 ASSERT_EQ('c', bl[2]);
5168 }
5169
5170 cache_ioctx.snap_set_read(my_snaps[0]);
5171 {
5172 bufferlist bl;
5173 ASSERT_EQ(4, cache_ioctx.read("foo", bl, 4, 0));
5174 ASSERT_EQ('b', bl[2]);
5175 }
5176}
5177
5178TEST_F(LibRadosTwoPoolsPP, ManifestFlushDupCount) {
5179 // skip test if not yet octopus
5180 if (_get_required_osd_release(cluster) < "octopus") {
5181 cout << "cluster is not yet octopus, skipping test" << std::endl;
5182 return;
5183 }
5184
5185 bufferlist inbl;
5186 ASSERT_EQ(0, cluster.mon_command(
5187 set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
5188 inbl, NULL, NULL));
5189 ASSERT_EQ(0, cluster.mon_command(
5190 set_pool_str(cache_pool_name, "dedup_tier", pool_name),
5191 inbl, NULL, NULL));
5192 ASSERT_EQ(0, cluster.mon_command(
5193 set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
5194 inbl, NULL, NULL));
5195 ASSERT_EQ(0, cluster.mon_command(
5196 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
5197 inbl, NULL, NULL));
5198
5199 // create object
5200 bufferlist gbl;
5201 {
5202 //bufferlist bl;
5203 generate_buffer(1024*8, &gbl);
5204 ObjectWriteOperation op;
5205 op.write_full(gbl);
5206 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
5207 }
5208 {
5209 bufferlist bl;
5210 bl.append("there hiHI");
5211 ObjectWriteOperation op;
5212 op.write_full(bl);
5213 ASSERT_EQ(0, ioctx.operate("bar", &op));
9f95a23c
TL		 5214 }
5215
9f95a23c
TL		 5216 // wait for maps to settle
5217 cluster.wait_for_latest_osdmap();
5218
f67539c2
TL		 5219 // set-chunk to set manifest object
5220 {
5221 ObjectReadOperation op;
5222 op.set_chunk(0, 2, ioctx, "bar", 0,
5223 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
5224 librados::AioCompletion *completion = cluster.aio_create_completion();
5225 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
5226 librados::OPERATION_IGNORE_CACHE, NULL));
5227 completion->wait_for_complete();
5228 ASSERT_EQ(0, completion->get_return_value());
5229 completion->release();
5230 }
5231
5232 // create a snapshot, clone
5233 vector<uint64_t> my_snaps(1);
5234 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
5235 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5236 my_snaps));
5237
5238 // make a dirty chunks
5239 {
5240 bufferlist bl;
5241 bl.append("Thbbe hi");
5242 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5243 }
5244
5245 // and another
5246 my_snaps.resize(2);
5247 my_snaps[1] = my_snaps[0];
5248 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
5249 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5250 my_snaps));
5251
5252 // make a dirty chunks
5253 {
5254 bufferlist bl;
5255 bl.append("Thcce hi");
5256 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5257 }
5258
5259 //flush on oldest snap
5260 cache_ioctx.snap_set_read(my_snaps[1]);
5261 // flush
5262 {
5263 ObjectReadOperation op;
5264 op.tier_flush();
5265 librados::AioCompletion *completion = cluster.aio_create_completion();
5266 ASSERT_EQ(0, cache_ioctx.aio_operate(
5267 "foo", completion, &op,
5268 librados::OPERATION_IGNORE_CACHE, NULL));
5269 completion->wait_for_complete();
5270 ASSERT_EQ(0, completion->get_return_value());
5271 completion->release();
5272 }
5273
5274 // flush on oldest snap
5275 cache_ioctx.snap_set_read(my_snaps[0]);
5276 // flush
5277 {
5278 ObjectReadOperation op;
5279 op.tier_flush();
5280 librados::AioCompletion *completion = cluster.aio_create_completion();
5281 ASSERT_EQ(0, cache_ioctx.aio_operate(
5282 "foo", completion, &op,
5283 librados::OPERATION_IGNORE_CACHE, NULL));
5284 completion->wait_for_complete();
5285 ASSERT_EQ(0, completion->get_return_value());
5286 completion->release();
5287 }
5288
5289 cache_ioctx.snap_set_read(librados::SNAP_HEAD);
5290 // flush
5291 {
5292 ObjectReadOperation op;
5293 op.tier_flush();
5294 librados::AioCompletion *completion = cluster.aio_create_completion();
5295 ASSERT_EQ(0, cache_ioctx.aio_operate(
5296 "foo", completion, &op,
5297 librados::OPERATION_IGNORE_CACHE, NULL));
5298 completion->wait_for_complete();
5299 ASSERT_EQ(0, completion->get_return_value());
5300 completion->release();
5301 }
5302
5303 std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
5304 vector<pair<uint64_t, uint64_t>> chunks;
5305 bufferlist chunk;
5306 cdc->calc_chunks(gbl, &chunks);
5307 chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
5308 string tgt_oid;
5309 // check chunk's refcount
5310 {
5311 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5312 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5313 bufferlist t;
5314 SHA1 sha1_gen;
5315 int size = chunk.length();
5316 sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
5317 sha1_gen.Final(fingerprint);
5318 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5319 tgt_oid = string(p_str);
5320 ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
5321 chunk_refs_t refs;
5322 try {
5323 auto iter = t.cbegin();
5324 decode(refs, iter);
5325 } catch (buffer::error& err) {
5326 ASSERT_TRUE(0);
5327 }
a4b75251 5328 ASSERT_LE(1u, refs.count());
f67539c2
TL		 5329 }
5330
5331 bufferlist chunk2;
5332 chunk2.substr_of(gbl, chunks[0].first, chunks[0].second);
5333 // check chunk's refcount
5334 {
5335 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5336 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5337 bufferlist t;
5338 SHA1 sha1_gen;
5339 int size = chunk2.length();
5340 sha1_gen.Update((const unsigned char *)chunk2.c_str(), size);
5341 sha1_gen.Final(fingerprint);
5342 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5343 tgt_oid = string(p_str);
5344 ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
5345 chunk_refs_t refs;
5346 try {
5347 auto iter = t.cbegin();
5348 decode(refs, iter);
5349 } catch (buffer::error& err) {
5350 ASSERT_TRUE(0);
5351 }
a4b75251 5352 ASSERT_LE(1u, refs.count());
f67539c2
TL		 5353 }
5354
5355 // make a dirty chunks
5356 {
5357 bufferlist bl;
5358 bl.append("ThDDe hi");
5359 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5360 }
5361
5362 // flush
9f95a23c 5363 {
f67539c2
TL		 5364 ObjectReadOperation op;
5365 op.tier_flush();
9f95a23c 5366 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2
TL		 5367 ASSERT_EQ(0, cache_ioctx.aio_operate(
5368 "foo", completion, &op,
5369 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c
TL		 5370 completion->wait_for_complete();
5371 ASSERT_EQ(0, completion->get_return_value());
5372 completion->release();
5373 }
f67539c2
TL		 5374
5375 bufferlist tmp;
5376 tmp.append("Thcce hi");
5377 gbl.begin(0).copy_in(tmp.length(), tmp);
5378 bufferlist chunk3;
5379 cdc->calc_chunks(gbl, &chunks);
5380 chunk3.substr_of(gbl, chunks[0].first, chunks[0].second);
5381 // check chunk's refcount
5382 {
5383 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5384 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5385 bufferlist t;
5386 SHA1 sha1_gen;
5387 int size = chunk2.length();
5388 sha1_gen.Update((const unsigned char *)chunk2.c_str(), size);
5389 sha1_gen.Final(fingerprint);
5390 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5391 is_intended_refcount_state(cache_ioctx, "foo", ioctx, p_str, 0);
5392 }
5393}
5394
5395TEST_F(LibRadosTwoPoolsPP, TierFlushDuringFlush) {
5396 // skip test if not yet octopus
5397 if (_get_required_osd_release(cluster) < "octopus") {
5398 cout << "cluster is not yet octopus, skipping test" << std::endl;
5399 return;
5400 }
5401
5402 bufferlist inbl;
5403
5404 // create a new pool
5405 std::string temp_pool_name = get_temp_pool_name() + "-test-flush";
5406 ASSERT_EQ(0, cluster.pool_create(temp_pool_name.c_str()));
5407
5408 ASSERT_EQ(0, cluster.mon_command(
5409 set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
5410 inbl, NULL, NULL));
5411 ASSERT_EQ(0, cluster.mon_command(
5412 set_pool_str(cache_pool_name, "dedup_tier", temp_pool_name),
5413 inbl, NULL, NULL));
5414 ASSERT_EQ(0, cluster.mon_command(
5415 set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
5416 inbl, NULL, NULL));
5417 ASSERT_EQ(0, cluster.mon_command(
5418 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
5419 inbl, NULL, NULL));
5420
5421 // create object
5422 bufferlist gbl;
5423 {
5424 //bufferlist bl;
5425 generate_buffer(1024*8, &gbl);
5426 ObjectWriteOperation op;
5427 op.write_full(gbl);
5428 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
5429 }
9f95a23c 5430 {
f67539c2
TL		 5431 bufferlist bl;
5432 bl.append("there hiHI");
9f95a23c 5433 ObjectWriteOperation op;
f67539c2
TL		 5434 op.write_full(bl);
5435 ASSERT_EQ(0, ioctx.operate("bar", &op));
5436 }
5437
5438 // set-chunk to set manifest object
5439 {
5440 ObjectReadOperation op;
5441 op.set_chunk(0, 2, ioctx, "bar", 0,
5442 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
9f95a23c 5443 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2
TL		 5444 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
5445 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c
TL		 5446 completion->wait_for_complete();
5447 ASSERT_EQ(0, completion->get_return_value());
5448 completion->release();
5449 }
f67539c2
TL		 5450
5451 // delete temp pool, so flushing chunk will fail
5452 ASSERT_EQ(0, s_cluster.pool_delete(temp_pool_name.c_str()));
5453
5454 // flush to check if proper error is returned
5455 {
5456 ObjectReadOperation op;
5457 op.tier_flush();
5458 librados::AioCompletion *completion = cluster.aio_create_completion();
5459 ASSERT_EQ(0, cache_ioctx.aio_operate(
5460 "foo", completion, &op,
5461 librados::OPERATION_IGNORE_CACHE, NULL));
5462 completion->wait_for_complete();
5463 ASSERT_EQ(-ENOENT, completion->get_return_value());
5464 completion->release();
5465 }
5466
5467}
5468
5469TEST_F(LibRadosTwoPoolsPP, ManifestSnapHasChunk) {
5470 // skip test if not yet octopus
5471 if (_get_required_osd_release(cluster) < "octopus") {
5472 cout << "cluster is not yet octopus, skipping test" << std::endl;
5473 return;
5474 }
5475
5476 bufferlist inbl;
5477 ASSERT_EQ(0, cluster.mon_command(
5478 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
5479 inbl, NULL, NULL));
5480 cluster.wait_for_latest_osdmap();
5481
5482 // create object
9f95a23c
TL		 5483 {
5484 bufferlist bl;
f67539c2 5485 bl.append("there HIHI");
9f95a23c 5486 ObjectWriteOperation op;
f67539c2
TL		 5487 op.write_full(bl);
5488 ASSERT_EQ(0, ioctx.operate("foo", &op));
9f95a23c 5489 }
f67539c2
TL		 5490
5491 string er_fp_oid, hi_fp_oid, HI_fp_oid, ai_fp_oid, bi_fp_oid,
5492 Er_fp_oid, Hi_fp_oid, SI_fp_oid;
5493
5494 // get fp_oid
5495 er_fp_oid = get_fp_oid("er", "sha1");
5496 hi_fp_oid = get_fp_oid("hi", "sha1");
5497 HI_fp_oid = get_fp_oid("HI", "sha1");
5498 ai_fp_oid = get_fp_oid("ai", "sha1");
5499 bi_fp_oid = get_fp_oid("bi", "sha1");
5500 Er_fp_oid = get_fp_oid("Er", "sha1");
5501 Hi_fp_oid = get_fp_oid("Hi", "sha1");
5502 SI_fp_oid = get_fp_oid("SI", "sha1");
5503
5504 // write
9f95a23c
TL		 5505 {
5506 ObjectWriteOperation op;
f67539c2
TL		 5507 bufferlist bl;
5508 bl.append("er");
5509 op.write_full(bl);
5510 ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
9f95a23c 5511 }
f67539c2
TL		 5512 // write
5513 {
5514 ObjectWriteOperation op;
5515 bufferlist bl;
5516 bl.append("hi");
5517 op.write_full(bl);
5518 ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
5519 }
5520 // write
5521 {
5522 ObjectWriteOperation op;
5523 bufferlist bl;
5524 bl.append("HI");
5525 op.write_full(bl);
5526 ASSERT_EQ(0, cache_ioctx.operate(HI_fp_oid, &op));
5527 }
5528 // write
5529 {
5530 ObjectWriteOperation op;
5531 bufferlist bl;
5532 bl.append("ai");
5533 op.write_full(bl);
5534 ASSERT_EQ(0, cache_ioctx.operate(ai_fp_oid, &op));
5535 }
5536 // write
5537 {
5538 ObjectWriteOperation op;
5539 bufferlist bl;
5540 bl.append("bi");
5541 op.write_full(bl);
5542 ASSERT_EQ(0, cache_ioctx.operate(bi_fp_oid, &op));
5543 }
5544 // write
5545 {
5546 ObjectWriteOperation op;
5547 bufferlist bl;
5548 bl.append("Er");
5549 op.write_full(bl);
5550 ASSERT_EQ(0, cache_ioctx.operate(Er_fp_oid, &op));
5551 }
5552 // write
9f95a23c 5553 {
f67539c2
TL		 5554 ObjectWriteOperation op;
5555 bufferlist bl;
5556 bl.append("Hi");
5557 op.write_full(bl);
5558 ASSERT_EQ(0, cache_ioctx.operate(Hi_fp_oid, &op));
5559 }
5560 // write
5561 {
5562 ObjectWriteOperation op;
9f95a23c 5563 bufferlist bl;
f67539c2
TL		 5564 bl.append("SI");
5565 op.write_full(bl);
5566 ASSERT_EQ(0, cache_ioctx.operate(SI_fp_oid, &op));
9f95a23c
TL		 5567 }
5568
f67539c2
TL		 5569 // set-chunk (dedup)
5570 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, HI_fp_oid, "foo");
5571 // set-chunk (dedup)
5572 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, HI_fp_oid, "foo");
9f95a23c 5573
f67539c2
TL		 5574 // foo head: [hi] [HI]
5575
5576 // create a snapshot, clone
5577 vector<uint64_t> my_snaps(1);
5578 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5579 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5580 my_snaps));
5581
5582
5583 // create a clone
5584 {
5585 bufferlist bl;
5586 bl.append("a");
5587 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
5588 }
5589 // write
5590 {
5591 bufferlist bl;
5592 bl.append("a");
5593 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
5594 }
5595 // write
5596 {
5597 bufferlist bl;
5598 bl.append("S");
5599 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 8));
5600 }
5601
5602 // foo snap[0]: [hi] [HI]
5603 // foo head : [er] [ai] [SI]
5604
5605 // set-chunk (dedup)
5606 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
5607 // set-chunk (dedup)
5608 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ai_fp_oid, "foo");
5609 // set-chunk (dedup)
5610 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, SI_fp_oid, "foo");
5611
5612 my_snaps.resize(2);
5613 my_snaps[1] = my_snaps[0];
5614 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5615 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5616 my_snaps));
5617
5618 // create a clone
5619 {
5620 bufferlist bl;
5621 bl.append("b");
5622 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
5623 }
5624 // write
5625 {
5626 bufferlist bl;
5627 bl.append("b");
5628 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
5629 }
5630
5631 // foo snap[1]: [HI] [HI]
5632 // foo snap[0]: [er] [ai] [SI]
5633 // foo head : [er] [bi] [SI]
5634
5635 // set-chunk (dedup)
5636 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, bi_fp_oid, "foo");
5637
5638 {
5639 ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", SI_fp_oid));
5640 ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", er_fp_oid));
5641 ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", ai_fp_oid));
5642 ASSERT_EQ(2, cls_cas_references_chunk(ioctx, "foo", HI_fp_oid));
5643 ASSERT_EQ(-ENOLINK, cls_cas_references_chunk(ioctx, "foo", Hi_fp_oid));
5644 }
9f95a23c
TL		 5645}
5646
7c673cae
FG		 5647class LibRadosTwoPoolsECPP : public RadosTestECPP
5648{
5649public:
5650 LibRadosTwoPoolsECPP() {};
5651 ~LibRadosTwoPoolsECPP() override {};
5652protected:
5653 static void SetUpTestCase() {
5654 pool_name = get_temp_pool_name();
5655 ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
5656 }
5657 static void TearDownTestCase() {
5658 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
5659 }
5660 static std::string cache_pool_name;
5661
5662 void SetUp() override {
5663 cache_pool_name = get_temp_pool_name();
5664 ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
5665 RadosTestECPP::SetUp();
c07f9fc5 5666
7c673cae 5667 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 5668 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 5669 cache_ioctx.set_namespace(nspace);
5670 }
5671 void TearDown() override {
5672 // flush + evict cache
5673 flush_evict_all(cluster, cache_ioctx);
5674
5675 bufferlist inbl;
5676 // tear down tiers
5677 ASSERT_EQ(0, cluster.mon_command(
5678 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
5679 "\"}",
5680 inbl, NULL, NULL));
5681 ASSERT_EQ(0, cluster.mon_command(
5682 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
5683 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5684 inbl, NULL, NULL));
5685
5686 // wait for maps to settle before next test
5687 cluster.wait_for_latest_osdmap();
5688
5689 RadosTestECPP::TearDown();
5690
5691 cleanup_default_namespace(cache_ioctx);
5692 cleanup_namespace(cache_ioctx, nspace);
5693
5694 cache_ioctx.close();
5695 ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
5696 }
5697
5698 librados::IoCtx cache_ioctx;
5699};
5700
5701std::string LibRadosTwoPoolsECPP::cache_pool_name;
5702
5703TEST_F(LibRadosTierECPP, Dirty) {
5704 {
5705 ObjectWriteOperation op;
5706 op.undirty();
5707 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
5708 }
5709 {
5710 ObjectWriteOperation op;
5711 op.create(true);
5712 ASSERT_EQ(0, ioctx.operate("foo", &op));
5713 }
5714 {
5715 bool dirty = false;
5716 int r = -1;
5717 ObjectReadOperation op;
5718 op.is_dirty(&dirty, &r);
5719 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
5720 ASSERT_TRUE(dirty);
5721 ASSERT_EQ(0, r);
5722 }
5723 {
5724 ObjectWriteOperation op;
5725 op.undirty();
5726 ASSERT_EQ(0, ioctx.operate("foo", &op));
5727 }
5728 {
5729 ObjectWriteOperation op;
5730 op.undirty();
5731 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
5732 }
5733 {
5734 bool dirty = false;
5735 int r = -1;
5736 ObjectReadOperation op;
5737 op.is_dirty(&dirty, &r);
5738 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
5739 ASSERT_FALSE(dirty);
5740 ASSERT_EQ(0, r);
5741 }
5742 //{
5743 // ObjectWriteOperation op;
5744 // op.truncate(0); // still a write even tho it is a no-op
5745 // ASSERT_EQ(0, ioctx.operate("foo", &op));
5746 //}
5747 //{
5748 // bool dirty = false;
5749 // int r = -1;
5750 // ObjectReadOperation op;
5751 // op.is_dirty(&dirty, &r);
5752 // ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
5753 // ASSERT_TRUE(dirty);
5754 // ASSERT_EQ(0, r);
5755 //}
5756}
5757
5758TEST_F(LibRadosTwoPoolsECPP, Overlay) {
5759 // create objects
5760 {
5761 bufferlist bl;
5762 bl.append("base");
5763 ObjectWriteOperation op;
5764 op.write_full(bl);
5765 ASSERT_EQ(0, ioctx.operate("foo", &op));
5766 }
5767 {
5768 bufferlist bl;
5769 bl.append("cache");
5770 ObjectWriteOperation op;
5771 op.write_full(bl);
5772 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
5773 }
5774
5775 // configure cache
5776 bufferlist inbl;
5777 ASSERT_EQ(0, cluster.mon_command(
5778 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5779 "\", \"tierpool\": \"" + cache_pool_name +
5780 "\", \"force_nonempty\": \"--force-nonempty\" }",
5781 inbl, NULL, NULL));
5782 ASSERT_EQ(0, cluster.mon_command(
5783 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5784 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5785 inbl, NULL, NULL));
5786
5787 // wait for maps to settle
5788 cluster.wait_for_latest_osdmap();
5789
5790 // by default, the overlay sends us to cache pool
5791 {
5792 bufferlist bl;
5793 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5794 ASSERT_EQ('c', bl[0]);
5795 }
5796 {
5797 bufferlist bl;
5798 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
5799 ASSERT_EQ('c', bl[0]);
5800 }
5801
5802 // unless we say otherwise
5803 {
5804 bufferlist bl;
5805 ObjectReadOperation op;
5806 op.read(0, 1, &bl, NULL);
5807 librados::AioCompletion *completion = cluster.aio_create_completion();
5808 ASSERT_EQ(0, ioctx.aio_operate(
5809 "foo", completion, &op,
5810 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 5811 completion->wait_for_complete();
7c673cae
FG		 5812 ASSERT_EQ(0, completion->get_return_value());
5813 completion->release();
5814 ASSERT_EQ('b', bl[0]);
5815 }
5816}
5817
5818TEST_F(LibRadosTwoPoolsECPP, Promote) {
5819 // create object
5820 {
5821 bufferlist bl;
5822 bl.append("hi there");
5823 ObjectWriteOperation op;
5824 op.write_full(bl);
5825 ASSERT_EQ(0, ioctx.operate("foo", &op));
5826 }
5827
5828 // configure cache
5829 bufferlist inbl;
5830 ASSERT_EQ(0, cluster.mon_command(
5831 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5832 "\", \"tierpool\": \"" + cache_pool_name +
5833 "\", \"force_nonempty\": \"--force-nonempty\" }",
5834 inbl, NULL, NULL));
5835 ASSERT_EQ(0, cluster.mon_command(
5836 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5837 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5838 inbl, NULL, NULL));
5839 ASSERT_EQ(0, cluster.mon_command(
5840 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5841 "\", \"mode\": \"writeback\"}",
5842 inbl, NULL, NULL));
5843
5844 // wait for maps to settle
5845 cluster.wait_for_latest_osdmap();
5846
5847 // read, trigger a promote
5848 {
5849 bufferlist bl;
5850 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5851 }
5852
5853 // read, trigger a whiteout
5854 {
5855 bufferlist bl;
5856 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
5857 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
5858 }
5859
5860 // verify the object is present in the cache tier
5861 {
5862 NObjectIterator it = cache_ioctx.nobjects_begin();
5863 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
5864 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
5865 ++it;
5866 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
5867 ++it;
5868 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
5869 }
5870}
5871
5872TEST_F(LibRadosTwoPoolsECPP, PromoteSnap) {
5873 // create object
5874 {
5875 bufferlist bl;
5876 bl.append("hi there");
5877 ObjectWriteOperation op;
5878 op.write_full(bl);
5879 ASSERT_EQ(0, ioctx.operate("foo", &op));
5880 }
5881 {
5882 bufferlist bl;
5883 bl.append("hi there");
5884 ObjectWriteOperation op;
5885 op.write_full(bl);
5886 ASSERT_EQ(0, ioctx.operate("bar", &op));
5887 }
5888 {
5889 bufferlist bl;
5890 bl.append("hi there");
5891 ObjectWriteOperation op;
5892 op.write_full(bl);
5893 ASSERT_EQ(0, ioctx.operate("baz", &op));
5894 }
5895 {
5896 bufferlist bl;
5897 bl.append("hi there");
5898 ObjectWriteOperation op;
5899 op.write_full(bl);
5900 ASSERT_EQ(0, ioctx.operate("bam", &op));
5901 }
5902
5903 // create a snapshot, clone
5904 vector<uint64_t> my_snaps(1);
5905 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5906 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5907 my_snaps));
5908 {
5909 bufferlist bl;
5910 bl.append("ciao!");
5911 ObjectWriteOperation op;
5912 op.write_full(bl);
5913 ASSERT_EQ(0, ioctx.operate("foo", &op));
5914 }
5915 {
5916 bufferlist bl;
5917 bl.append("ciao!");
5918 ObjectWriteOperation op;
5919 op.write_full(bl);
5920 ASSERT_EQ(0, ioctx.operate("bar", &op));
5921 }
5922 {
5923 ObjectWriteOperation op;
5924 op.remove();
5925 ASSERT_EQ(0, ioctx.operate("baz", &op));
5926 }
5927 {
5928 bufferlist bl;
5929 bl.append("ciao!");
5930 ObjectWriteOperation op;
5931 op.write_full(bl);
5932 ASSERT_EQ(0, ioctx.operate("bam", &op));
5933 }
5934
5935 // configure cache
5936 bufferlist inbl;
5937 ASSERT_EQ(0, cluster.mon_command(
5938 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5939 "\", \"tierpool\": \"" + cache_pool_name +
5940 "\", \"force_nonempty\": \"--force-nonempty\" }",
5941 inbl, NULL, NULL));
5942 ASSERT_EQ(0, cluster.mon_command(
5943 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5944 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5945 inbl, NULL, NULL));
5946 ASSERT_EQ(0, cluster.mon_command(
5947 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5948 "\", \"mode\": \"writeback\"}",
5949 inbl, NULL, NULL));
5950
5951 // wait for maps to settle
5952 cluster.wait_for_latest_osdmap();
5953
5954 // read, trigger a promote on the head
5955 {
5956 bufferlist bl;
5957 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5958 ASSERT_EQ('c', bl[0]);
5959 }
5960 {
5961 bufferlist bl;
5962 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
5963 ASSERT_EQ('c', bl[0]);
5964 }
5965
5966 ioctx.snap_set_read(my_snaps[0]);
5967
5968 // stop and scrub this pg (to make sure scrub can handle missing
5969 // clones in the cache tier)
5970 // This test requires cache tier and base tier to have the same pg_num/pgp_num
5971 {
5972 for (int tries = 0; tries < 5; ++tries) {
5973 IoCtx cache_ioctx;
5974 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
5975 uint32_t hash;
5976 ASSERT_EQ(0, ioctx.get_object_pg_hash_position2("foo", &hash));
5977 ostringstream ss;
5978 ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
5979 << cache_ioctx.get_id() << "."
5980 << hash
5981 << "\"}";
5982 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
5983 if (r == -EAGAIN ||
5984 r == -ENOENT) { // in case mgr osdmap is a bit stale
5985 sleep(5);
5986 continue;
5987 }
5988 ASSERT_EQ(0, r);
5989 break;
5990 }
5991 // give it a few seconds to go. this is sloppy but is usually enough time
5992 cout << "waiting for scrub..." << std::endl;
5993 sleep(15);
5994 cout << "done waiting" << std::endl;
5995 }
5996
5997 // read foo snap
5998 {
5999 bufferlist bl;
6000 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6001 ASSERT_EQ('h', bl[0]);
6002 }
6003
6004 // read bar snap
6005 {
6006 bufferlist bl;
6007 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
6008 ASSERT_EQ('h', bl[0]);
6009 }
6010
6011 // read baz snap
6012 {
6013 bufferlist bl;
6014 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
6015 ASSERT_EQ('h', bl[0]);
6016 }
6017
6018 ioctx.snap_set_read(librados::SNAP_HEAD);
6019
6020 // read foo
6021 {
6022 bufferlist bl;
6023 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6024 ASSERT_EQ('c', bl[0]);
6025 }
6026
6027 // read bar
6028 {
6029 bufferlist bl;
6030 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
6031 ASSERT_EQ('c', bl[0]);
6032 }
6033
6034 // read baz
6035 {
6036 bufferlist bl;
6037 ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
6038 }
6039
6040 // cleanup
6041 ioctx.selfmanaged_snap_remove(my_snaps[0]);
6042}
6043
6044TEST_F(LibRadosTwoPoolsECPP, PromoteSnapTrimRace) {
6045 // create object
6046 {
6047 bufferlist bl;
6048 bl.append("hi there");
6049 ObjectWriteOperation op;
6050 op.write_full(bl);
6051 ASSERT_EQ(0, ioctx.operate("foo", &op));
6052 }
6053
6054 // create a snapshot, clone
6055 vector<uint64_t> my_snaps(1);
6056 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
6057 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
6058 my_snaps));
6059 {
6060 bufferlist bl;
6061 bl.append("ciao!");
6062 ObjectWriteOperation op;
6063 op.write_full(bl);
6064 ASSERT_EQ(0, ioctx.operate("foo", &op));
6065 }
6066
6067 // configure cache
6068 bufferlist inbl;
6069 ASSERT_EQ(0, cluster.mon_command(
6070 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6071 "\", \"tierpool\": \"" + cache_pool_name +
6072 "\", \"force_nonempty\": \"--force-nonempty\" }",
6073 inbl, NULL, NULL));
6074 ASSERT_EQ(0, cluster.mon_command(
6075 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6076 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6077 inbl, NULL, NULL));
6078 ASSERT_EQ(0, cluster.mon_command(
6079 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6080 "\", \"mode\": \"writeback\"}",
6081 inbl, NULL, NULL));
6082
6083 // wait for maps to settle
6084 cluster.wait_for_latest_osdmap();
6085
6086 // delete the snap
6087 ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
6088
6089 ioctx.snap_set_read(my_snaps[0]);
6090
9f95a23c
TL		 6091 // read foo snap. the OSD may or may not realize that this snap has
6092 // been logically deleted; either response is valid.
7c673cae
FG		 6093 {
6094 bufferlist bl;
9f95a23c
TL		 6095 int r = ioctx.read("foo", bl, 1, 0);
6096 ASSERT_TRUE(r == 1 || r == -ENOENT);
7c673cae
FG		 6097 }
6098
6099 // cleanup
6100 ioctx.selfmanaged_snap_remove(my_snaps[0]);
6101}
6102
6103TEST_F(LibRadosTwoPoolsECPP, Whiteout) {
6104 // create object
6105 {
6106 bufferlist bl;
6107 bl.append("hi there");
6108 ObjectWriteOperation op;
6109 op.write_full(bl);
6110 ASSERT_EQ(0, ioctx.operate("foo", &op));
6111 }
6112
6113 // configure cache
6114 bufferlist inbl;
6115 ASSERT_EQ(0, cluster.mon_command(
6116 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6117 "\", \"tierpool\": \"" + cache_pool_name +
6118 "\", \"force_nonempty\": \"--force-nonempty\" }",
6119 inbl, NULL, NULL));
6120 ASSERT_EQ(0, cluster.mon_command(
6121 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6122 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6123 inbl, NULL, NULL));
6124 ASSERT_EQ(0, cluster.mon_command(
6125 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6126 "\", \"mode\": \"writeback\"}",
6127 inbl, NULL, NULL));
6128
6129 // wait for maps to settle
6130 cluster.wait_for_latest_osdmap();
6131
6132 // create some whiteouts, verify they behave
6133 {
6134 ObjectWriteOperation op;
6135 op.assert_exists();
6136 op.remove();
6137 ASSERT_EQ(0, ioctx.operate("foo", &op));
6138 }
6139
6140 {
6141 ObjectWriteOperation op;
6142 op.assert_exists();
6143 op.remove();
6144 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
6145 }
6146 {
6147 ObjectWriteOperation op;
6148 op.assert_exists();
6149 op.remove();
6150 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
6151 }
6152
6153 // verify the whiteouts are there in the cache tier
6154 {
6155 NObjectIterator it = cache_ioctx.nobjects_begin();
6156 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6157 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6158 ++it;
6159 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6160 ++it;
6161 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6162 }
6163
6164 // delete a whiteout and verify it goes away
6165 ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
6166 {
6167 ObjectWriteOperation op;
6168 op.remove();
6169 librados::AioCompletion *completion = cluster.aio_create_completion();
6170 ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
6171 librados::OPERATION_IGNORE_CACHE));
9f95a23c 6172 completion->wait_for_complete();
7c673cae
FG		 6173 ASSERT_EQ(0, completion->get_return_value());
6174 completion->release();
6175
6176 NObjectIterator it = cache_ioctx.nobjects_begin();
6177 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6178 ASSERT_TRUE(it->get_oid() == string("foo"));
6179 ++it;
6180 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6181 }
6182
6183 // recreate an object and verify we can read it
6184 {
6185 bufferlist bl;
6186 bl.append("hi there");
6187 ObjectWriteOperation op;
6188 op.write_full(bl);
6189 ASSERT_EQ(0, ioctx.operate("foo", &op));
6190 }
6191 {
6192 bufferlist bl;
6193 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6194 ASSERT_EQ('h', bl[0]);
6195 }
6196}
6197
6198TEST_F(LibRadosTwoPoolsECPP, Evict) {
6199 // create object
6200 {
6201 bufferlist bl;
6202 bl.append("hi there");
6203 ObjectWriteOperation op;
6204 op.write_full(bl);
6205 ASSERT_EQ(0, ioctx.operate("foo", &op));
6206 }
6207
6208 // configure cache
6209 bufferlist inbl;
6210 ASSERT_EQ(0, cluster.mon_command(
6211 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6212 "\", \"tierpool\": \"" + cache_pool_name +
6213 "\", \"force_nonempty\": \"--force-nonempty\" }",
6214 inbl, NULL, NULL));
6215 ASSERT_EQ(0, cluster.mon_command(
6216 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6217 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6218 inbl, NULL, NULL));
6219 ASSERT_EQ(0, cluster.mon_command(
6220 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6221 "\", \"mode\": \"writeback\"}",
6222 inbl, NULL, NULL));
6223
6224 // wait for maps to settle
6225 cluster.wait_for_latest_osdmap();
6226
6227 // read, trigger a promote
6228 {
6229 bufferlist bl;
6230 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6231 }
6232
6233 // read, trigger a whiteout, and a dirty object
6234 {
6235 bufferlist bl;
6236 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
6237 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
6238 ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
6239 }
6240
6241 // verify the object is present in the cache tier
6242 {
6243 NObjectIterator it = cache_ioctx.nobjects_begin();
6244 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6245 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6246 ++it;
6247 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6248 ++it;
6249 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6250 }
6251
6252 // pin
6253 {
6254 ObjectWriteOperation op;
6255 op.cache_pin();
6256 librados::AioCompletion *completion = cluster.aio_create_completion();
6257 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6258 completion->wait_for_complete();
7c673cae
FG		 6259 ASSERT_EQ(0, completion->get_return_value());
6260 completion->release();
6261 }
6262
6263 // evict the pinned object with -EPERM
6264 {
6265 ObjectReadOperation op;
6266 op.cache_evict();
6267 librados::AioCompletion *completion = cluster.aio_create_completion();
6268 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
6269 librados::OPERATION_IGNORE_CACHE,
6270 NULL));
9f95a23c 6271 completion->wait_for_complete();
7c673cae
FG		 6272 ASSERT_EQ(-EPERM, completion->get_return_value());
6273 completion->release();
6274 }
6275
6276 // unpin
6277 {
6278 ObjectWriteOperation op;
6279 op.cache_unpin();
6280 librados::AioCompletion *completion = cluster.aio_create_completion();
6281 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6282 completion->wait_for_complete();
7c673cae
FG		 6283 ASSERT_EQ(0, completion->get_return_value());
6284 completion->release();
6285 }
6286
6287 // flush
6288 {
6289 ObjectReadOperation op;
6290 op.cache_flush();
6291 librados::AioCompletion *completion = cluster.aio_create_completion();
6292 ASSERT_EQ(0, cache_ioctx.aio_operate(
6293 "foo", completion, &op,
6294 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 6295 completion->wait_for_complete();
7c673cae
FG		 6296 ASSERT_EQ(0, completion->get_return_value());
6297 completion->release();
6298 }
6299
6300 // verify clean
6301 {
6302 bool dirty = false;
6303 int r = -1;
6304 ObjectReadOperation op;
6305 op.is_dirty(&dirty, &r);
6306 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
6307 ASSERT_FALSE(dirty);
6308 ASSERT_EQ(0, r);
6309 }
6310
6311 // evict
6312 {
6313 ObjectReadOperation op;
6314 op.cache_evict();
6315 librados::AioCompletion *completion = cluster.aio_create_completion();
6316 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
6317 librados::OPERATION_IGNORE_CACHE,
6318 NULL));
9f95a23c 6319 completion->wait_for_complete();
7c673cae
FG		 6320 ASSERT_EQ(0, completion->get_return_value());
6321 completion->release();
6322 }
6323 {
6324 ObjectReadOperation op;
6325 op.cache_evict();
6326 librados::AioCompletion *completion = cluster.aio_create_completion();
6327 ASSERT_EQ(0, cache_ioctx.aio_operate(
6328 "foo", completion, &op,
6329 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6330 completion->wait_for_complete();
7c673cae
FG		 6331 ASSERT_EQ(0, completion->get_return_value());
6332 completion->release();
6333 }
6334 {
6335 ObjectReadOperation op;
6336 op.cache_evict();
6337 librados::AioCompletion *completion = cluster.aio_create_completion();
6338 ASSERT_EQ(0, cache_ioctx.aio_operate(
6339 "bar", completion, &op,
6340 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6341 completion->wait_for_complete();
7c673cae
FG		 6342 ASSERT_EQ(-EBUSY, completion->get_return_value());
6343 completion->release();
6344 }
6345}
6346
6347TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
6348 // create object
6349 {
6350 bufferlist bl;
6351 bl.append("hi there");
6352 ObjectWriteOperation op;
6353 op.write_full(bl);
6354 ASSERT_EQ(0, ioctx.operate("foo", &op));
6355 }
6356 {
6357 bufferlist bl;
6358 bl.append("hi there");
6359 ObjectWriteOperation op;
6360 op.write_full(bl);
6361 ASSERT_EQ(0, ioctx.operate("bar", &op));
6362 }
6363 {
6364 bufferlist bl;
6365 bl.append("hi there");
6366 ObjectWriteOperation op;
6367 op.write_full(bl);
6368 ASSERT_EQ(0, ioctx.operate("baz", &op));
6369 }
6370 {
6371 bufferlist bl;
6372 bl.append("hi there");
6373 ObjectWriteOperation op;
6374 op.write_full(bl);
6375 ASSERT_EQ(0, ioctx.operate("bam", &op));
6376 }
6377
6378 // create a snapshot, clone
6379 vector<uint64_t> my_snaps(1);
6380 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
6381 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
6382 my_snaps));
6383 {
6384 bufferlist bl;
6385 bl.append("ciao!");
6386 ObjectWriteOperation op;
6387 op.write_full(bl);
6388 ASSERT_EQ(0, ioctx.operate("foo", &op));
6389 }
6390 {
6391 bufferlist bl;
6392 bl.append("ciao!");
6393 ObjectWriteOperation op;
6394 op.write_full(bl);
6395 ASSERT_EQ(0, ioctx.operate("bar", &op));
6396 }
6397 {
6398 ObjectWriteOperation op;
6399 op.remove();
6400 ASSERT_EQ(0, ioctx.operate("baz", &op));
6401 }
6402 {
6403 bufferlist bl;
6404 bl.append("ciao!");
6405 ObjectWriteOperation op;
6406 op.write_full(bl);
6407 ASSERT_EQ(0, ioctx.operate("bam", &op));
6408 }
6409
6410 // configure cache
6411 bufferlist inbl;
6412 ASSERT_EQ(0, cluster.mon_command(
6413 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6414 "\", \"tierpool\": \"" + cache_pool_name +
6415 "\", \"force_nonempty\": \"--force-nonempty\" }",
6416 inbl, NULL, NULL));
6417 ASSERT_EQ(0, cluster.mon_command(
6418 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6419 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6420 inbl, NULL, NULL));
6421 ASSERT_EQ(0, cluster.mon_command(
6422 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6423 "\", \"mode\": \"writeback\"}",
6424 inbl, NULL, NULL));
6425
6426 // wait for maps to settle
6427 cluster.wait_for_latest_osdmap();
6428
6429 // read, trigger a promote on the head
6430 {
6431 bufferlist bl;
6432 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6433 ASSERT_EQ('c', bl[0]);
6434 }
6435 {
6436 bufferlist bl;
6437 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
6438 ASSERT_EQ('c', bl[0]);
6439 }
6440
6441 // evict bam
6442 {
6443 ObjectReadOperation op;
6444 op.cache_evict();
6445 librados::AioCompletion *completion = cluster.aio_create_completion();
6446 ASSERT_EQ(0, cache_ioctx.aio_operate(
6447 "bam", completion, &op,
6448 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6449 completion->wait_for_complete();
7c673cae
FG		 6450 ASSERT_EQ(0, completion->get_return_value());
6451 completion->release();
6452 }
6453 {
6454 bufferlist bl;
6455 ObjectReadOperation op;
6456 op.read(1, 0, &bl, NULL);
6457 librados::AioCompletion *completion = cluster.aio_create_completion();
6458 ASSERT_EQ(0, cache_ioctx.aio_operate(
6459 "bam", completion, &op,
6460 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6461 completion->wait_for_complete();
7c673cae
FG		 6462 ASSERT_EQ(-ENOENT, completion->get_return_value());
6463 completion->release();
6464 }
6465
6466 // read foo snap
6467 ioctx.snap_set_read(my_snaps[0]);
6468 {
6469 bufferlist bl;
6470 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6471 ASSERT_EQ('h', bl[0]);
6472 }
6473
6474 // evict foo snap
6475 {
6476 ObjectReadOperation op;
6477 op.cache_evict();
6478 librados::AioCompletion *completion = cluster.aio_create_completion();
6479 ASSERT_EQ(0, ioctx.aio_operate(
6480 "foo", completion, &op,
6481 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6482 completion->wait_for_complete();
7c673cae
FG		 6483 ASSERT_EQ(0, completion->get_return_value());
6484 completion->release();
6485 }
6486 // snap is gone...
6487 {
6488 bufferlist bl;
6489 ObjectReadOperation op;
6490 op.read(1, 0, &bl, NULL);
6491 librados::AioCompletion *completion = cluster.aio_create_completion();
6492 ASSERT_EQ(0, ioctx.aio_operate(
6493 "foo", completion, &op,
6494 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6495 completion->wait_for_complete();
7c673cae
FG		 6496 ASSERT_EQ(-ENOENT, completion->get_return_value());
6497 completion->release();
6498 }
6499 // head is still there...
6500 ioctx.snap_set_read(librados::SNAP_HEAD);
6501 {
6502 bufferlist bl;
6503 ObjectReadOperation op;
6504 op.read(1, 0, &bl, NULL);
6505 librados::AioCompletion *completion = cluster.aio_create_completion();
6506 ASSERT_EQ(0, ioctx.aio_operate(
6507 "foo", completion, &op,
6508 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6509 completion->wait_for_complete();
7c673cae
FG		 6510 ASSERT_EQ(0, completion->get_return_value());
6511 completion->release();
6512 }
6513
6514 // promote head + snap of bar
6515 ioctx.snap_set_read(librados::SNAP_HEAD);
6516 {
6517 bufferlist bl;
6518 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
6519 ASSERT_EQ('c', bl[0]);
6520 }
6521 ioctx.snap_set_read(my_snaps[0]);
6522 {
6523 bufferlist bl;
6524 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
6525 ASSERT_EQ('h', bl[0]);
6526 }
6527
6528 // evict bar head (fail)
6529 ioctx.snap_set_read(librados::SNAP_HEAD);
6530 {
6531 ObjectReadOperation op;
6532 op.cache_evict();
6533 librados::AioCompletion *completion = cluster.aio_create_completion();
6534 ASSERT_EQ(0, ioctx.aio_operate(
6535 "bar", completion, &op,
6536 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6537 completion->wait_for_complete();
7c673cae
FG		 6538 ASSERT_EQ(-EBUSY, completion->get_return_value());
6539 completion->release();
6540 }
6541
6542 // evict bar snap
6543 ioctx.snap_set_read(my_snaps[0]);
6544 {
6545 ObjectReadOperation op;
6546 op.cache_evict();
6547 librados::AioCompletion *completion = cluster.aio_create_completion();
6548 ASSERT_EQ(0, ioctx.aio_operate(
6549 "bar", completion, &op,
6550 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6551 completion->wait_for_complete();
7c673cae
FG		 6552 ASSERT_EQ(0, completion->get_return_value());
6553 completion->release();
6554 }
6555 // ...and then head
6556 ioctx.snap_set_read(librados::SNAP_HEAD);
6557 {
6558 bufferlist bl;
6559 ObjectReadOperation op;
6560 op.read(1, 0, &bl, NULL);
6561 librados::AioCompletion *completion = cluster.aio_create_completion();
6562 ASSERT_EQ(0, ioctx.aio_operate(
6563 "bar", completion, &op,
6564 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6565 completion->wait_for_complete();
7c673cae
FG		 6566 ASSERT_EQ(0, completion->get_return_value());
6567 completion->release();
6568 }
6569 {
6570 ObjectReadOperation op;
6571 op.cache_evict();
6572 librados::AioCompletion *completion = cluster.aio_create_completion();
6573 ASSERT_EQ(0, ioctx.aio_operate(
6574 "bar", completion, &op,
6575 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6576 completion->wait_for_complete();
7c673cae
FG		 6577 ASSERT_EQ(0, completion->get_return_value());
6578 completion->release();
6579 }
6580
6581 // cleanup
6582 ioctx.selfmanaged_snap_remove(my_snaps[0]);
6583}
6584
6585TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
6586 // configure cache
6587 bufferlist inbl;
6588 ASSERT_EQ(0, cluster.mon_command(
6589 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6590 "\", \"tierpool\": \"" + cache_pool_name +
6591 "\", \"force_nonempty\": \"--force-nonempty\" }",
6592 inbl, NULL, NULL));
6593 ASSERT_EQ(0, cluster.mon_command(
6594 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6595 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6596 inbl, NULL, NULL));
6597 ASSERT_EQ(0, cluster.mon_command(
6598 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6599 "\", \"mode\": \"writeback\"}",
6600 inbl, NULL, NULL));
6601
6602 // wait for maps to settle
6603 cluster.wait_for_latest_osdmap();
6604
6605 // create object
6606 {
6607 bufferlist bl;
6608 bl.append("hi there");
6609 ObjectWriteOperation op;
6610 op.write_full(bl);
6611 ASSERT_EQ(0, ioctx.operate("foo", &op));
6612 }
6613
6614 // verify the object is present in the cache tier
6615 {
6616 NObjectIterator it = cache_ioctx.nobjects_begin();
6617 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6618 ASSERT_TRUE(it->get_oid() == string("foo"));
6619 ++it;
6620 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6621 }
6622
6623 // verify the object is NOT present in the base tier
6624 {
6625 NObjectIterator it = ioctx.nobjects_begin();
6626 ASSERT_TRUE(it == ioctx.nobjects_end());
6627 }
6628
6629 // verify dirty
6630 {
6631 bool dirty = false;
6632 int r = -1;
6633 ObjectReadOperation op;
6634 op.is_dirty(&dirty, &r);
6635 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
6636 ASSERT_TRUE(dirty);
6637 ASSERT_EQ(0, r);
6638 }
6639
6640 // pin
6641 {
6642 ObjectWriteOperation op;
6643 op.cache_pin();
6644 librados::AioCompletion *completion = cluster.aio_create_completion();
6645 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6646 completion->wait_for_complete();
7c673cae
FG		 6647 ASSERT_EQ(0, completion->get_return_value());
6648 completion->release();
6649 }
6650
6651 // flush the pinned object with -EPERM
6652 {
6653 ObjectReadOperation op;
6654 op.cache_try_flush();
6655 librados::AioCompletion *completion = cluster.aio_create_completion();
6656 ASSERT_EQ(0, cache_ioctx.aio_operate(
6657 "foo", completion, &op,
6658 librados::OPERATION_IGNORE_OVERLAY |
6659 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 6660 completion->wait_for_complete();
7c673cae
FG		 6661 ASSERT_EQ(-EPERM, completion->get_return_value());
6662 completion->release();
6663 }
6664
6665 // unpin
6666 {
6667 ObjectWriteOperation op;
6668 op.cache_unpin();
6669 librados::AioCompletion *completion = cluster.aio_create_completion();
6670 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6671 completion->wait_for_complete();
7c673cae
FG		 6672 ASSERT_EQ(0, completion->get_return_value());
6673 completion->release();
6674 }
6675
6676 // flush
6677 {
6678 ObjectReadOperation op;
6679 op.cache_try_flush();
6680 librados::AioCompletion *completion = cluster.aio_create_completion();
6681 ASSERT_EQ(0, cache_ioctx.aio_operate(
6682 "foo", completion, &op,
6683 librados::OPERATION_IGNORE_OVERLAY |
6684 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 6685 completion->wait_for_complete();
7c673cae
FG		 6686 ASSERT_EQ(0, completion->get_return_value());
6687 completion->release();
6688 }
6689
6690 // verify clean
6691 {
6692 bool dirty = false;
6693 int r = -1;
6694 ObjectReadOperation op;
6695 op.is_dirty(&dirty, &r);
6696 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
6697 ASSERT_FALSE(dirty);
6698 ASSERT_EQ(0, r);
6699 }
6700
6701 // verify in base tier
6702 {
6703 NObjectIterator it = ioctx.nobjects_begin();
6704 ASSERT_TRUE(it != ioctx.nobjects_end());
6705 ASSERT_TRUE(it->get_oid() == string("foo"));
6706 ++it;
6707 ASSERT_TRUE(it == ioctx.nobjects_end());
6708 }
6709
6710 // evict it
6711 {
6712 ObjectReadOperation op;
6713 op.cache_evict();
6714 librados::AioCompletion *completion = cluster.aio_create_completion();
6715 ASSERT_EQ(0, cache_ioctx.aio_operate(
6716 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6717 completion->wait_for_complete();
7c673cae
FG		 6718 ASSERT_EQ(0, completion->get_return_value());
6719 completion->release();
6720 }
6721
6722 // verify no longer in cache tier
6723 {
6724 NObjectIterator it = cache_ioctx.nobjects_begin();
6725 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6726 }
6727}
6728
6729TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
6730 // configure cache
6731 bufferlist inbl;
6732 ASSERT_EQ(0, cluster.mon_command(
6733 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6734 "\", \"tierpool\": \"" + cache_pool_name +
6735 "\", \"force_nonempty\": \"--force-nonempty\" }",
6736 inbl, NULL, NULL));
6737 ASSERT_EQ(0, cluster.mon_command(
6738 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6739 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6740 inbl, NULL, NULL));
6741 ASSERT_EQ(0, cluster.mon_command(
6742 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6743 "\", \"mode\": \"writeback\"}",
6744 inbl, NULL, NULL));
6745
6746 // wait for maps to settle
6747 cluster.wait_for_latest_osdmap();
6748
6749 // create object
6750 {
6751 bufferlist bl;
6752 bl.append("hi there");
6753 ObjectWriteOperation op;
6754 op.write_full(bl);
6755 ASSERT_EQ(0, ioctx.operate("foo", &op));
6756 }
6757
6758 // verify the object is present in the cache tier
6759 {
6760 NObjectIterator it = cache_ioctx.nobjects_begin();
6761 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6762 ASSERT_TRUE(it->get_oid() == string("foo"));
6763 ++it;
6764 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6765 }
6766
6767 // verify the object is NOT present in the base tier
6768 {
6769 NObjectIterator it = ioctx.nobjects_begin();
6770 ASSERT_TRUE(it == ioctx.nobjects_end());
6771 }
6772
6773 // set omap
6774 {
6775 ObjectWriteOperation op;
6776 std::map<std::string, bufferlist> omap;
6777 omap["somekey"] = bufferlist();
6778 op.omap_set(omap);
6779 librados::AioCompletion *completion = cluster.aio_create_completion();
6780 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6781 completion->wait_for_complete();
7c673cae
FG		 6782 ASSERT_EQ(0, completion->get_return_value());
6783 completion->release();
6784 }
6785
6786 // flush
6787 {
6788 ObjectReadOperation op;
6789 op.cache_flush();
6790 librados::AioCompletion *completion = cluster.aio_create_completion();
6791 ASSERT_EQ(0, cache_ioctx.aio_operate(
6792 "foo", completion, &op,
6793 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 6794 completion->wait_for_complete();
7c673cae
FG		 6795 ASSERT_NE(0, completion->get_return_value());
6796 completion->release();
6797 }
6798
6799 // get omap
6800 {
6801 ObjectReadOperation op;
6802 bufferlist bl;
6803 int prval = 0;
6804 std::set<std::string> keys;
6805 keys.insert("somekey");
6806 std::map<std::string, bufferlist> map;
6807
6808 op.omap_get_vals_by_keys(keys, &map, &prval);
6809 librados::AioCompletion *completion = cluster.aio_create_completion();
6810 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op, &bl));
6811 sleep(5);
6812 bool completed = completion->is_complete();
6813 if( !completed ) {
6814 cache_ioctx.aio_cancel(completion);
6815 std::cerr << "Most probably test case will hang here, please reset manually" << std::endl;
6816 ASSERT_TRUE(completed); //in fact we are locked forever at test case shutdown unless fix for http://tracker.ceph.com/issues/14511 is applied. Seems there is no workaround for that
6817 }
6818 completion->release();
6819 }
6820 // verify still not in base tier
6821 {
6822 ASSERT_TRUE(ioctx.nobjects_begin() == ioctx.nobjects_end());
6823 }
6824 // erase it
6825 {
6826 ObjectWriteOperation op;
6827 op.remove();
6828 ASSERT_EQ(0, ioctx.operate("foo", &op));
6829 }
6830 // flush whiteout
6831 {
6832 ObjectReadOperation op;
6833 op.cache_flush();
6834 librados::AioCompletion *completion = cluster.aio_create_completion();
6835 ASSERT_EQ(0, cache_ioctx.aio_operate(
6836 "foo", completion, &op,
6837 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 6838 completion->wait_for_complete();
7c673cae
FG		 6839 ASSERT_EQ(0, completion->get_return_value());
6840 completion->release();
6841 }
6842 // evict
6843 {
6844 ObjectReadOperation op;
6845 op.cache_evict();
6846 librados::AioCompletion *completion = cluster.aio_create_completion();
6847 ASSERT_EQ(0, cache_ioctx.aio_operate(
6848 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6849 completion->wait_for_complete();
7c673cae
FG		 6850 ASSERT_EQ(0, completion->get_return_value());
6851 completion->release();
6852 }
6853
6854 // verify no longer in cache tier
6855 {
6856 NObjectIterator it = cache_ioctx.nobjects_begin();
6857 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6858 }
6859 // or base tier
6860 {
6861 NObjectIterator it = ioctx.nobjects_begin();
6862 ASSERT_TRUE(it == ioctx.nobjects_end());
6863 }
6864}
6865
6866TEST_F(LibRadosTwoPoolsECPP, Flush) {
6867 // configure cache
6868 bufferlist inbl;
6869 ASSERT_EQ(0, cluster.mon_command(
6870 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6871 "\", \"tierpool\": \"" + cache_pool_name +
6872 "\", \"force_nonempty\": \"--force-nonempty\" }",
6873 inbl, NULL, NULL));
6874 ASSERT_EQ(0, cluster.mon_command(
6875 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6876 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6877 inbl, NULL, NULL));
6878 ASSERT_EQ(0, cluster.mon_command(
6879 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6880 "\", \"mode\": \"writeback\"}",
6881 inbl, NULL, NULL));
6882
6883 // wait for maps to settle
6884 cluster.wait_for_latest_osdmap();
6885
6886 uint64_t user_version = 0;
6887
6888 // create object
6889 {
6890 bufferlist bl;
6891 bl.append("hi there");
6892 ObjectWriteOperation op;
6893 op.write_full(bl);
6894 ASSERT_EQ(0, ioctx.operate("foo", &op));
6895 }
6896
6897 // verify the object is present in the cache tier
6898 {
6899 NObjectIterator it = cache_ioctx.nobjects_begin();
6900 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6901 ASSERT_TRUE(it->get_oid() == string("foo"));
6902 ++it;
6903 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6904 }
6905
6906 // verify the object is NOT present in the base tier
6907 {
6908 NObjectIterator it = ioctx.nobjects_begin();
6909 ASSERT_TRUE(it == ioctx.nobjects_end());
6910 }
6911
6912 // verify dirty
6913 {
6914 bool dirty = false;
6915 int r = -1;
6916 ObjectReadOperation op;
6917 op.is_dirty(&dirty, &r);
6918 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
6919 ASSERT_TRUE(dirty);
6920 ASSERT_EQ(0, r);
6921 user_version = cache_ioctx.get_last_version();
6922 }
6923
6924 // pin
6925 {
6926 ObjectWriteOperation op;
6927 op.cache_pin();
6928 librados::AioCompletion *completion = cluster.aio_create_completion();
6929 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6930 completion->wait_for_complete();
7c673cae
FG		 6931 ASSERT_EQ(0, completion->get_return_value());
6932 completion->release();
6933 }
6934
6935 // flush the pinned object with -EPERM
6936 {
6937 ObjectReadOperation op;
6938 op.cache_try_flush();
6939 librados::AioCompletion *completion = cluster.aio_create_completion();
6940 ASSERT_EQ(0, cache_ioctx.aio_operate(
6941 "foo", completion, &op,
6942 librados::OPERATION_IGNORE_OVERLAY |
6943 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 6944 completion->wait_for_complete();
7c673cae
FG		 6945 ASSERT_EQ(-EPERM, completion->get_return_value());
6946 completion->release();
6947 }
6948
6949 // unpin
6950 {
6951 ObjectWriteOperation op;
6952 op.cache_unpin();
6953 librados::AioCompletion *completion = cluster.aio_create_completion();
6954 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6955 completion->wait_for_complete();
7c673cae
FG		 6956 ASSERT_EQ(0, completion->get_return_value());
6957 completion->release();
6958 }
6959
6960 // flush
6961 {
6962 ObjectReadOperation op;
6963 op.cache_flush();
6964 librados::AioCompletion *completion = cluster.aio_create_completion();
6965 ASSERT_EQ(0, cache_ioctx.aio_operate(
6966 "foo", completion, &op,
6967 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 6968 completion->wait_for_complete();
7c673cae
FG		 6969 ASSERT_EQ(0, completion->get_return_value());
6970 completion->release();
6971 }
6972
6973 // verify clean
6974 {
6975 bool dirty = false;
6976 int r = -1;
6977 ObjectReadOperation op;
6978 op.is_dirty(&dirty, &r);
6979 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
6980 ASSERT_FALSE(dirty);
6981 ASSERT_EQ(0, r);
6982 }
6983
6984 // verify in base tier
6985 {
6986 NObjectIterator it = ioctx.nobjects_begin();
6987 ASSERT_TRUE(it != ioctx.nobjects_end());
6988 ASSERT_TRUE(it->get_oid() == string("foo"));
6989 ++it;
6990 ASSERT_TRUE(it == ioctx.nobjects_end());
6991 }
6992
6993 // evict it
6994 {
6995 ObjectReadOperation op;
6996 op.cache_evict();
6997 librados::AioCompletion *completion = cluster.aio_create_completion();
6998 ASSERT_EQ(0, cache_ioctx.aio_operate(
6999 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7000 completion->wait_for_complete();
7c673cae
FG		 7001 ASSERT_EQ(0, completion->get_return_value());
7002 completion->release();
7003 }
7004
7005 // verify no longer in cache tier
7006 {
7007 NObjectIterator it = cache_ioctx.nobjects_begin();
7008 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7009 }
7010
7011 // read it again and verify the version is consistent
7012 {
7013 bufferlist bl;
7014 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
7015 ASSERT_EQ(user_version, cache_ioctx.get_last_version());
7016 }
7017
7018 // erase it
7019 {
7020 ObjectWriteOperation op;
7021 op.remove();
7022 ASSERT_EQ(0, ioctx.operate("foo", &op));
7023 }
7024
7025 // flush whiteout
7026 {
7027 ObjectReadOperation op;
7028 op.cache_flush();
7029 librados::AioCompletion *completion = cluster.aio_create_completion();
7030 ASSERT_EQ(0, cache_ioctx.aio_operate(
7031 "foo", completion, &op,
7032 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 7033 completion->wait_for_complete();
7c673cae
FG		 7034 ASSERT_EQ(0, completion->get_return_value());
7035 completion->release();
7036 }
7037
7038 // evict
7039 {
7040 ObjectReadOperation op;
7041 op.cache_evict();
7042 librados::AioCompletion *completion = cluster.aio_create_completion();
7043 ASSERT_EQ(0, cache_ioctx.aio_operate(
7044 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7045 completion->wait_for_complete();
7c673cae
FG		 7046 ASSERT_EQ(0, completion->get_return_value());
7047 completion->release();
7048 }
7049
7050 // verify no longer in cache tier
7051 {
7052 NObjectIterator it = cache_ioctx.nobjects_begin();
7053 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7054 }
7055 // or base tier
7056 {
7057 NObjectIterator it = ioctx.nobjects_begin();
7058 ASSERT_TRUE(it == ioctx.nobjects_end());
7059 }
7060}
7061
7062TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
7063 // configure cache
7064 bufferlist inbl;
7065 ASSERT_EQ(0, cluster.mon_command(
7066 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7067 "\", \"tierpool\": \"" + cache_pool_name +
7068 "\", \"force_nonempty\": \"--force-nonempty\" }",
7069 inbl, NULL, NULL));
7070 ASSERT_EQ(0, cluster.mon_command(
7071 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7072 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7073 inbl, NULL, NULL));
7074 ASSERT_EQ(0, cluster.mon_command(
7075 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7076 "\", \"mode\": \"writeback\"}",
7077 inbl, NULL, NULL));
7078
7079 // wait for maps to settle
7080 cluster.wait_for_latest_osdmap();
7081
7082 // create object
7083 {
7084 bufferlist bl;
7085 bl.append("a");
7086 ObjectWriteOperation op;
7087 op.write_full(bl);
7088 ASSERT_EQ(0, ioctx.operate("foo", &op));
7089 }
7090
7091 // create a snapshot, clone
7092 vector<uint64_t> my_snaps(1);
7093 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
7094 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
7095 my_snaps));
7096 {
7097 bufferlist bl;
7098 bl.append("b");
7099 ObjectWriteOperation op;
7100 op.write_full(bl);
7101 ASSERT_EQ(0, ioctx.operate("foo", &op));
7102 }
7103
7104 // and another
7105 my_snaps.resize(2);
7106 my_snaps[1] = my_snaps[0];
7107 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
7108 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
7109 my_snaps));
7110 {
7111 bufferlist bl;
7112 bl.append("c");
7113 ObjectWriteOperation op;
7114 op.write_full(bl);
7115 ASSERT_EQ(0, ioctx.operate("foo", &op));
7116 }
7117
7118 // verify the object is present in the cache tier
7119 {
7120 NObjectIterator it = cache_ioctx.nobjects_begin();
7121 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
7122 ASSERT_TRUE(it->get_oid() == string("foo"));
7123 ++it;
7124 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7125 }
7126
7127 // verify the object is NOT present in the base tier
7128 {
7129 NObjectIterator it = ioctx.nobjects_begin();
7130 ASSERT_TRUE(it == ioctx.nobjects_end());
7131 }
7132
7133 // flush on head (should fail)
7134 ioctx.snap_set_read(librados::SNAP_HEAD);
7135 {
7136 ObjectReadOperation op;
7137 op.cache_flush();
7138 librados::AioCompletion *completion = cluster.aio_create_completion();
7139 ASSERT_EQ(0, ioctx.aio_operate(
7140 "foo", completion, &op,
7141 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7142 completion->wait_for_complete();
7c673cae
FG		 7143 ASSERT_EQ(-EBUSY, completion->get_return_value());
7144 completion->release();
7145 }
7146 // flush on recent snap (should fail)
7147 ioctx.snap_set_read(my_snaps[0]);
7148 {
7149 ObjectReadOperation op;
7150 op.cache_flush();
7151 librados::AioCompletion *completion = cluster.aio_create_completion();
7152 ASSERT_EQ(0, ioctx.aio_operate(
7153 "foo", completion, &op,
7154 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7155 completion->wait_for_complete();
7c673cae
FG		 7156 ASSERT_EQ(-EBUSY, completion->get_return_value());
7157 completion->release();
7158 }
7159 // flush on oldest snap
7160 ioctx.snap_set_read(my_snaps[1]);
7161 {
7162 ObjectReadOperation op;
7163 op.cache_flush();
7164 librados::AioCompletion *completion = cluster.aio_create_completion();
7165 ASSERT_EQ(0, ioctx.aio_operate(
7166 "foo", completion, &op,
7167 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7168 completion->wait_for_complete();
7c673cae
FG		 7169 ASSERT_EQ(0, completion->get_return_value());
7170 completion->release();
7171 }
7172 // flush on next oldest snap
7173 ioctx.snap_set_read(my_snaps[0]);
7174 {
7175 ObjectReadOperation op;
7176 op.cache_flush();
7177 librados::AioCompletion *completion = cluster.aio_create_completion();
7178 ASSERT_EQ(0, ioctx.aio_operate(
7179 "foo", completion, &op,
7180 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7181 completion->wait_for_complete();
7c673cae
FG		 7182 ASSERT_EQ(0, completion->get_return_value());
7183 completion->release();
7184 }
7185 // flush on head
7186 ioctx.snap_set_read(librados::SNAP_HEAD);
7187 {
7188 ObjectReadOperation op;
7189 op.cache_flush();
7190 librados::AioCompletion *completion = cluster.aio_create_completion();
7191 ASSERT_EQ(0, ioctx.aio_operate(
7192 "foo", completion, &op,
7193 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7194 completion->wait_for_complete();
7c673cae
FG		 7195 ASSERT_EQ(0, completion->get_return_value());
7196 completion->release();
7197 }
7198
7199 // verify i can read the snaps from the cache pool
7200 ioctx.snap_set_read(librados::SNAP_HEAD);
7201 {
7202 bufferlist bl;
7203 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7204 ASSERT_EQ('c', bl[0]);
7205 }
7206 ioctx.snap_set_read(my_snaps[0]);
7207 {
7208 bufferlist bl;
7209 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7210 ASSERT_EQ('b', bl[0]);
7211 }
7212 ioctx.snap_set_read(my_snaps[1]);
7213 {
7214 bufferlist bl;
7215 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7216 ASSERT_EQ('a', bl[0]);
7217 }
7218
7219 // tear down tiers
7220 ASSERT_EQ(0, cluster.mon_command(
7221 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
7222 "\"}",
7223 inbl, NULL, NULL));
7224
7225 // wait for maps to settle
7226 cluster.wait_for_latest_osdmap();
7227
7228 // verify i can read the snaps from the base pool
7229 ioctx.snap_set_read(librados::SNAP_HEAD);
7230 {
7231 bufferlist bl;
7232 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7233 ASSERT_EQ('c', bl[0]);
7234 }
7235 ioctx.snap_set_read(my_snaps[0]);
7236 {
7237 bufferlist bl;
7238 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7239 ASSERT_EQ('b', bl[0]);
7240 }
7241 ioctx.snap_set_read(my_snaps[1]);
7242 {
7243 bufferlist bl;
7244 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7245 ASSERT_EQ('a', bl[0]);
7246 }
7247
7248 ASSERT_EQ(0, cluster.mon_command(
7249 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7250 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7251 inbl, NULL, NULL));
7252 cluster.wait_for_latest_osdmap();
7253
7254 // cleanup
7255 ioctx.selfmanaged_snap_remove(my_snaps[0]);
7256}
7257
7258TEST_F(LibRadosTierECPP, FlushWriteRaces) {
7259 Rados cluster;
7260 std::string pool_name = get_temp_pool_name();
7261 std::string cache_pool_name = pool_name + "-cache";
7262 ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
7263 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
7264 IoCtx cache_ioctx;
7265 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 7266 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 7267 IoCtx ioctx;
7268 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
7269
7270 // configure cache
7271 bufferlist inbl;
7272 ASSERT_EQ(0, cluster.mon_command(
7273 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7274 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
7275 inbl, NULL, NULL));
7276 ASSERT_EQ(0, cluster.mon_command(
7277 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7278 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7279 inbl, NULL, NULL));
7280 ASSERT_EQ(0, cluster.mon_command(
7281 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7282 "\", \"mode\": \"writeback\"}",
7283 inbl, NULL, NULL));
7284
7285 // wait for maps to settle
7286 cluster.wait_for_latest_osdmap();
7287
7288 // create/dirty object
7289 bufferlist bl;
7290 bl.append("hi there");
7291 {
7292 ObjectWriteOperation op;
7293 op.write_full(bl);
7294 ASSERT_EQ(0, ioctx.operate("foo", &op));
7295 }
7296
7297 // flush + write
7298 {
7299 ObjectReadOperation op;
7300 op.cache_flush();
7301 librados::AioCompletion *completion = cluster.aio_create_completion();
7302 ASSERT_EQ(0, cache_ioctx.aio_operate(
7303 "foo", completion, &op,
7304 librados::OPERATION_IGNORE_OVERLAY, NULL));
7305
7306 ObjectWriteOperation op2;
7307 op2.write_full(bl);
7308 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7309 ASSERT_EQ(0, ioctx.aio_operate(
7310 "foo", completion2, &op2, 0));
7311
9f95a23c
TL		 7312 completion->wait_for_complete();
7313 completion2->wait_for_complete();
7c673cae
FG		 7314 ASSERT_EQ(0, completion->get_return_value());
7315 ASSERT_EQ(0, completion2->get_return_value());
7316 completion->release();
7317 completion2->release();
7318 }
7319
7320 int tries = 1000;
7321 do {
7322 // create/dirty object
7323 {
7324 bufferlist bl;
7325 bl.append("hi there");
7326 ObjectWriteOperation op;
7327 op.write_full(bl);
7328 ASSERT_EQ(0, ioctx.operate("foo", &op));
7329 }
7330
7331 // try-flush + write
7332 {
7333 ObjectReadOperation op;
7334 op.cache_try_flush();
7335 librados::AioCompletion *completion = cluster.aio_create_completion();
7336 ASSERT_EQ(0, cache_ioctx.aio_operate(
7337 "foo", completion, &op,
7338 librados::OPERATION_IGNORE_OVERLAY |
7339 librados::OPERATION_SKIPRWLOCKS, NULL));
7340
7341 ObjectWriteOperation op2;
7342 op2.write_full(bl);
7343 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7344 ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
7345
9f95a23c
TL		 7346 completion->wait_for_complete();
7347 completion2->wait_for_complete();
7c673cae
FG		 7348 int r = completion->get_return_value();
7349 ASSERT_TRUE(r == -EBUSY || r == 0);
7350 ASSERT_EQ(0, completion2->get_return_value());
7351 completion->release();
7352 completion2->release();
7353 if (r == -EBUSY)
7354 break;
7355 cout << "didn't get EBUSY, trying again" << std::endl;
7356 }
7357 ASSERT_TRUE(--tries);
7358 } while (true);
7359
7360 // tear down tiers
7361 ASSERT_EQ(0, cluster.mon_command(
7362 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
7363 "\"}",
7364 inbl, NULL, NULL));
7365 ASSERT_EQ(0, cluster.mon_command(
7366 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
7367 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
7368 inbl, NULL, NULL));
7369
7370 // wait for maps to settle before next test
7371 cluster.wait_for_latest_osdmap();
7372
7373 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
7374 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
7375}
7376
7377TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
7378 // configure cache
7379 bufferlist inbl;
7380 ASSERT_EQ(0, cluster.mon_command(
7381 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7382 "\", \"tierpool\": \"" + cache_pool_name +
7383 "\", \"force_nonempty\": \"--force-nonempty\" }",
7384 inbl, NULL, NULL));
7385 ASSERT_EQ(0, cluster.mon_command(
7386 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7387 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7388 inbl, NULL, NULL));
7389 ASSERT_EQ(0, cluster.mon_command(
7390 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7391 "\", \"mode\": \"writeback\"}",
7392 inbl, NULL, NULL));
7393
7394 // wait for maps to settle
7395 cluster.wait_for_latest_osdmap();
7396
7397 // create/dirty object
7398 {
7399 bufferlist bl;
7400 bl.append("hi there");
7401 ObjectWriteOperation op;
7402 op.write_full(bl);
7403 ASSERT_EQ(0, ioctx.operate("foo", &op));
7404 }
7405
7406 // flush + flush
7407 {
7408 ObjectReadOperation op;
7409 op.cache_flush();
7410 librados::AioCompletion *completion = cluster.aio_create_completion();
7411 ASSERT_EQ(0, cache_ioctx.aio_operate(
7412 "foo", completion, &op,
7413 librados::OPERATION_IGNORE_OVERLAY, NULL));
7414
7415 ObjectReadOperation op2;
7416 op2.cache_flush();
7417 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7418 ASSERT_EQ(0, cache_ioctx.aio_operate(
7419 "foo", completion2, &op2,
7420 librados::OPERATION_IGNORE_OVERLAY, NULL));
7421
9f95a23c
TL		 7422 completion->wait_for_complete();
7423 completion2->wait_for_complete();
7c673cae
FG		 7424 ASSERT_EQ(0, completion->get_return_value());
7425 ASSERT_EQ(0, completion2->get_return_value());
7426 completion->release();
7427 completion2->release();
7428 }
7429
7430 // create/dirty object
7431 {
7432 bufferlist bl;
7433 bl.append("hi there");
7434 ObjectWriteOperation op;
7435 op.write_full(bl);
7436 ASSERT_EQ(0, ioctx.operate("foo", &op));
7437 }
7438
7439 // flush + try-flush
7440 {
7441 ObjectReadOperation op;
7442 op.cache_flush();
7443 librados::AioCompletion *completion = cluster.aio_create_completion();
7444 ASSERT_EQ(0, cache_ioctx.aio_operate(
7445 "foo", completion, &op,
7446 librados::OPERATION_IGNORE_OVERLAY, NULL));
7447
7448 ObjectReadOperation op2;
7449 op2.cache_try_flush();
7450 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7451 ASSERT_EQ(0, cache_ioctx.aio_operate(
7452 "foo", completion2, &op2,
7453 librados::OPERATION_IGNORE_OVERLAY |
7454 librados::OPERATION_SKIPRWLOCKS, NULL));
7455
9f95a23c
TL		 7456 completion->wait_for_complete();
7457 completion2->wait_for_complete();
7c673cae
FG		 7458 ASSERT_EQ(0, completion->get_return_value());
7459 ASSERT_EQ(0, completion2->get_return_value());
7460 completion->release();
7461 completion2->release();
7462 }
7463
7464 // create/dirty object
7465 int tries = 1000;
7466 do {
7467 {
7468 bufferlist bl;
7469 bl.append("hi there");
7470 ObjectWriteOperation op;
7471 op.write_full(bl);
7472 ASSERT_EQ(0, ioctx.operate("foo", &op));
7473 }
7474
7475 // try-flush + flush
7476 // (flush will not piggyback on try-flush)
7477 {
7478 ObjectReadOperation op;
7479 op.cache_try_flush();
7480 librados::AioCompletion *completion = cluster.aio_create_completion();
7481 ASSERT_EQ(0, cache_ioctx.aio_operate(
7482 "foo", completion, &op,
7483 librados::OPERATION_IGNORE_OVERLAY |
7484 librados::OPERATION_SKIPRWLOCKS, NULL));
7485
7486 ObjectReadOperation op2;
7487 op2.cache_flush();
7488 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7489 ASSERT_EQ(0, cache_ioctx.aio_operate(
7490 "foo", completion2, &op2,
7491 librados::OPERATION_IGNORE_OVERLAY, NULL));
7492
9f95a23c
TL		 7493 completion->wait_for_complete();
7494 completion2->wait_for_complete();
7c673cae
FG		 7495 int r = completion->get_return_value();
7496 ASSERT_TRUE(r == -EBUSY || r == 0);
7497 ASSERT_EQ(0, completion2->get_return_value());
7498 completion->release();
7499 completion2->release();
7500 if (r == -EBUSY)
7501 break;
7502 cout << "didn't get EBUSY, trying again" << std::endl;
7503 }
7504 ASSERT_TRUE(--tries);
7505 } while (true);
7506
7507 // create/dirty object
7508 {
7509 bufferlist bl;
7510 bl.append("hi there");
7511 ObjectWriteOperation op;
7512 op.write_full(bl);
7513 ASSERT_EQ(0, ioctx.operate("foo", &op));
7514 }
7515
7516 // try-flush + try-flush
7517 {
7518 ObjectReadOperation op;
7519 op.cache_try_flush();
7520 librados::AioCompletion *completion = cluster.aio_create_completion();
7521 ASSERT_EQ(0, cache_ioctx.aio_operate(
7522 "foo", completion, &op,
7523 librados::OPERATION_IGNORE_OVERLAY |
7524 librados::OPERATION_SKIPRWLOCKS, NULL));
7525
7526 ObjectReadOperation op2;
7527 op2.cache_try_flush();
7528 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7529 ASSERT_EQ(0, cache_ioctx.aio_operate(
7530 "foo", completion2, &op2,
7531 librados::OPERATION_IGNORE_OVERLAY |
7532 librados::OPERATION_SKIPRWLOCKS, NULL));
7533
9f95a23c
TL		 7534 completion->wait_for_complete();
7535 completion2->wait_for_complete();
7c673cae
FG		 7536 ASSERT_EQ(0, completion->get_return_value());
7537 ASSERT_EQ(0, completion2->get_return_value());
7538 completion->release();
7539 completion2->release();
7540 }
7541}
7542
7543TEST_F(LibRadosTwoPoolsECPP, TryFlushReadRace) {
7544 // configure cache
7545 bufferlist inbl;
7546 ASSERT_EQ(0, cluster.mon_command(
7547 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7548 "\", \"tierpool\": \"" + cache_pool_name +
7549 "\", \"force_nonempty\": \"--force-nonempty\" }",
7550 inbl, NULL, NULL));
7551 ASSERT_EQ(0, cluster.mon_command(
7552 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7553 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7554 inbl, NULL, NULL));
7555 ASSERT_EQ(0, cluster.mon_command(
7556 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7557 "\", \"mode\": \"writeback\"}",
7558 inbl, NULL, NULL));
7559
7560 // wait for maps to settle
7561 cluster.wait_for_latest_osdmap();
7562
7563 // create/dirty object
7564 {
7565 bufferlist bl;
7566 bl.append("hi there");
7567 bufferptr bp(4000000); // make it big!
7568 bp.zero();
7569 bl.append(bp);
7570 ObjectWriteOperation op;
7571 op.write_full(bl);
7572 ASSERT_EQ(0, ioctx.operate("foo", &op));
7573 }
7574
7575 // start a continuous stream of reads
7576 read_ioctx = &ioctx;
9f95a23c 7577 test_lock.lock();
7c673cae
FG		 7578 for (int i = 0; i < max_reads; ++i) {
7579 start_flush_read();
7580 num_reads++;
7581 }
9f95a23c 7582 test_lock.unlock();
7c673cae
FG		 7583
7584 // try-flush
7585 ObjectReadOperation op;
7586 op.cache_try_flush();
7587 librados::AioCompletion *completion = cluster.aio_create_completion();
7588 ASSERT_EQ(0, cache_ioctx.aio_operate(
7589 "foo", completion, &op,
7590 librados::OPERATION_IGNORE_OVERLAY |
7591 librados::OPERATION_SKIPRWLOCKS, NULL));
7592
9f95a23c 7593 completion->wait_for_complete();
7c673cae
FG		 7594 ASSERT_EQ(0, completion->get_return_value());
7595 completion->release();
7596
7597 // stop reads
9f95a23c
TL		 7598 std::unique_lock locker{test_lock};
7599 max_reads = 0;
7600 cond.wait(locker, [] { return num_reads == 0;});
7c673cae
FG		 7601}
7602
7603TEST_F(LibRadosTierECPP, CallForcesPromote) {
7604 Rados cluster;
7605 std::string pool_name = get_temp_pool_name();
7606 std::string cache_pool_name = pool_name + "-cache";
7607 ASSERT_EQ("", create_one_ec_pool_pp(pool_name, cluster));
7608 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
7609 IoCtx cache_ioctx;
7610 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 7611 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 7612 IoCtx ioctx;
7613 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
7614
7615 // configure cache
7616 bufferlist inbl;
7617 ASSERT_EQ(0, cluster.mon_command(
7618 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7619 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
7620 inbl, NULL, NULL));
7621 ASSERT_EQ(0, cluster.mon_command(
7622 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7623 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7624 inbl, NULL, NULL));
7625 ASSERT_EQ(0, cluster.mon_command(
7626 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7627 "\", \"mode\": \"writeback\"}",
7628 inbl, NULL, NULL));
7629
7630 // set things up such that the op would normally be proxied
7631 ASSERT_EQ(0, cluster.mon_command(
7632 set_pool_str(cache_pool_name, "hit_set_count", 2),
7633 inbl, NULL, NULL));
7634 ASSERT_EQ(0, cluster.mon_command(
7635 set_pool_str(cache_pool_name, "hit_set_period", 600),
7636 inbl, NULL, NULL));
7637 ASSERT_EQ(0, cluster.mon_command(
7638 set_pool_str(cache_pool_name, "hit_set_type",
7639 "explicit_object"),
7640 inbl, NULL, NULL));
7641 ASSERT_EQ(0, cluster.mon_command(
7642 set_pool_str(cache_pool_name, "min_read_recency_for_promote",
7643 "4"),
7644 inbl, NULL, NULL));
7645
7646 // wait for maps to settle
7647 cluster.wait_for_latest_osdmap();
7648
7649 // create/dirty object
7650 bufferlist bl;
7651 bl.append("hi there");
7652 {
7653 ObjectWriteOperation op;
7654 op.write_full(bl);
7655 ASSERT_EQ(0, ioctx.operate("foo", &op));
7656 }
7657
7658 // flush
7659 {
7660 ObjectReadOperation op;
7661 op.cache_flush();
7662 librados::AioCompletion *completion = cluster.aio_create_completion();
7663 ASSERT_EQ(0, cache_ioctx.aio_operate(
7664 "foo", completion, &op,
7665 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 7666 completion->wait_for_complete();
7c673cae
FG		 7667 ASSERT_EQ(0, completion->get_return_value());
7668 completion->release();
7669 }
7670
7671 // evict
7672 {
7673 ObjectReadOperation op;
7674 op.cache_evict();
7675 librados::AioCompletion *completion = cluster.aio_create_completion();
7676 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
7677 librados::OPERATION_IGNORE_CACHE,
7678 NULL));
9f95a23c 7679 completion->wait_for_complete();
7c673cae
FG		 7680 ASSERT_EQ(0, completion->get_return_value());
7681 completion->release();
7682 }
7683
7684 // call
7685 {
7686 ObjectReadOperation op;
7687 bufferlist bl;
7688 op.exec("rbd", "get_id", bl);
7689 bufferlist out;
7690 // should get EIO (not an rbd object), not -EOPNOTSUPP (we didn't promote)
7691 ASSERT_EQ(-5, ioctx.operate("foo", &op, &out));
7692 }
7693
7694 // make sure foo is back in the cache tier
7695 {
7696 NObjectIterator it = cache_ioctx.nobjects_begin();
7697 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
7698 ASSERT_TRUE(it->get_oid() == string("foo"));
7699 ++it;
7700 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7701 }
7702
7703 // tear down tiers
7704 ASSERT_EQ(0, cluster.mon_command(
7705 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
7706 "\"}",
7707 inbl, NULL, NULL));
7708 ASSERT_EQ(0, cluster.mon_command(
7709 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
7710 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
7711 inbl, NULL, NULL));
7712
7713 // wait for maps to settle before next test
7714 cluster.wait_for_latest_osdmap();
7715
7716 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
9f95a23c 7717 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, cluster));
7c673cae
FG		 7718}
7719
7720TEST_F(LibRadosTierECPP, HitSetNone) {
7721 {
7722 list< pair<time_t,time_t> > ls;
7723 AioCompletion *c = librados::Rados::aio_create_completion();
7724 ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
7725 c->wait_for_complete();
7726 ASSERT_EQ(0, c->get_return_value());
7727 ASSERT_TRUE(ls.empty());
7728 c->release();
7729 }
7730 {
7731 bufferlist bl;
7732 AioCompletion *c = librados::Rados::aio_create_completion();
7733 ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
7734 c->wait_for_complete();
7735 ASSERT_EQ(-ENOENT, c->get_return_value());
7736 c->release();
7737 }
7738}
7739
7740TEST_F(LibRadosTwoPoolsECPP, HitSetRead) {
7741 // make it a tier
7742 bufferlist inbl;
7743 ASSERT_EQ(0, cluster.mon_command(
7744 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7745 "\", \"tierpool\": \"" + cache_pool_name +
7746 "\", \"force_nonempty\": \"--force-nonempty\" }",
7747 inbl, NULL, NULL));
7748
7749 // enable hitset tracking for this pool
7750 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
7751 inbl, NULL, NULL));
7752 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
7753 inbl, NULL, NULL));
7754 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
7755 "explicit_object"),
7756 inbl, NULL, NULL));
7757
7758 // wait for maps to settle
7759 cluster.wait_for_latest_osdmap();
7760
7761 cache_ioctx.set_namespace("");
7762
7763 // keep reading until we see our object appear in the HitSet
7764 utime_t start = ceph_clock_now();
7765 utime_t hard_stop = start + utime_t(600, 0);
7766
7767 while (true) {
7768 utime_t now = ceph_clock_now();
7769 ASSERT_TRUE(now < hard_stop);
7770
7771 string name = "foo";
7772 uint32_t hash;
7773 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
7774 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
7775 cluster.pool_lookup(cache_pool_name.c_str()), "");
7776
7777 bufferlist bl;
7778 ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
7779
7780 bufferlist hbl;
7781 AioCompletion *c = librados::Rados::aio_create_completion();
7782 ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
7783 c->wait_for_complete();
7784 c->release();
7785
7786 if (hbl.length()) {
11fdf7f2 7787 auto p = hbl.cbegin();
7c673cae 7788 HitSet hs;
11fdf7f2 7789 decode(hs, p);
7c673cae
FG		 7790 if (hs.contains(oid)) {
7791 cout << "ok, hit_set contains " << oid << std::endl;
7792 break;
7793 }
7794 cout << "hmm, not in HitSet yet" << std::endl;
7795 } else {
7796 cout << "hmm, no HitSet yet" << std::endl;
7797 }
7798
7799 sleep(1);
7800 }
7801}
7802
7803// disable this test until hitset-get reliably works on EC pools
7804#if 0
7805TEST_F(LibRadosTierECPP, HitSetWrite) {
7806 int num_pg = _get_pg_num(cluster, pool_name);
11fdf7f2 7807 ceph_assert(num_pg > 0);
7c673cae
FG		 7808
7809 // enable hitset tracking for this pool
7810 bufferlist inbl;
7811 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 8),
7812 inbl, NULL, NULL));
7813 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", 600),
7814 inbl, NULL, NULL));
7815 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type",
7816 "explicit_hash"),
7817 inbl, NULL, NULL));
7818
7819 // wait for maps to settle
7820 cluster.wait_for_latest_osdmap();
7821
7822 ioctx.set_namespace("");
7823
7824 // do a bunch of writes
7825 for (int i=0; i<1000; ++i) {
7826 bufferlist bl;
7827 bl.append("a");
7828 ASSERT_EQ(0, ioctx.write(stringify(i), bl, 1, 0));
7829 }
7830
7831 // get HitSets
7832 std::map<int,HitSet> hitsets;
7833 for (int i=0; i<num_pg; ++i) {
7834 list< pair<time_t,time_t> > ls;
7835 AioCompletion *c = librados::Rados::aio_create_completion();
7836 ASSERT_EQ(0, ioctx.hit_set_list(i, c, &ls));
7837 c->wait_for_complete();
7838 c->release();
7839 std::cout << "pg " << i << " ls " << ls << std::endl;
7840 ASSERT_FALSE(ls.empty());
7841
7842 // get the latest
7843 c = librados::Rados::aio_create_completion();
7844 bufferlist bl;
7845 ASSERT_EQ(0, ioctx.hit_set_get(i, c, ls.back().first, &bl));
7846 c->wait_for_complete();
7847 c->release();
7848
7849 //std::cout << "bl len is " << bl.length() << "\n";
7850 //bl.hexdump(std::cout);
7851 //std::cout << std::endl;
7852
11fdf7f2
TL		 7853 auto p = bl.cbegin();
7854 decode(hitsets[i], p);
7c673cae
FG		 7855
7856 // cope with racing splits by refreshing pg_num
7857 if (i == num_pg - 1)
7858 num_pg = _get_pg_num(cluster, pool_name);
7859 }
7860
7861 for (int i=0; i<1000; ++i) {
7862 string n = stringify(i);
7863 uint32_t hash = ioctx.get_object_hash_position(n);
7864 hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
7865 cluster.pool_lookup(pool_name.c_str()), "");
7866 std::cout << "checking for " << oid << std::endl;
7867 bool found = false;
7868 for (int p=0; p<num_pg; ++p) {
7869 if (hitsets[p].contains(oid)) {
7870 found = true;
7871 break;
7872 }
7873 }
7874 ASSERT_TRUE(found);
7875 }
7876}
7877#endif
7878
7879TEST_F(LibRadosTwoPoolsECPP, HitSetTrim) {
7880 unsigned count = 3;
7881 unsigned period = 3;
7882
7883 // make it a tier
7884 bufferlist inbl;
7885 ASSERT_EQ(0, cluster.mon_command(
7886 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7887 "\", \"tierpool\": \"" + cache_pool_name +
7888 "\", \"force_nonempty\": \"--force-nonempty\" }",
7889 inbl, NULL, NULL));
7890
7891 // enable hitset tracking for this pool
7892 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
7893 inbl, NULL, NULL));
7894 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
7895 inbl, NULL, NULL));
7896 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
7897 inbl, NULL, NULL));
7898 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
7899 inbl, NULL, NULL));
7900
7901 // wait for maps to settle
7902 cluster.wait_for_latest_osdmap();
7903
7904 cache_ioctx.set_namespace("");
7905
7906 // do a bunch of writes and make sure the hitsets rotate
7907 utime_t start = ceph_clock_now();
7908 utime_t hard_stop = start + utime_t(count * period * 50, 0);
7909
7910 time_t first = 0;
7911 int bsize = alignment;
7912 char *buf = (char *)new char[bsize];
7913 memset(buf, 'f', bsize);
7914
7915 while (true) {
7916 string name = "foo";
7917 uint32_t hash;
7918 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
7919 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
7920
7921 bufferlist bl;
7922 bl.append(buf, bsize);
7923 ASSERT_EQ(0, cache_ioctx.append("foo", bl, bsize));
7924
7925 list<pair<time_t, time_t> > ls;
7926 AioCompletion *c = librados::Rados::aio_create_completion();
7927 ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
7928 c->wait_for_complete();
7929 c->release();
7930
7931 cout << " got ls " << ls << std::endl;
7932 if (!ls.empty()) {
7933 if (!first) {
7934 first = ls.front().first;
7935 cout << "first is " << first << std::endl;
7936 } else {
7937 if (ls.front().first != first) {
7938 cout << "first now " << ls.front().first << ", trimmed" << std::endl;
7939 break;
7940 }
7941 }
7942 }
7943
7944 utime_t now = ceph_clock_now();
7945 ASSERT_TRUE(now < hard_stop);
7946
7947 sleep(1);
7948 }
7949 delete[] buf;
7950}
7951
7952TEST_F(LibRadosTwoPoolsECPP, PromoteOn2ndRead) {
7953 // create object
7954 for (int i=0; i<20; ++i) {
7955 bufferlist bl;
7956 bl.append("hi there");
7957 ObjectWriteOperation op;
7958 op.write_full(bl);
7959 ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
7960 }
7961
7962 // configure cache
7963 bufferlist inbl;
7964 ASSERT_EQ(0, cluster.mon_command(
7965 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7966 "\", \"tierpool\": \"" + cache_pool_name +
7967 "\", \"force_nonempty\": \"--force-nonempty\" }",
7968 inbl, NULL, NULL));
7969 ASSERT_EQ(0, cluster.mon_command(
7970 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7971 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7972 inbl, NULL, NULL));
7973 ASSERT_EQ(0, cluster.mon_command(
7974 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7975 "\", \"mode\": \"writeback\"}",
7976 inbl, NULL, NULL));
7977
7978 // enable hitset tracking for this pool
7979 ASSERT_EQ(0, cluster.mon_command(
7980 set_pool_str(cache_pool_name, "hit_set_count", 2),
7981 inbl, NULL, NULL));
7982 ASSERT_EQ(0, cluster.mon_command(
7983 set_pool_str(cache_pool_name, "hit_set_period", 600),
7984 inbl, NULL, NULL));
7985 ASSERT_EQ(0, cluster.mon_command(
7986 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
7987 inbl, NULL, NULL));
7988 ASSERT_EQ(0, cluster.mon_command(
7989 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
7990 inbl, NULL, NULL));
7991 ASSERT_EQ(0, cluster.mon_command(
7992 set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
7993 inbl, NULL, NULL));
7994 ASSERT_EQ(0, cluster.mon_command(
7995 set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
7996 inbl, NULL, NULL));
7997
7998 // wait for maps to settle
7999 cluster.wait_for_latest_osdmap();
8000
8001 int fake = 0; // set this to non-zero to test spurious promotion,
8002 // e.g. from thrashing
8003 int attempt = 0;
8004 string obj;
8005 while (true) {
8006 // 1st read, don't trigger a promote
8007 obj = "foo" + stringify(attempt);
8008 cout << obj << std::endl;
8009 {
8010 bufferlist bl;
8011 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
8012 if (--fake >= 0) {
8013 sleep(1);
8014 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
8015 sleep(1);
8016 }
8017 }
8018
8019 // verify the object is NOT present in the cache tier
8020 {
8021 bool found = false;
8022 NObjectIterator it = cache_ioctx.nobjects_begin();
8023 while (it != cache_ioctx.nobjects_end()) {
8024 cout << " see " << it->get_oid() << std::endl;
8025 if (it->get_oid() == string(obj.c_str())) {
8026 found = true;
8027 break;
8028 }
8029 ++it;
8030 }
8031 if (!found)
8032 break;
8033 }
8034
8035 ++attempt;
8036 ASSERT_LE(attempt, 20);
8037 cout << "hrm, object is present in cache on attempt " << attempt
8038 << ", retrying" << std::endl;
8039 }
8040
8041 // Read until the object is present in the cache tier
8042 cout << "verifying " << obj << " is eventually promoted" << std::endl;
8043 while (true) {
8044 bufferlist bl;
8045 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
8046
8047 bool there = false;
8048 NObjectIterator it = cache_ioctx.nobjects_begin();
8049 while (it != cache_ioctx.nobjects_end()) {
8050 if (it->get_oid() == string(obj.c_str())) {
8051 there = true;
8052 break;
8053 }
8054 ++it;
8055 }
8056 if (there)
8057 break;
8058
8059 sleep(1);
8060 }
8061
8062 // tear down tiers
8063 ASSERT_EQ(0, cluster.mon_command(
8064 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
8065 "\"}",
8066 inbl, NULL, NULL));
8067 ASSERT_EQ(0, cluster.mon_command(
8068 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8069 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8070 inbl, NULL, NULL));
8071
8072 // wait for maps to settle before next test
8073 cluster.wait_for_latest_osdmap();
8074}
8075
8076TEST_F(LibRadosTwoPoolsECPP, ProxyRead) {
8077 // create object
8078 {
8079 bufferlist bl;
8080 bl.append("hi there");
8081 ObjectWriteOperation op;
8082 op.write_full(bl);
8083 ASSERT_EQ(0, ioctx.operate("foo", &op));
8084 }
8085
8086 // configure cache
8087 bufferlist inbl;
8088 ASSERT_EQ(0, cluster.mon_command(
8089 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8090 "\", \"tierpool\": \"" + cache_pool_name +
8091 "\", \"force_nonempty\": \"--force-nonempty\" }",
8092 inbl, NULL, NULL));
8093 ASSERT_EQ(0, cluster.mon_command(
8094 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8095 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8096 inbl, NULL, NULL));
8097 ASSERT_EQ(0, cluster.mon_command(
8098 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8099 "\", \"mode\": \"readproxy\"}",
8100 inbl, NULL, NULL));
8101
8102 // wait for maps to settle
8103 cluster.wait_for_latest_osdmap();
8104
8105 // read and verify the object
8106 {
8107 bufferlist bl;
8108 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
8109 ASSERT_EQ('h', bl[0]);
8110 }
8111
8112 // Verify 10 times the object is NOT present in the cache tier
8113 uint32_t i = 0;
8114 while (i++ < 10) {
8115 NObjectIterator it = cache_ioctx.nobjects_begin();
8116 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
8117 sleep(1);
8118 }
8119
8120 // tear down tiers
8121 ASSERT_EQ(0, cluster.mon_command(
8122 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
8123 "\"}",
8124 inbl, NULL, NULL));
8125 ASSERT_EQ(0, cluster.mon_command(
8126 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8127 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8128 inbl, NULL, NULL));
8129
8130 // wait for maps to settle before next test
8131 cluster.wait_for_latest_osdmap();
8132}
8133
8134TEST_F(LibRadosTwoPoolsECPP, CachePin) {
8135 // create object
8136 {
8137 bufferlist bl;
8138 bl.append("hi there");
8139 ObjectWriteOperation op;
8140 op.write_full(bl);
8141 ASSERT_EQ(0, ioctx.operate("foo", &op));
8142 }
8143 {
8144 bufferlist bl;
8145 bl.append("hi there");
8146 ObjectWriteOperation op;
8147 op.write_full(bl);
8148 ASSERT_EQ(0, ioctx.operate("bar", &op));
8149 }
8150 {
8151 bufferlist bl;
8152 bl.append("hi there");
8153 ObjectWriteOperation op;
8154 op.write_full(bl);
8155 ASSERT_EQ(0, ioctx.operate("baz", &op));
8156 }
8157 {
8158 bufferlist bl;
8159 bl.append("hi there");
8160 ObjectWriteOperation op;
8161 op.write_full(bl);
8162 ASSERT_EQ(0, ioctx.operate("bam", &op));
8163 }
8164
8165 // configure cache
8166 bufferlist inbl;
8167 ASSERT_EQ(0, cluster.mon_command(
8168 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8169 "\", \"tierpool\": \"" + cache_pool_name +
8170 "\", \"force_nonempty\": \"--force-nonempty\" }",
8171 inbl, NULL, NULL));
8172 ASSERT_EQ(0, cluster.mon_command(
8173 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8174 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8175 inbl, NULL, NULL));
8176 ASSERT_EQ(0, cluster.mon_command(
8177 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8178 "\", \"mode\": \"writeback\"}",
8179 inbl, NULL, NULL));
8180
8181 // wait for maps to settle
8182 cluster.wait_for_latest_osdmap();
8183
8184 // read, trigger promote
8185 {
8186 bufferlist bl;
8187 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
8188 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
8189 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
8190 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
8191 }
8192
8193 // verify the objects are present in the cache tier
8194 {
8195 NObjectIterator it = cache_ioctx.nobjects_begin();
8196 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
8197 for (uint32_t i = 0; i < 4; i++) {
8198 ASSERT_TRUE(it->get_oid() == string("foo") ||
8199 it->get_oid() == string("bar") ||
8200 it->get_oid() == string("baz") ||
8201 it->get_oid() == string("bam"));
8202 ++it;
8203 }
8204 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
8205 }
8206
8207 // pin objects
8208 {
8209 ObjectWriteOperation op;
8210 op.cache_pin();
8211 librados::AioCompletion *completion = cluster.aio_create_completion();
8212 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 8213 completion->wait_for_complete();
7c673cae
FG		 8214 ASSERT_EQ(0, completion->get_return_value());
8215 completion->release();
8216 }
8217 {
8218 ObjectWriteOperation op;
8219 op.cache_pin();
8220 librados::AioCompletion *completion = cluster.aio_create_completion();
8221 ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
9f95a23c 8222 completion->wait_for_complete();
7c673cae
FG		 8223 ASSERT_EQ(0, completion->get_return_value());
8224 completion->release();
8225 }
8226
8227 // enable agent
8228 ASSERT_EQ(0, cluster.mon_command(
8229 set_pool_str(cache_pool_name, "hit_set_count", 2),
8230 inbl, NULL, NULL));
8231 ASSERT_EQ(0, cluster.mon_command(
8232 set_pool_str(cache_pool_name, "hit_set_period", 600),
8233 inbl, NULL, NULL));
8234 ASSERT_EQ(0, cluster.mon_command(
8235 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
8236 inbl, NULL, NULL));
8237 ASSERT_EQ(0, cluster.mon_command(
8238 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
8239 inbl, NULL, NULL));
8240 ASSERT_EQ(0, cluster.mon_command(
8241 set_pool_str(cache_pool_name, "target_max_objects", 1),
8242 inbl, NULL, NULL));
8243
8244 sleep(10);
8245
8246 // Verify the pinned object 'foo' is not flushed/evicted
8247 uint32_t count = 0;
8248 while (true) {
8249 bufferlist bl;
8250 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
8251
8252 count = 0;
8253 NObjectIterator it = cache_ioctx.nobjects_begin();
8254 while (it != cache_ioctx.nobjects_end()) {
8255 ASSERT_TRUE(it->get_oid() == string("foo") ||
8256 it->get_oid() == string("bar") ||
8257 it->get_oid() == string("baz") ||
8258 it->get_oid() == string("bam"));
8259 ++count;
8260 ++it;
8261 }
8262 if (count == 2) {
8263 ASSERT_TRUE(it->get_oid() == string("foo") ||
8264 it->get_oid() == string("baz"));
8265 break;
8266 }
8267
8268 sleep(1);
8269 }
8270
8271 // tear down tiers
8272 ASSERT_EQ(0, cluster.mon_command(
8273 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
8274 "\"}",
8275 inbl, NULL, NULL));
8276 ASSERT_EQ(0, cluster.mon_command(
8277 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8278 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8279 inbl, NULL, NULL));
8280
8281 // wait for maps to settle before next test
8282 cluster.wait_for_latest_osdmap();
8283}
31f18b77 8284TEST_F(LibRadosTwoPoolsECPP, SetRedirectRead) {
31f18b77
FG		 8285 // create object
8286 {
8287 bufferlist bl;
8288 bl.append("hi there");
8289 ObjectWriteOperation op;
8290 op.write_full(bl);
8291 ASSERT_EQ(0, ioctx.operate("foo", &op));
8292 }
8293 {
8294 bufferlist bl;
8295 bl.append("there");
8296 ObjectWriteOperation op;
8297 op.write_full(bl);
8298 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
8299 }
8300
8301 // configure tier
8302 bufferlist inbl;
8303 ASSERT_EQ(0, cluster.mon_command(
8304 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8305 "\", \"tierpool\": \"" + cache_pool_name +
8306 "\", \"force_nonempty\": \"--force-nonempty\" }",
8307 inbl, NULL, NULL));
8308
8309 // wait for maps to settle
8310 cluster.wait_for_latest_osdmap();
8311
8312 {
8313 ObjectWriteOperation op;
8314 op.set_redirect("bar", cache_ioctx, 0);
8315 librados::AioCompletion *completion = cluster.aio_create_completion();
8316 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 8317 completion->wait_for_complete();
31f18b77
FG		 8318 ASSERT_EQ(0, completion->get_return_value());
8319 completion->release();
8320 }
8321 // read and verify the object
8322 {
8323 bufferlist bl;
8324 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
8325 ASSERT_EQ('t', bl[0]);
8326 }
8327
8328 ASSERT_EQ(0, cluster.mon_command(
8329 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8330 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8331 inbl, NULL, NULL));
8332
8333 // wait for maps to settle before next test
8334 cluster.wait_for_latest_osdmap();
8335}
11fdf7f2
TL		 8336
8337TEST_F(LibRadosTwoPoolsECPP, SetChunkRead) {
8338 // note: require >= mimic
8339
11fdf7f2 8340 {
f67539c2
TL		 8341 bufferlist bl;
8342 bl.append("there hi");
11fdf7f2 8343 ObjectWriteOperation op;
f67539c2
TL		 8344 op.write_full(bl);
8345 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
11fdf7f2 8346 }
f67539c2 8347
11fdf7f2
TL		 8348 {
8349 bufferlist bl;
f67539c2 8350 bl.append("There hi");
11fdf7f2
TL		 8351 ObjectWriteOperation op;
8352 op.write_full(bl);
f67539c2 8353 ASSERT_EQ(0, ioctx.operate("bar", &op));
11fdf7f2
TL		 8354 }
8355
11fdf7f2
TL		 8356 // wait for maps to settle
8357 cluster.wait_for_latest_osdmap();
8358
8359 // set_chunk
f67539c2
TL		 8360 manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 4, "bar", "foo");
8361
8362 // promote
11fdf7f2
TL		 8363 {
8364 ObjectWriteOperation op;
f67539c2 8365 op.tier_promote();
11fdf7f2 8366 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2 8367 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 8368 completion->wait_for_complete();
11fdf7f2
TL		 8369 ASSERT_EQ(0, completion->get_return_value());
8370 completion->release();
8371 }
8372
11fdf7f2
TL		 8373 // read and verify the object
8374 {
8375 bufferlist bl;
f67539c2 8376 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
11fdf7f2
TL		 8377 ASSERT_EQ('T', bl[0]);
8378 }
8379
11fdf7f2
TL		 8380 // wait for maps to settle before next test
8381 cluster.wait_for_latest_osdmap();
8382}
8383
8384TEST_F(LibRadosTwoPoolsECPP, ManifestPromoteRead) {
8385 // note: require >= mimic
8386
8387 // create object
8388 {
8389 bufferlist bl;
f67539c2 8390 bl.append("hiaa there");
11fdf7f2
TL		 8391 ObjectWriteOperation op;
8392 op.write_full(bl);
f67539c2 8393 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
11fdf7f2
TL		 8394 }
8395 {
f67539c2
TL		 8396 bufferlist bl;
8397 bl.append("base chunk");
11fdf7f2 8398 ObjectWriteOperation op;
f67539c2
TL		 8399 op.write_full(bl);
8400 ASSERT_EQ(0, cache_ioctx.operate("foo-chunk", &op));
11fdf7f2
TL		 8401 }
8402 {
8403 bufferlist bl;
f67539c2 8404 bl.append("HIaa there");
11fdf7f2
TL		 8405 ObjectWriteOperation op;
8406 op.write_full(bl);
f67539c2 8407 ASSERT_EQ(0, ioctx.operate("bar", &op));
11fdf7f2
TL		 8408 }
8409 {
8410 bufferlist bl;
8411 bl.append("BASE CHUNK");
8412 ObjectWriteOperation op;
8413 op.write_full(bl);
f67539c2 8414 ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
11fdf7f2
TL		 8415 }
8416
11fdf7f2
TL		 8417 // set-redirect
8418 {
8419 ObjectWriteOperation op;
f67539c2 8420 op.set_redirect("bar", ioctx, 0);
11fdf7f2 8421 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2 8422 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 8423 completion->wait_for_complete();
11fdf7f2
TL		 8424 ASSERT_EQ(0, completion->get_return_value());
8425 completion->release();
8426 }
8427 // set-chunk
f67539c2 8428 manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "bar-chunk", "foo-chunk");
11fdf7f2
TL		 8429 // promote
8430 {
8431 ObjectWriteOperation op;
8432 op.tier_promote();
8433 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2 8434 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 8435 completion->wait_for_complete();
11fdf7f2
TL		 8436 ASSERT_EQ(0, completion->get_return_value());
8437 completion->release();
8438 }
8439 // read and verify the object (redirect)
8440 {
8441 bufferlist bl;
f67539c2 8442 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
11fdf7f2
TL		 8443 ASSERT_EQ('H', bl[0]);
8444 }
8445 // promote
8446 {
8447 ObjectWriteOperation op;
8448 op.tier_promote();
8449 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2 8450 ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 8451 completion->wait_for_complete();
11fdf7f2
TL		 8452 ASSERT_EQ(0, completion->get_return_value());
8453 completion->release();
8454 }
8455 // read and verify the object
8456 {
8457 bufferlist bl;
f67539c2 8458 ASSERT_EQ(1, cache_ioctx.read("foo-chunk", bl, 1, 0));
11fdf7f2
TL		 8459 ASSERT_EQ('B', bl[0]);
8460 }
8461
11fdf7f2
TL		 8462 // wait for maps to settle before next test
8463 cluster.wait_for_latest_osdmap();
8464}
8465
f67539c2
TL		 8466TEST_F(LibRadosTwoPoolsECPP, TrySetDedupTier) {
8467 // note: require >= mimic
8468
8469 bufferlist inbl;
8470 ASSERT_EQ(-EOPNOTSUPP, cluster.mon_command(
8471 set_pool_str(pool_name, "dedup_tier", cache_pool_name),
8472 inbl, NULL, NULL));
8473}
8474
11fdf7f2
TL		 8475TEST_F(LibRadosTwoPoolsPP, PropagateBaseTierError) {
8476 // write object to base tier
8477 bufferlist omap_bl;
8478 encode(static_cast<uint32_t>(0U), omap_bl);
8479
8480 ObjectWriteOperation op1;
8481 op1.omap_set({{"somekey", omap_bl}});
8482 ASSERT_EQ(0, ioctx.operate("propagate-base-tier-error", &op1));
8483
8484 // configure cache
8485 bufferlist inbl;
8486 ASSERT_EQ(0, cluster.mon_command(
8487 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8488 "\", \"tierpool\": \"" + cache_pool_name +
8489 "\", \"force_nonempty\": \"--force-nonempty\" }",
8490 inbl, NULL, NULL));
8491 ASSERT_EQ(0, cluster.mon_command(
8492 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8493 "\", \"mode\": \"writeback\"}",
8494 inbl, NULL, NULL));
8495 ASSERT_EQ(0, cluster.mon_command(
8496 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8497 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8498 inbl, NULL, NULL));
8499
8500 ASSERT_EQ(0, cluster.mon_command(
8501 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
8502 inbl, NULL, NULL));
8503 ASSERT_EQ(0, cluster.mon_command(
8504 set_pool_str(cache_pool_name, "hit_set_count", 1),
8505 inbl, NULL, NULL));
8506 ASSERT_EQ(0, cluster.mon_command(
8507 set_pool_str(cache_pool_name, "hit_set_period", 600),
8508 inbl, NULL, NULL));
8509 ASSERT_EQ(0, cluster.mon_command(
8510 set_pool_str(cache_pool_name, "target_max_objects", 250),
8511 inbl, NULL, NULL));
8512
8513 // wait for maps to settle
8514 cluster.wait_for_latest_osdmap();
8515
8516 // guarded op should fail so expect error to propagate to cache tier
8517 bufferlist test_omap_bl;
8518 encode(static_cast<uint32_t>(1U), test_omap_bl);
8519
8520 ObjectWriteOperation op2;
8521 op2.omap_cmp({{"somekey", {test_omap_bl, CEPH_OSD_CMPXATTR_OP_EQ}}}, nullptr);
8522 op2.omap_set({{"somekey", test_omap_bl}});
8523
8524 ASSERT_EQ(-ECANCELED, ioctx.operate("propagate-base-tier-error", &op2));
8525}
9f95a23c
TL		 8526
8527TEST_F(LibRadosTwoPoolsPP, HelloWriteReturn) {
8528 // configure cache
8529 bufferlist inbl;
8530 ASSERT_EQ(0, cluster.mon_command(
8531 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8532 "\", \"tierpool\": \"" + cache_pool_name +
8533 "\", \"force_nonempty\": \"--force-nonempty\" }",
8534 inbl, NULL, NULL));
8535 ASSERT_EQ(0, cluster.mon_command(
8536 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8537 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8538 inbl, NULL, NULL));
8539 ASSERT_EQ(0, cluster.mon_command(
8540 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8541 "\", \"mode\": \"writeback\"}",
8542 inbl, NULL, NULL));
8543
8544 // set things up such that the op would normally be proxied
8545 ASSERT_EQ(0, cluster.mon_command(
8546 set_pool_str(cache_pool_name, "hit_set_count", 2),
8547 inbl, NULL, NULL));
8548 ASSERT_EQ(0, cluster.mon_command(
8549 set_pool_str(cache_pool_name, "hit_set_period", 600),
8550 inbl, NULL, NULL));
8551 ASSERT_EQ(0, cluster.mon_command(
8552 set_pool_str(cache_pool_name, "hit_set_type",
8553 "explicit_object"),
8554 inbl, NULL, NULL));
8555 ASSERT_EQ(0, cluster.mon_command(
8556 set_pool_str(cache_pool_name, "min_read_recency_for_promote",
8557 "10000"),
8558 inbl, NULL, NULL));
8559
8560 // wait for maps to settle
8561 cluster.wait_for_latest_osdmap();
8562
8563 // this *will* return data due to the RETURNVEC flag
8564 {
8565 bufferlist in, out;
8566 int rval;
8567 ObjectWriteOperation o;
8568 o.exec("hello", "write_return_data", in, &out, &rval);
8569 librados::AioCompletion *completion = cluster.aio_create_completion();
8570 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
8571 librados::OPERATION_RETURNVEC));
8572 completion->wait_for_complete();
8573 ASSERT_EQ(42, completion->get_return_value());
8574 ASSERT_EQ(42, rval);
8575 out.hexdump(std::cout);
8576 ASSERT_EQ("you might see this", std::string(out.c_str(), out.length()));
8577 }
8578
8579 // this will overflow because the return data is too big
8580 {
8581 bufferlist in, out;
8582 int rval;
8583 ObjectWriteOperation o;
8584 o.exec("hello", "write_too_much_return_data", in, &out, &rval);
8585 librados::AioCompletion *completion = cluster.aio_create_completion();
8586 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
8587 librados::OPERATION_RETURNVEC));
8588 completion->wait_for_complete();
8589 ASSERT_EQ(-EOVERFLOW, completion->get_return_value());
8590 ASSERT_EQ(-EOVERFLOW, rval);
8591 ASSERT_EQ("", std::string(out.c_str(), out.length()));
8592 }
8593}
Ceph