]> git.proxmox.com Git - ceph.git/blame - ceph/src/test/librados/tier_cxx.cc
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
import quincy beta 17.1.0
[ceph.git] / ceph / src / test / librados / tier_cxx.cc
CommitLineData
7c673cae
FG		 1// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2// vim: ts=8 sw=2 smarttab
3#include "gtest/gtest.h"
4
5#include "mds/mdstypes.h"
6#include "include/buffer.h"
7#include "include/rbd_types.h"
7c673cae
FG		 8#include "include/rados/librados.hpp"
9#include "include/stringify.h"
10#include "include/types.h"
11#include "global/global_context.h"
12#include "common/Cond.h"
11fdf7f2
TL		 13#include "test/librados/test_cxx.h"
14#include "test/librados/testcase_cxx.h"
7c673cae 15#include "json_spirit/json_spirit.h"
11fdf7f2 16#include "cls/cas/cls_cas_ops.h"
f67539c2 17#include "cls/cas/cls_cas_internal.h"
7c673cae
FG		 18
19#include "osd/HitSet.h"
20
21#include <errno.h>
22#include <map>
23#include <sstream>
24#include <string>
25
f67539c2
TL		 26#include "cls/cas/cls_cas_client.h"
27#include "cls/cas/cls_cas_internal.h"
28
20effc67 29using namespace std;
7c673cae 30using namespace librados;
7c673cae
FG		 31
32typedef RadosTestPP LibRadosTierPP;
33typedef RadosTestECPP LibRadosTierECPP;
34
35void flush_evict_all(librados::Rados& cluster, librados::IoCtx& cache_ioctx)
36{
37 bufferlist inbl;
38 cache_ioctx.set_namespace(all_nspaces);
39 for (NObjectIterator it = cache_ioctx.nobjects_begin();
40 it != cache_ioctx.nobjects_end(); ++it) {
41 cache_ioctx.locator_set_key(it->get_locator());
42 cache_ioctx.set_namespace(it->get_nspace());
43 {
44 ObjectReadOperation op;
45 op.cache_flush();
46 librados::AioCompletion *completion = cluster.aio_create_completion();
47 cache_ioctx.aio_operate(
48 it->get_oid(), completion, &op,
49 librados::OPERATION_IGNORE_OVERLAY, NULL);
9f95a23c 50 completion->wait_for_complete();
7c673cae
FG		 51 completion->get_return_value();
52 completion->release();
53 }
54 {
55 ObjectReadOperation op;
56 op.cache_evict();
57 librados::AioCompletion *completion = cluster.aio_create_completion();
58 cache_ioctx.aio_operate(
59 it->get_oid(), completion, &op,
60 librados::OPERATION_IGNORE_OVERLAY, NULL);
9f95a23c 61 completion->wait_for_complete();
7c673cae
FG		 62 completion->get_return_value();
63 completion->release();
64 }
65 }
66}
67
9f95a23c
TL		 68static string _get_required_osd_release(Rados& cluster)
69{
70 bufferlist inbl;
71 string cmd = string("{\"prefix\": \"osd dump\",\"format\":\"json\"}");
72 bufferlist outbl;
73 int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
74 ceph_assert(r >= 0);
75 string outstr(outbl.c_str(), outbl.length());
76 json_spirit::Value v;
77 if (!json_spirit::read(outstr, v)) {
78 cerr <<" unable to parse json " << outstr << std::endl;
79 return "";
80 }
81
82 json_spirit::Object& o = v.get_obj();
83 for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
84 json_spirit::Pair& p = o[i];
85 if (p.name_ == "require_osd_release") {
86 cout << "require_osd_release = " << p.value_.get_str() << std::endl;
87 return p.value_.get_str();
88 }
89 }
90 cerr << "didn't find require_osd_release in " << outstr << std::endl;
91 return "";
92}
93
f67539c2
TL		 94void manifest_set_chunk(Rados& cluster, librados::IoCtx& src_ioctx,
95 librados::IoCtx& tgt_ioctx,
96 uint64_t src_offset, uint64_t length,
97 std::string src_oid, std::string tgt_oid)
98{
99 ObjectReadOperation op;
100 op.set_chunk(src_offset, length, src_ioctx, src_oid, 0,
101 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
102 librados::AioCompletion *completion = cluster.aio_create_completion();
103 ASSERT_EQ(0, tgt_ioctx.aio_operate(tgt_oid, completion, &op,
104 librados::OPERATION_IGNORE_CACHE, NULL));
105 completion->wait_for_complete();
106 ASSERT_EQ(0, completion->get_return_value());
107 completion->release();
108}
109
110#include "common/ceph_crypto.h"
111using ceph::crypto::SHA1;
112#include "rgw/rgw_common.h"
113
114void check_fp_oid_refcount(librados::IoCtx& ioctx, std::string foid, uint64_t count,
115 std::string fp_algo = NULL)
116{
117 bufferlist t;
118 int size = foid.length();
119 if (fp_algo == "sha1") {
120 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
121 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
122 SHA1 sha1_gen;
123 sha1_gen.Update((const unsigned char *)foid.c_str(), size);
124 sha1_gen.Final(fingerprint);
125 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
126 ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
127 } else if (fp_algo.empty()) {
128 ioctx.getxattr(foid, CHUNK_REFCOUNT_ATTR, t);
129 } else if (!fp_algo.empty()) {
130 ceph_assert(0 == "unrecognized fingerprint algorithm");
131 }
132
133 chunk_refs_t refs;
134 try {
135 auto iter = t.cbegin();
136 decode(refs, iter);
137 } catch (buffer::error& err) {
138 ASSERT_TRUE(0);
139 }
a4b75251 140 ASSERT_LE(count, refs.count());
f67539c2
TL		 141}
142
143string get_fp_oid(string oid, std::string fp_algo = NULL)
144{
145 if (fp_algo == "sha1") {
146 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
147 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
148 SHA1 sha1_gen;
149 int size = oid.length();
150 sha1_gen.Update((const unsigned char *)oid.c_str(), size);
151 sha1_gen.Final(fingerprint);
152 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
153 return string(p_str);
154 }
155
156 return string();
157}
158
159void is_intended_refcount_state(librados::IoCtx& src_ioctx,
160 std::string src_oid,
161 librados::IoCtx& dst_ioctx,
162 std::string dst_oid,
163 int expected_refcount)
164{
165 int src_refcount = 0, dst_refcount = 0;
166 bufferlist t;
167 int r = dst_ioctx.getxattr(dst_oid, CHUNK_REFCOUNT_ATTR, t);
168 if (r == -ENOENT) {
169 dst_refcount = 0;
170 } else {
171 chunk_refs_t refs;
172 try {
173 auto iter = t.cbegin();
174 decode(refs, iter);
175 } catch (buffer::error& err) {
176 ceph_assert(0);
177 }
178 dst_refcount = refs.count();
179 }
20effc67
TL		 180 int tries = 0;
181 for (; tries < 30; ++tries) {
f67539c2
TL		 182 r = cls_cas_references_chunk(src_ioctx, src_oid, dst_oid);
183 if (r == -ENOENT || r == -ENOLINK) {
184 src_refcount = 0;
185 } else if (r == -EBUSY) {
20effc67 186 sleep(20);
f67539c2
TL		 187 continue;
188 } else {
189 src_refcount = r;
190 }
191 break;
192 }
20effc67 193 ASSERT_TRUE(tries < 30);
f67539c2
TL		 194 ASSERT_TRUE(src_refcount >= 0);
195 ASSERT_TRUE(src_refcount == expected_refcount);
196 ASSERT_TRUE(src_refcount <= dst_refcount);
197}
198
7c673cae
FG		 199class LibRadosTwoPoolsPP : public RadosTestPP
200{
201public:
202 LibRadosTwoPoolsPP() {};
203 ~LibRadosTwoPoolsPP() override {};
204protected:
205 static void SetUpTestCase() {
206 pool_name = get_temp_pool_name();
207 ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
208 }
209 static void TearDownTestCase() {
210 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
211 }
212 static std::string cache_pool_name;
213
214 void SetUp() override {
215 cache_pool_name = get_temp_pool_name();
216 ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
217 RadosTestPP::SetUp();
c07f9fc5 218
7c673cae 219 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 220 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 221 cache_ioctx.set_namespace(nspace);
222 }
223 void TearDown() override {
224 // flush + evict cache
225 flush_evict_all(cluster, cache_ioctx);
226
227 bufferlist inbl;
228 // tear down tiers
229 ASSERT_EQ(0, cluster.mon_command(
230 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
231 "\"}",
232 inbl, NULL, NULL));
233 ASSERT_EQ(0, cluster.mon_command(
234 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
235 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
236 inbl, NULL, NULL));
237
238 // wait for maps to settle before next test
239 cluster.wait_for_latest_osdmap();
240
241 RadosTestPP::TearDown();
242
243 cleanup_default_namespace(cache_ioctx);
244 cleanup_namespace(cache_ioctx, nspace);
245
246 cache_ioctx.close();
247 ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
248 }
249 librados::IoCtx cache_ioctx;
250};
251
252class Completions
253{
254public:
255 Completions() = default;
256 librados::AioCompletion* getCompletion() {
257 librados::AioCompletion* comp = librados::Rados::aio_create_completion();
258 m_completions.push_back(comp);
259 return comp;
260 }
261
262 ~Completions() {
263 for (auto& comp : m_completions) {
264 comp->release();
265 }
266 }
267
268private:
269 vector<librados::AioCompletion *> m_completions;
270};
271
272Completions completions;
273
274std::string LibRadosTwoPoolsPP::cache_pool_name;
275
276TEST_F(LibRadosTierPP, Dirty) {
277 {
278 ObjectWriteOperation op;
279 op.undirty();
280 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
281 }
282 {
283 ObjectWriteOperation op;
284 op.create(true);
285 ASSERT_EQ(0, ioctx.operate("foo", &op));
286 }
287 {
288 bool dirty = false;
289 int r = -1;
290 ObjectReadOperation op;
291 op.is_dirty(&dirty, &r);
292 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
293 ASSERT_TRUE(dirty);
294 ASSERT_EQ(0, r);
295 }
296 {
297 ObjectWriteOperation op;
298 op.undirty();
299 ASSERT_EQ(0, ioctx.operate("foo", &op));
300 }
301 {
302 ObjectWriteOperation op;
303 op.undirty();
304 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
305 }
306 {
307 bool dirty = false;
308 int r = -1;
309 ObjectReadOperation op;
310 op.is_dirty(&dirty, &r);
311 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
312 ASSERT_FALSE(dirty);
313 ASSERT_EQ(0, r);
314 }
315 {
316 ObjectWriteOperation op;
317 op.truncate(0); // still a write even tho it is a no-op
318 ASSERT_EQ(0, ioctx.operate("foo", &op));
319 }
320 {
321 bool dirty = false;
322 int r = -1;
323 ObjectReadOperation op;
324 op.is_dirty(&dirty, &r);
325 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
326 ASSERT_TRUE(dirty);
327 ASSERT_EQ(0, r);
328 }
329}
330
331TEST_F(LibRadosTwoPoolsPP, Overlay) {
332 // create objects
333 {
334 bufferlist bl;
335 bl.append("base");
336 ObjectWriteOperation op;
337 op.write_full(bl);
338 ASSERT_EQ(0, ioctx.operate("foo", &op));
339 }
340 {
341 bufferlist bl;
342 bl.append("cache");
343 ObjectWriteOperation op;
344 op.write_full(bl);
345 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
346 }
347
348 // configure cache
349 bufferlist inbl;
350 ASSERT_EQ(0, cluster.mon_command(
351 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
352 "\", \"tierpool\": \"" + cache_pool_name +
353 "\", \"force_nonempty\": \"--force-nonempty\" }",
354 inbl, NULL, NULL));
355 ASSERT_EQ(0, cluster.mon_command(
356 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
357 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
358 inbl, NULL, NULL));
359
360 // wait for maps to settle
361 cluster.wait_for_latest_osdmap();
362
363 // by default, the overlay sends us to cache pool
364 {
365 bufferlist bl;
366 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
367 ASSERT_EQ('c', bl[0]);
368 }
369 {
370 bufferlist bl;
371 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
372 ASSERT_EQ('c', bl[0]);
373 }
374
375 // unless we say otherwise
376 {
377 bufferlist bl;
378 ObjectReadOperation op;
379 op.read(0, 1, &bl, NULL);
380 librados::AioCompletion *completion = cluster.aio_create_completion();
381 ASSERT_EQ(0, ioctx.aio_operate(
382 "foo", completion, &op,
383 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 384 completion->wait_for_complete();
7c673cae
FG		 385 ASSERT_EQ(0, completion->get_return_value());
386 completion->release();
387 ASSERT_EQ('b', bl[0]);
388 }
389}
390
391TEST_F(LibRadosTwoPoolsPP, Promote) {
392 // create object
393 {
394 bufferlist bl;
395 bl.append("hi there");
396 ObjectWriteOperation op;
397 op.write_full(bl);
398 ASSERT_EQ(0, ioctx.operate("foo", &op));
399 }
400
401 // configure cache
402 bufferlist inbl;
403 ASSERT_EQ(0, cluster.mon_command(
404 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
405 "\", \"tierpool\": \"" + cache_pool_name +
406 "\", \"force_nonempty\": \"--force-nonempty\" }",
407 inbl, NULL, NULL));
408 ASSERT_EQ(0, cluster.mon_command(
409 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
410 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
411 inbl, NULL, NULL));
412 ASSERT_EQ(0, cluster.mon_command(
413 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
414 "\", \"mode\": \"writeback\"}",
415 inbl, NULL, NULL));
416
417 // wait for maps to settle
418 cluster.wait_for_latest_osdmap();
419
420 // read, trigger a promote
421 {
422 bufferlist bl;
423 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
424 }
425
426 // read, trigger a whiteout
427 {
428 bufferlist bl;
429 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
430 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
431 }
432
433 // verify the object is present in the cache tier
434 {
435 NObjectIterator it = cache_ioctx.nobjects_begin();
436 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
437 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
438 ++it;
439 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
440 ++it;
441 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
442 }
443}
444
445TEST_F(LibRadosTwoPoolsPP, PromoteSnap) {
446 // create object
447 {
448 bufferlist bl;
449 bl.append("hi there");
450 ObjectWriteOperation op;
451 op.write_full(bl);
452 ASSERT_EQ(0, ioctx.operate("foo", &op));
453 }
454 {
455 bufferlist bl;
456 bl.append("hi there");
457 ObjectWriteOperation op;
458 op.write_full(bl);
459 ASSERT_EQ(0, ioctx.operate("bar", &op));
460 }
461 {
462 bufferlist bl;
463 bl.append("hi there");
464 ObjectWriteOperation op;
465 op.write_full(bl);
466 ASSERT_EQ(0, ioctx.operate("baz", &op));
467 }
468 {
469 bufferlist bl;
470 bl.append("hi there");
471 ObjectWriteOperation op;
472 op.write_full(bl);
473 ASSERT_EQ(0, ioctx.operate("bam", &op));
474 }
475
476 // create a snapshot, clone
477 vector<uint64_t> my_snaps(1);
478 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
479 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
480 my_snaps));
481 {
482 bufferlist bl;
483 bl.append("ciao!");
484 ObjectWriteOperation op;
485 op.write_full(bl);
486 ASSERT_EQ(0, ioctx.operate("foo", &op));
487 }
488 {
489 bufferlist bl;
490 bl.append("ciao!");
491 ObjectWriteOperation op;
492 op.write_full(bl);
493 ASSERT_EQ(0, ioctx.operate("bar", &op));
494 }
495 {
496 ObjectWriteOperation op;
497 op.remove();
498 ASSERT_EQ(0, ioctx.operate("baz", &op));
499 }
500 {
501 bufferlist bl;
502 bl.append("ciao!");
503 ObjectWriteOperation op;
504 op.write_full(bl);
505 ASSERT_EQ(0, ioctx.operate("bam", &op));
506 }
507
508 // configure cache
509 bufferlist inbl;
510 ASSERT_EQ(0, cluster.mon_command(
511 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
512 "\", \"tierpool\": \"" + cache_pool_name +
513 "\", \"force_nonempty\": \"--force-nonempty\" }",
514 inbl, NULL, NULL));
515 ASSERT_EQ(0, cluster.mon_command(
516 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
517 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
518 inbl, NULL, NULL));
519 ASSERT_EQ(0, cluster.mon_command(
520 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
521 "\", \"mode\": \"writeback\"}",
522 inbl, NULL, NULL));
523
524 // wait for maps to settle
525 cluster.wait_for_latest_osdmap();
526
527 // read, trigger a promote on the head
528 {
529 bufferlist bl;
530 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
531 ASSERT_EQ('c', bl[0]);
532 }
533 {
534 bufferlist bl;
535 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
536 ASSERT_EQ('c', bl[0]);
537 }
538
539 ioctx.snap_set_read(my_snaps[0]);
540
541 // read foo snap
542 {
543 bufferlist bl;
544 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
545 ASSERT_EQ('h', bl[0]);
546 }
547
548 // read bar snap
549 {
550 bufferlist bl;
551 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
552 ASSERT_EQ('h', bl[0]);
553 }
554
555 // read baz snap
556 {
557 bufferlist bl;
558 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
559 ASSERT_EQ('h', bl[0]);
560 }
561
562 ioctx.snap_set_read(librados::SNAP_HEAD);
563
564 // read foo
565 {
566 bufferlist bl;
567 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
568 ASSERT_EQ('c', bl[0]);
569 }
570
571 // read bar
572 {
573 bufferlist bl;
574 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
575 ASSERT_EQ('c', bl[0]);
576 }
577
578 // read baz
579 {
580 bufferlist bl;
581 ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
582 }
583
584 // cleanup
585 ioctx.selfmanaged_snap_remove(my_snaps[0]);
586}
587
588TEST_F(LibRadosTwoPoolsPP, PromoteSnapScrub) {
589 int num = 100;
590
591 // create objects
592 for (int i=0; i<num; ++i) {
593 bufferlist bl;
594 bl.append("hi there");
595 ObjectWriteOperation op;
596 op.write_full(bl);
597 ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
598 }
599
600 vector<uint64_t> my_snaps;
601 for (int snap=0; snap<4; ++snap) {
602 // create a snapshot, clone
603 vector<uint64_t> ns(1);
604 ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
605 my_snaps.swap(ns);
606 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
607 cout << "my_snaps " << my_snaps << std::endl;
608 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
609 my_snaps));
610 for (int i=0; i<num; ++i) {
611 bufferlist bl;
612 bl.append(string("ciao! snap") + stringify(snap));
613 ObjectWriteOperation op;
614 op.write_full(bl);
615 ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
616 }
617 }
618
619 // configure cache
620 bufferlist inbl;
621 ASSERT_EQ(0, cluster.mon_command(
622 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
623 "\", \"tierpool\": \"" + cache_pool_name +
624 "\", \"force_nonempty\": \"--force-nonempty\" }",
625 inbl, NULL, NULL));
626 ASSERT_EQ(0, cluster.mon_command(
627 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
628 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
629 inbl, NULL, NULL));
630 ASSERT_EQ(0, cluster.mon_command(
631 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
632 "\", \"mode\": \"writeback\"}",
633 inbl, NULL, NULL));
634
635 // wait for maps to settle
636 cluster.wait_for_latest_osdmap();
637
638 // read, trigger a promote on _some_ heads to make sure we handle cases
639 // where snaps are present and where they are not.
640 cout << "promoting some heads" << std::endl;
641 for (int i=0; i<num; ++i) {
642 if (i % 5 == 0 || i > num - 3) {
643 bufferlist bl;
644 ASSERT_EQ(1, ioctx.read(string("foo") + stringify(i), bl, 1, 0));
645 ASSERT_EQ('c', bl[0]);
646 }
647 }
648
649 for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
650 cout << "promoting from clones for snap " << my_snaps[snap] << std::endl;
651 ioctx.snap_set_read(my_snaps[snap]);
652
653 // read some snaps, semi-randomly
654 for (int i=0; i<50; ++i) {
655 bufferlist bl;
656 string o = string("foo") + stringify((snap * i * 137) % 80);
657 //cout << o << std::endl;
658 ASSERT_EQ(1, ioctx.read(o, bl, 1, 0));
659 }
660 }
661
662 // ok, stop and scrub this pool (to make sure scrub can handle
663 // missing clones in the cache tier).
664 {
665 IoCtx cache_ioctx;
666 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
667 for (int i=0; i<10; ++i) {
668 do {
669 ostringstream ss;
670 ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
671 << cache_ioctx.get_id() << "." << i
672 << "\"}";
673 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
674 if (r == -ENOENT || // in case mgr osdmap is stale
675 r == -EAGAIN) {
676 sleep(5);
677 continue;
678 }
679 } while (false);
680 }
681
682 // give it a few seconds to go. this is sloppy but is usually enough time
683 cout << "waiting for scrubs..." << std::endl;
684 sleep(30);
685 cout << "done waiting" << std::endl;
686 }
687
688 ioctx.snap_set_read(librados::SNAP_HEAD);
689
690 //cleanup
691 for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
692 ioctx.selfmanaged_snap_remove(my_snaps[snap]);
693 }
694}
695
696TEST_F(LibRadosTwoPoolsPP, PromoteSnapTrimRace) {
697 // create object
698 {
699 bufferlist bl;
700 bl.append("hi there");
701 ObjectWriteOperation op;
702 op.write_full(bl);
703 ASSERT_EQ(0, ioctx.operate("foo", &op));
704 }
705
706 // create a snapshot, clone
707 vector<uint64_t> my_snaps(1);
708 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
709 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
710 my_snaps));
711 {
712 bufferlist bl;
713 bl.append("ciao!");
714 ObjectWriteOperation op;
715 op.write_full(bl);
716 ASSERT_EQ(0, ioctx.operate("foo", &op));
717 }
718
719 // configure cache
720 bufferlist inbl;
721 ASSERT_EQ(0, cluster.mon_command(
722 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
723 "\", \"tierpool\": \"" + cache_pool_name +
724 "\", \"force_nonempty\": \"--force-nonempty\" }",
725 inbl, NULL, NULL));
726 ASSERT_EQ(0, cluster.mon_command(
727 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
728 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
729 inbl, NULL, NULL));
730 ASSERT_EQ(0, cluster.mon_command(
731 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
732 "\", \"mode\": \"writeback\"}",
733 inbl, NULL, NULL));
734
735 // wait for maps to settle
736 cluster.wait_for_latest_osdmap();
737
738 // delete the snap
739 ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
740
741 ioctx.snap_set_read(my_snaps[0]);
742
9f95a23c
TL		 743 // read foo snap. the OSD may or may not realize that this snap has
744 // been logically deleted; either response is valid.
7c673cae
FG		 745 {
746 bufferlist bl;
9f95a23c
TL		 747 int r = ioctx.read("foo", bl, 1, 0);
748 ASSERT_TRUE(r == 1 || r == -ENOENT);
7c673cae
FG		 749 }
750
751 // cleanup
752 ioctx.selfmanaged_snap_remove(my_snaps[0]);
753}
754
755TEST_F(LibRadosTwoPoolsPP, Whiteout) {
756 // create object
757 {
758 bufferlist bl;
759 bl.append("hi there");
760 ObjectWriteOperation op;
761 op.write_full(bl);
762 ASSERT_EQ(0, ioctx.operate("foo", &op));
763 }
764
765 // configure cache
766 bufferlist inbl;
767 ASSERT_EQ(0, cluster.mon_command(
768 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
769 "\", \"tierpool\": \"" + cache_pool_name +
770 "\", \"force_nonempty\": \"--force-nonempty\" }",
771 inbl, NULL, NULL));
772 ASSERT_EQ(0, cluster.mon_command(
773 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
774 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
775 inbl, NULL, NULL));
776 ASSERT_EQ(0, cluster.mon_command(
777 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
778 "\", \"mode\": \"writeback\"}",
779 inbl, NULL, NULL));
780
781 // wait for maps to settle
782 cluster.wait_for_latest_osdmap();
783
784 // create some whiteouts, verify they behave
785 {
786 ObjectWriteOperation op;
787 op.assert_exists();
788 op.remove();
789 ASSERT_EQ(0, ioctx.operate("foo", &op));
790 }
791
792 {
793 ObjectWriteOperation op;
794 op.assert_exists();
795 op.remove();
796 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
797 }
798 {
799 ObjectWriteOperation op;
800 op.assert_exists();
801 op.remove();
802 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
803 }
804
805 // verify the whiteouts are there in the cache tier
806 {
807 NObjectIterator it = cache_ioctx.nobjects_begin();
808 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
809 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
810 ++it;
811 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
812 ++it;
813 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
814 }
815
816 // delete a whiteout and verify it goes away
817 ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
818 {
819 ObjectWriteOperation op;
820 op.remove();
821 librados::AioCompletion *completion = cluster.aio_create_completion();
822 ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
823 librados::OPERATION_IGNORE_CACHE));
9f95a23c 824 completion->wait_for_complete();
7c673cae
FG		 825 ASSERT_EQ(0, completion->get_return_value());
826 completion->release();
827
828 NObjectIterator it = cache_ioctx.nobjects_begin();
829 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
830 ASSERT_TRUE(it->get_oid() == string("foo"));
831 ++it;
832 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
833 }
834
835 // recreate an object and verify we can read it
836 {
837 bufferlist bl;
838 bl.append("hi there");
839 ObjectWriteOperation op;
840 op.write_full(bl);
841 ASSERT_EQ(0, ioctx.operate("foo", &op));
842 }
843 {
844 bufferlist bl;
845 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
846 ASSERT_EQ('h', bl[0]);
847 }
848}
849
850TEST_F(LibRadosTwoPoolsPP, WhiteoutDeleteCreate) {
851 // configure cache
852 bufferlist inbl;
853 ASSERT_EQ(0, cluster.mon_command(
854 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
855 "\", \"tierpool\": \"" + cache_pool_name +
856 "\", \"force_nonempty\": \"--force-nonempty\" }",
857 inbl, NULL, NULL));
858 ASSERT_EQ(0, cluster.mon_command(
859 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
860 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
861 inbl, NULL, NULL));
862 ASSERT_EQ(0, cluster.mon_command(
863 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
864 "\", \"mode\": \"writeback\"}",
865 inbl, NULL, NULL));
866
867 // wait for maps to settle
868 cluster.wait_for_latest_osdmap();
869
870 // create an object
871 {
872 bufferlist bl;
873 bl.append("foo");
874 ASSERT_EQ(0, ioctx.write_full("foo", bl));
875 }
876
877 // do delete + create operation
878 {
879 ObjectWriteOperation op;
880 op.remove();
881 bufferlist bl;
882 bl.append("bar");
883 op.write_full(bl);
884 ASSERT_EQ(0, ioctx.operate("foo", &op));
885 }
886
887 // verify it still "exists" (w/ new content)
888 {
889 bufferlist bl;
890 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
891 ASSERT_EQ('b', bl[0]);
892 }
893}
894
895TEST_F(LibRadosTwoPoolsPP, Evict) {
896 // create object
897 {
898 bufferlist bl;
899 bl.append("hi there");
900 ObjectWriteOperation op;
901 op.write_full(bl);
902 ASSERT_EQ(0, ioctx.operate("foo", &op));
903 }
904
905 // configure cache
906 bufferlist inbl;
907 ASSERT_EQ(0, cluster.mon_command(
908 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
909 "\", \"tierpool\": \"" + cache_pool_name +
910 "\", \"force_nonempty\": \"--force-nonempty\" }",
911 inbl, NULL, NULL));
912 ASSERT_EQ(0, cluster.mon_command(
913 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
914 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
915 inbl, NULL, NULL));
916 ASSERT_EQ(0, cluster.mon_command(
917 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
918 "\", \"mode\": \"writeback\"}",
919 inbl, NULL, NULL));
920
921 // wait for maps to settle
922 cluster.wait_for_latest_osdmap();
923
924 // read, trigger a promote
925 {
926 bufferlist bl;
927 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
928 }
929
930 // read, trigger a whiteout, and a dirty object
931 {
932 bufferlist bl;
933 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
934 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
935 ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
936 }
937
938 // verify the object is present in the cache tier
939 {
940 NObjectIterator it = cache_ioctx.nobjects_begin();
941 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
942 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
943 ++it;
944 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
945 ++it;
946 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
947 }
948
949 // pin
950 {
951 ObjectWriteOperation op;
952 op.cache_pin();
953 librados::AioCompletion *completion = cluster.aio_create_completion();
954 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 955 completion->wait_for_complete();
7c673cae
FG		 956 ASSERT_EQ(0, completion->get_return_value());
957 completion->release();
958 }
959
960 // evict the pinned object with -EPERM
961 {
962 ObjectReadOperation op;
963 op.cache_evict();
964 librados::AioCompletion *completion = cluster.aio_create_completion();
965 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
966 librados::OPERATION_IGNORE_CACHE,
967 NULL));
9f95a23c 968 completion->wait_for_complete();
7c673cae
FG		 969 ASSERT_EQ(-EPERM, completion->get_return_value());
970 completion->release();
971 }
972
973 // unpin
974 {
975 ObjectWriteOperation op;
976 op.cache_unpin();
977 librados::AioCompletion *completion = cluster.aio_create_completion();
978 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 979 completion->wait_for_complete();
7c673cae
FG		 980 ASSERT_EQ(0, completion->get_return_value());
981 completion->release();
982 }
983
984 // flush
985 {
986 ObjectReadOperation op;
987 op.cache_flush();
988 librados::AioCompletion *completion = cluster.aio_create_completion();
989 ASSERT_EQ(0, cache_ioctx.aio_operate(
990 "foo", completion, &op,
991 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 992 completion->wait_for_complete();
7c673cae
FG		 993 ASSERT_EQ(0, completion->get_return_value());
994 completion->release();
995 }
996
997 // verify clean
998 {
999 bool dirty = false;
1000 int r = -1;
1001 ObjectReadOperation op;
1002 op.is_dirty(&dirty, &r);
1003 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1004 ASSERT_FALSE(dirty);
1005 ASSERT_EQ(0, r);
1006 }
1007
1008 // evict
1009 {
1010 ObjectReadOperation op;
1011 op.cache_evict();
1012 librados::AioCompletion *completion = cluster.aio_create_completion();
1013 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
1014 librados::OPERATION_IGNORE_CACHE,
1015 NULL));
9f95a23c 1016 completion->wait_for_complete();
7c673cae
FG		 1017 ASSERT_EQ(0, completion->get_return_value());
1018 completion->release();
1019 }
1020 {
1021 ObjectReadOperation op;
1022 op.cache_evict();
1023 librados::AioCompletion *completion = cluster.aio_create_completion();
1024 ASSERT_EQ(0, cache_ioctx.aio_operate(
1025 "foo", completion, &op,
1026 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1027 completion->wait_for_complete();
7c673cae
FG		 1028 ASSERT_EQ(0, completion->get_return_value());
1029 completion->release();
1030 }
1031 {
1032 ObjectReadOperation op;
1033 op.cache_evict();
1034 librados::AioCompletion *completion = cluster.aio_create_completion();
1035 ASSERT_EQ(0, cache_ioctx.aio_operate(
1036 "bar", completion, &op,
1037 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1038 completion->wait_for_complete();
7c673cae
FG		 1039 ASSERT_EQ(-EBUSY, completion->get_return_value());
1040 completion->release();
1041 }
1042}
1043
1044TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
1045 // create object
1046 {
1047 bufferlist bl;
1048 bl.append("hi there");
1049 ObjectWriteOperation op;
1050 op.write_full(bl);
1051 ASSERT_EQ(0, ioctx.operate("foo", &op));
1052 }
1053 {
1054 bufferlist bl;
1055 bl.append("hi there");
1056 ObjectWriteOperation op;
1057 op.write_full(bl);
1058 ASSERT_EQ(0, ioctx.operate("bar", &op));
1059 }
1060 {
1061 bufferlist bl;
1062 bl.append("hi there");
1063 ObjectWriteOperation op;
1064 op.write_full(bl);
1065 ASSERT_EQ(0, ioctx.operate("baz", &op));
1066 }
1067 {
1068 bufferlist bl;
1069 bl.append("hi there");
1070 ObjectWriteOperation op;
1071 op.write_full(bl);
1072 ASSERT_EQ(0, ioctx.operate("bam", &op));
1073 }
1074
1075 // create a snapshot, clone
1076 vector<uint64_t> my_snaps(1);
1077 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1078 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1079 my_snaps));
1080 {
1081 bufferlist bl;
1082 bl.append("ciao!");
1083 ObjectWriteOperation op;
1084 op.write_full(bl);
1085 ASSERT_EQ(0, ioctx.operate("foo", &op));
1086 }
1087 {
1088 bufferlist bl;
1089 bl.append("ciao!");
1090 ObjectWriteOperation op;
1091 op.write_full(bl);
1092 ASSERT_EQ(0, ioctx.operate("bar", &op));
1093 }
1094 {
1095 ObjectWriteOperation op;
1096 op.remove();
1097 ASSERT_EQ(0, ioctx.operate("baz", &op));
1098 }
1099 {
1100 bufferlist bl;
1101 bl.append("ciao!");
1102 ObjectWriteOperation op;
1103 op.write_full(bl);
1104 ASSERT_EQ(0, ioctx.operate("bam", &op));
1105 }
1106
1107 // configure cache
1108 bufferlist inbl;
1109 ASSERT_EQ(0, cluster.mon_command(
1110 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1111 "\", \"tierpool\": \"" + cache_pool_name +
1112 "\", \"force_nonempty\": \"--force-nonempty\" }",
1113 inbl, NULL, NULL));
1114 ASSERT_EQ(0, cluster.mon_command(
1115 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1116 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1117 inbl, NULL, NULL));
1118 ASSERT_EQ(0, cluster.mon_command(
1119 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1120 "\", \"mode\": \"writeback\"}",
1121 inbl, NULL, NULL));
1122
1123 // wait for maps to settle
1124 cluster.wait_for_latest_osdmap();
1125
1126 // read, trigger a promote on the head
1127 {
1128 bufferlist bl;
1129 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1130 ASSERT_EQ('c', bl[0]);
1131 }
1132 {
1133 bufferlist bl;
1134 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
1135 ASSERT_EQ('c', bl[0]);
1136 }
1137
1138 // evict bam
1139 {
1140 ObjectReadOperation op;
1141 op.cache_evict();
1142 librados::AioCompletion *completion = cluster.aio_create_completion();
1143 ASSERT_EQ(0, cache_ioctx.aio_operate(
1144 "bam", completion, &op,
1145 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1146 completion->wait_for_complete();
7c673cae
FG		 1147 ASSERT_EQ(0, completion->get_return_value());
1148 completion->release();
1149 }
1150 {
1151 bufferlist bl;
1152 ObjectReadOperation op;
1153 op.read(1, 0, &bl, NULL);
1154 librados::AioCompletion *completion = cluster.aio_create_completion();
1155 ASSERT_EQ(0, cache_ioctx.aio_operate(
1156 "bam", completion, &op,
1157 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1158 completion->wait_for_complete();
7c673cae
FG		 1159 ASSERT_EQ(-ENOENT, completion->get_return_value());
1160 completion->release();
1161 }
1162
1163 // read foo snap
1164 ioctx.snap_set_read(my_snaps[0]);
1165 {
1166 bufferlist bl;
1167 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1168 ASSERT_EQ('h', bl[0]);
1169 }
1170
1171 // evict foo snap
1172 {
1173 ObjectReadOperation op;
1174 op.cache_evict();
1175 librados::AioCompletion *completion = cluster.aio_create_completion();
1176 ASSERT_EQ(0, ioctx.aio_operate(
1177 "foo", completion, &op,
1178 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1179 completion->wait_for_complete();
7c673cae
FG		 1180 ASSERT_EQ(0, completion->get_return_value());
1181 completion->release();
1182 }
1183 // snap is gone...
1184 {
1185 bufferlist bl;
1186 ObjectReadOperation op;
1187 op.read(1, 0, &bl, NULL);
1188 librados::AioCompletion *completion = cluster.aio_create_completion();
1189 ASSERT_EQ(0, ioctx.aio_operate(
1190 "foo", completion, &op,
1191 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1192 completion->wait_for_complete();
7c673cae
FG		 1193 ASSERT_EQ(-ENOENT, completion->get_return_value());
1194 completion->release();
1195 }
1196 // head is still there...
1197 ioctx.snap_set_read(librados::SNAP_HEAD);
1198 {
1199 bufferlist bl;
1200 ObjectReadOperation op;
1201 op.read(1, 0, &bl, NULL);
1202 librados::AioCompletion *completion = cluster.aio_create_completion();
1203 ASSERT_EQ(0, ioctx.aio_operate(
1204 "foo", completion, &op,
1205 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1206 completion->wait_for_complete();
7c673cae
FG		 1207 ASSERT_EQ(0, completion->get_return_value());
1208 completion->release();
1209 }
1210
1211 // promote head + snap of bar
1212 ioctx.snap_set_read(librados::SNAP_HEAD);
1213 {
1214 bufferlist bl;
1215 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
1216 ASSERT_EQ('c', bl[0]);
1217 }
1218 ioctx.snap_set_read(my_snaps[0]);
1219 {
1220 bufferlist bl;
1221 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
1222 ASSERT_EQ('h', bl[0]);
1223 }
1224
1225 // evict bar head (fail)
1226 ioctx.snap_set_read(librados::SNAP_HEAD);
1227 {
1228 ObjectReadOperation op;
1229 op.cache_evict();
1230 librados::AioCompletion *completion = cluster.aio_create_completion();
1231 ASSERT_EQ(0, ioctx.aio_operate(
1232 "bar", completion, &op,
1233 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1234 completion->wait_for_complete();
7c673cae
FG		 1235 ASSERT_EQ(-EBUSY, completion->get_return_value());
1236 completion->release();
1237 }
1238
1239 // evict bar snap
1240 ioctx.snap_set_read(my_snaps[0]);
1241 {
1242 ObjectReadOperation op;
1243 op.cache_evict();
1244 librados::AioCompletion *completion = cluster.aio_create_completion();
1245 ASSERT_EQ(0, ioctx.aio_operate(
1246 "bar", completion, &op,
1247 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1248 completion->wait_for_complete();
7c673cae
FG		 1249 ASSERT_EQ(0, completion->get_return_value());
1250 completion->release();
1251 }
1252 // ...and then head
1253 ioctx.snap_set_read(librados::SNAP_HEAD);
1254 {
1255 bufferlist bl;
1256 ObjectReadOperation op;
1257 op.read(1, 0, &bl, NULL);
1258 librados::AioCompletion *completion = cluster.aio_create_completion();
1259 ASSERT_EQ(0, ioctx.aio_operate(
1260 "bar", completion, &op,
1261 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1262 completion->wait_for_complete();
7c673cae
FG		 1263 ASSERT_EQ(0, completion->get_return_value());
1264 completion->release();
1265 }
1266 {
1267 ObjectReadOperation op;
1268 op.cache_evict();
1269 librados::AioCompletion *completion = cluster.aio_create_completion();
1270 ASSERT_EQ(0, ioctx.aio_operate(
1271 "bar", completion, &op,
1272 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1273 completion->wait_for_complete();
7c673cae
FG		 1274 ASSERT_EQ(0, completion->get_return_value());
1275 completion->release();
1276 }
1277
1278 // cleanup
1279 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1280}
1281
1282// this test case reproduces http://tracker.ceph.com/issues/8629
1283TEST_F(LibRadosTwoPoolsPP, EvictSnap2) {
1284 // create object
1285 {
1286 bufferlist bl;
1287 bl.append("hi there");
1288 ObjectWriteOperation op;
1289 op.write_full(bl);
1290 ASSERT_EQ(0, ioctx.operate("foo", &op));
1291 }
1292 // create a snapshot, clone
1293 vector<uint64_t> my_snaps(1);
1294 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1295 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1296 my_snaps));
1297 {
1298 bufferlist bl;
1299 bl.append("ciao!");
1300 ObjectWriteOperation op;
1301 op.write_full(bl);
1302 ASSERT_EQ(0, ioctx.operate("foo", &op));
1303 }
1304 // configure cache
1305 bufferlist inbl;
1306 ASSERT_EQ(0, cluster.mon_command(
1307 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1308 "\", \"tierpool\": \"" + cache_pool_name +
1309 "\", \"force_nonempty\": \"--force-nonempty\" }",
1310 inbl, NULL, NULL));
1311 ASSERT_EQ(0, cluster.mon_command(
1312 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1313 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1314 inbl, NULL, NULL));
1315 ASSERT_EQ(0, cluster.mon_command(
1316 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1317 "\", \"mode\": \"writeback\"}",
1318 inbl, NULL, NULL));
1319
1320 // wait for maps to settle
1321 cluster.wait_for_latest_osdmap();
1322
1323 // read, trigger a promote on the head
1324 {
1325 bufferlist bl;
1326 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1327 ASSERT_EQ('c', bl[0]);
1328 }
1329
1330 // evict
1331 {
1332 ObjectReadOperation op;
1333 op.cache_evict();
1334 librados::AioCompletion *completion = cluster.aio_create_completion();
1335 ASSERT_EQ(0, cache_ioctx.aio_operate(
1336 "foo", completion, &op,
1337 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1338 completion->wait_for_complete();
7c673cae
FG		 1339 ASSERT_EQ(0, completion->get_return_value());
1340 completion->release();
1341 }
1342
1343 // verify the snapdir is not present in the cache pool
1344 {
1345 ObjectReadOperation op;
1346 librados::snap_set_t snapset;
1347 op.list_snaps(&snapset, NULL);
1348 ioctx.snap_set_read(librados::SNAP_DIR);
1349 librados::AioCompletion *completion = cluster.aio_create_completion();
1350 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op,
1351 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1352 completion->wait_for_complete();
7c673cae
FG		 1353 ASSERT_EQ(-ENOENT, completion->get_return_value());
1354 completion->release();
1355 }
1356}
1357
11fdf7f2
TL		 1358//This test case reproduces http://tracker.ceph.com/issues/17445
1359TEST_F(LibRadosTwoPoolsPP, ListSnap){
1360 // Create object
1361 {
1362 bufferlist bl;
1363 bl.append("hi there");
1364 ObjectWriteOperation op;
1365 op.write_full(bl);
1366 ASSERT_EQ(0, ioctx.operate("foo", &op));
1367 }
1368 {
1369 bufferlist bl;
1370 bl.append("hi there");
1371 ObjectWriteOperation op;
1372 op.write_full(bl);
1373 ASSERT_EQ(0, ioctx.operate("bar", &op));
1374 }
1375 {
1376 bufferlist bl;
1377 bl.append("hi there");
1378 ObjectWriteOperation op;
1379 op.write_full(bl);
1380 ASSERT_EQ(0, ioctx.operate("baz", &op));
1381 }
1382 {
1383 bufferlist bl;
1384 bl.append("hi there");
1385 ObjectWriteOperation op;
1386 op.write_full(bl);
1387 ASSERT_EQ(0, ioctx.operate("bam", &op));
1388 }
1389
1390 // Create a snapshot, clone
1391 vector<uint64_t> my_snaps(1);
1392 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1393 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1394 my_snaps));
1395 {
1396 bufferlist bl;
1397 bl.append("ciao!");
1398 ObjectWriteOperation op;
1399 op.write_full(bl);
1400 ASSERT_EQ(0, ioctx.operate("foo", &op));
1401 }
1402 {
1403 bufferlist bl;
1404 bl.append("ciao!");
1405 ObjectWriteOperation op;
1406 op.write_full(bl);
1407 ASSERT_EQ(0, ioctx.operate("bar", &op));
1408 }
1409 {
1410 ObjectWriteOperation op;
1411 op.remove();
1412 ASSERT_EQ(0, ioctx.operate("baz", &op));
1413 }
1414 {
1415 bufferlist bl;
1416 bl.append("ciao!");
1417 ObjectWriteOperation op;
1418 op.write_full(bl);
1419 ASSERT_EQ(0, ioctx.operate("bam", &op));
1420 }
1421
1422 // Configure cache
1423 bufferlist inbl;
1424 ASSERT_EQ(0, cluster.mon_command(
1425 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1426 "\", \"tierpool\": \"" + cache_pool_name +
1427 "\", \"force_nonempty\": \"--force-nonempty\" }",
1428 inbl, NULL, NULL));
1429 ASSERT_EQ(0, cluster.mon_command(
1430 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1431 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1432 inbl, NULL, NULL));
1433 ASSERT_EQ(0, cluster.mon_command(
1434 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1435 "\", \"mode\": \"writeback\"}",
1436 inbl, NULL, NULL));
1437
1438 // Wait for maps to settle
1439 cluster.wait_for_latest_osdmap();
1440
1441 // Read, trigger a promote on the head
1442 {
1443 bufferlist bl;
1444 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1445 ASSERT_EQ('c', bl[0]);
1446 }
1447
1448 // Read foo snap
1449 ioctx.snap_set_read(my_snaps[0]);
1450 {
1451 bufferlist bl;
1452 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1453 ASSERT_EQ('h', bl[0]);
1454 }
1455
1456 // Evict foo snap
1457 {
1458 ObjectReadOperation op;
1459 op.cache_evict();
1460 librados::AioCompletion *completion = cluster.aio_create_completion();
1461 ASSERT_EQ(0, ioctx.aio_operate(
1462 "foo", completion, &op,
1463 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1464 completion->wait_for_complete();
11fdf7f2
TL		 1465 ASSERT_EQ(0, completion->get_return_value());
1466 completion->release();
1467 }
1468 // Snap is gone...
1469 {
1470 bufferlist bl;
1471 ObjectReadOperation op;
1472 op.read(1, 0, &bl, NULL);
1473 librados::AioCompletion *completion = cluster.aio_create_completion();
1474 ASSERT_EQ(0, ioctx.aio_operate(
1475 "foo", completion, &op,
1476 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1477 completion->wait_for_complete();
11fdf7f2
TL		 1478 ASSERT_EQ(-ENOENT, completion->get_return_value());
1479 completion->release();
1480 }
1481
1482 // Do list-snaps
1483 ioctx.snap_set_read(CEPH_SNAPDIR);
1484 {
1485 snap_set_t snap_set;
1486 int snap_ret;
1487 ObjectReadOperation op;
1488 op.list_snaps(&snap_set, &snap_ret);
1489 librados::AioCompletion *completion = cluster.aio_create_completion();
1490 ASSERT_EQ(0, ioctx.aio_operate(
1491 "foo", completion, &op,
1492 0, NULL));
9f95a23c 1493 completion->wait_for_complete();
11fdf7f2
TL		 1494 ASSERT_EQ(0, snap_ret);
1495 ASSERT_LT(0u, snap_set.clones.size());
1496 for (vector<librados::clone_info_t>::const_iterator r = snap_set.clones.begin();
1497 r != snap_set.clones.end();
1498 ++r) {
1499 if (r->cloneid != librados::SNAP_HEAD) {
1500 ASSERT_LT(0u, r->snaps.size());
1501 }
1502 }
1503 }
1504
1505 // Cleanup
1506 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1507}
1508
20effc67
TL		 1509// This test case reproduces https://tracker.ceph.com/issues/49409
1510TEST_F(LibRadosTwoPoolsPP, EvictSnapRollbackReadRace) {
1511 // create object
1512 {
1513 bufferlist bl;
1514 int len = string("hi there").length() * 2;
1515 // append more chrunk data make sure the second promote
1516 // op coming before the first promote op finished
1517 for (int i=0; i<4*1024*1024/len; ++i)
1518 bl.append("hi therehi there");
1519 ObjectWriteOperation op;
1520 op.write_full(bl);
1521 ASSERT_EQ(0, ioctx.operate("foo", &op));
1522 }
1523
1524 // create two snapshot, a clone
1525 vector<uint64_t> my_snaps(2);
1526 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[1]));
1527 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1528 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1529 my_snaps));
1530 {
1531 bufferlist bl;
1532 bl.append("ciao!");
1533 ObjectWriteOperation op;
1534 op.write_full(bl);
1535 ASSERT_EQ(0, ioctx.operate("foo", &op));
1536 }
1537
1538 // configure cache
1539 bufferlist inbl;
1540 ASSERT_EQ(0, cluster.mon_command(
1541 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1542 "\", \"tierpool\": \"" + cache_pool_name +
1543 "\", \"force_nonempty\": \"--force-nonempty\" }",
1544 inbl, NULL, NULL));
1545 ASSERT_EQ(0, cluster.mon_command(
1546 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1547 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1548 inbl, NULL, NULL));
1549 ASSERT_EQ(0, cluster.mon_command(
1550 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1551 "\", \"mode\": \"writeback\"}",
1552 inbl, NULL, NULL));
1553
1554 // wait for maps to settle
1555 cluster.wait_for_latest_osdmap();
1556
1557 // read, trigger a promote on the head
1558 {
1559 bufferlist bl;
1560 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1561 ASSERT_EQ('c', bl[0]);
1562 }
1563
1564 // try more times
1565 int retries = 50;
1566 for (int i=0; i<retries; ++i)
1567 {
1568 {
1569 librados::AioCompletion * completion = cluster.aio_create_completion();
1570 librados::AioCompletion * completion1 = cluster.aio_create_completion();
1571
1572 // send a snap rollback op and a snap read op parallel
1573 // trigger two promote(copy) to the same snap clone obj
1574 // the second snap read op is read-ordered make sure
1575 // op not wait for objects_blocked_on_snap_promotion
1576 ObjectWriteOperation op;
1577 op.selfmanaged_snap_rollback(my_snaps[0]);
1578 ASSERT_EQ(0, ioctx.aio_operate(
1579 "foo", completion, &op));
1580
1581 ioctx.snap_set_read(my_snaps[1]);
1582 std::map<uint64_t, uint64_t> extents;
1583 bufferlist read_bl;
1584 int rval = -1;
1585 ObjectReadOperation op1;
1586 op1.sparse_read(0, 8, &extents, &read_bl, &rval);
1587 ASSERT_EQ(0, ioctx.aio_operate("foo", completion1, &op1, &read_bl));
1588 ioctx.snap_set_read(librados::SNAP_HEAD);
1589
1590 completion->wait_for_complete();
1591 ASSERT_EQ(0, completion->get_return_value());
1592 completion->release();
1593
1594 completion1->wait_for_complete();
1595 ASSERT_EQ(0, completion1->get_return_value());
1596 completion1->release();
1597 }
1598
1599 // evict foo snap
1600 ioctx.snap_set_read(my_snaps[0]);
1601 {
1602 ObjectReadOperation op;
1603 op.cache_evict();
1604 librados::AioCompletion *completion = cluster.aio_create_completion();
1605 ASSERT_EQ(0, ioctx.aio_operate(
1606 "foo", completion, &op,
1607 librados::OPERATION_IGNORE_CACHE, NULL));
1608 completion->wait_for_complete();
1609 ASSERT_EQ(0, completion->get_return_value());
1610 completion->release();
1611 }
1612 ioctx.snap_set_read(librados::SNAP_HEAD);
1613 }
1614
1615 // cleanup
1616 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1617 ioctx.selfmanaged_snap_remove(my_snaps[1]);
1618}
1619
7c673cae
FG		 1620TEST_F(LibRadosTwoPoolsPP, TryFlush) {
1621 // configure cache
1622 bufferlist inbl;
1623 ASSERT_EQ(0, cluster.mon_command(
1624 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1625 "\", \"tierpool\": \"" + cache_pool_name +
1626 "\", \"force_nonempty\": \"--force-nonempty\" }",
1627 inbl, NULL, NULL));
1628 ASSERT_EQ(0, cluster.mon_command(
1629 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1630 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1631 inbl, NULL, NULL));
1632 ASSERT_EQ(0, cluster.mon_command(
1633 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1634 "\", \"mode\": \"writeback\"}",
1635 inbl, NULL, NULL));
1636
1637 // wait for maps to settle
1638 cluster.wait_for_latest_osdmap();
1639
1640 // create object
1641 {
1642 bufferlist bl;
1643 bl.append("hi there");
1644 ObjectWriteOperation op;
1645 op.write_full(bl);
1646 ASSERT_EQ(0, ioctx.operate("foo", &op));
1647 }
1648
1649 // verify the object is present in the cache tier
1650 {
1651 NObjectIterator it = cache_ioctx.nobjects_begin();
1652 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1653 ASSERT_TRUE(it->get_oid() == string("foo"));
1654 ++it;
1655 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1656 }
1657
1658 // verify the object is NOT present in the base tier
1659 {
1660 NObjectIterator it = ioctx.nobjects_begin();
1661 ASSERT_TRUE(it == ioctx.nobjects_end());
1662 }
1663
1664 // verify dirty
1665 {
1666 bool dirty = false;
1667 int r = -1;
1668 ObjectReadOperation op;
1669 op.is_dirty(&dirty, &r);
1670 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1671 ASSERT_TRUE(dirty);
1672 ASSERT_EQ(0, r);
1673 }
1674
1675 // pin
1676 {
1677 ObjectWriteOperation op;
1678 op.cache_pin();
1679 librados::AioCompletion *completion = cluster.aio_create_completion();
1680 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1681 completion->wait_for_complete();
7c673cae
FG		 1682 ASSERT_EQ(0, completion->get_return_value());
1683 completion->release();
1684 }
1685
1686 // flush the pinned object with -EPERM
1687 {
1688 ObjectReadOperation op;
1689 op.cache_try_flush();
1690 librados::AioCompletion *completion = cluster.aio_create_completion();
1691 ASSERT_EQ(0, cache_ioctx.aio_operate(
1692 "foo", completion, &op,
1693 librados::OPERATION_IGNORE_OVERLAY |
1694 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1695 completion->wait_for_complete();
7c673cae
FG		 1696 ASSERT_EQ(-EPERM, completion->get_return_value());
1697 completion->release();
1698 }
1699
1700 // unpin
1701 {
1702 ObjectWriteOperation op;
1703 op.cache_unpin();
1704 librados::AioCompletion *completion = cluster.aio_create_completion();
1705 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1706 completion->wait_for_complete();
7c673cae
FG		 1707 ASSERT_EQ(0, completion->get_return_value());
1708 completion->release();
1709 }
1710
1711 // flush
1712 {
1713 ObjectReadOperation op;
1714 op.cache_try_flush();
1715 librados::AioCompletion *completion = cluster.aio_create_completion();
1716 ASSERT_EQ(0, cache_ioctx.aio_operate(
1717 "foo", completion, &op,
1718 librados::OPERATION_IGNORE_OVERLAY |
1719 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1720 completion->wait_for_complete();
7c673cae
FG		 1721 ASSERT_EQ(0, completion->get_return_value());
1722 completion->release();
1723 }
1724
1725 // verify clean
1726 {
1727 bool dirty = false;
1728 int r = -1;
1729 ObjectReadOperation op;
1730 op.is_dirty(&dirty, &r);
1731 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1732 ASSERT_FALSE(dirty);
1733 ASSERT_EQ(0, r);
1734 }
1735
1736 // verify in base tier
1737 {
1738 NObjectIterator it = ioctx.nobjects_begin();
1739 ASSERT_TRUE(it != ioctx.nobjects_end());
1740 ASSERT_TRUE(it->get_oid() == string("foo"));
1741 ++it;
1742 ASSERT_TRUE(it == ioctx.nobjects_end());
1743 }
1744
1745 // evict it
1746 {
1747 ObjectReadOperation op;
1748 op.cache_evict();
1749 librados::AioCompletion *completion = cluster.aio_create_completion();
1750 ASSERT_EQ(0, cache_ioctx.aio_operate(
1751 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1752 completion->wait_for_complete();
7c673cae
FG		 1753 ASSERT_EQ(0, completion->get_return_value());
1754 completion->release();
1755 }
1756
1757 // verify no longer in cache tier
1758 {
1759 NObjectIterator it = cache_ioctx.nobjects_begin();
1760 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1761 }
1762}
1763
1764TEST_F(LibRadosTwoPoolsPP, Flush) {
1765 // configure cache
1766 bufferlist inbl;
1767 ASSERT_EQ(0, cluster.mon_command(
1768 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1769 "\", \"tierpool\": \"" + cache_pool_name +
1770 "\", \"force_nonempty\": \"--force-nonempty\" }",
1771 inbl, NULL, NULL));
1772 ASSERT_EQ(0, cluster.mon_command(
1773 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1774 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1775 inbl, NULL, NULL));
1776 ASSERT_EQ(0, cluster.mon_command(
1777 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1778 "\", \"mode\": \"writeback\"}",
1779 inbl, NULL, NULL));
1780
1781 // wait for maps to settle
1782 cluster.wait_for_latest_osdmap();
1783
1784 uint64_t user_version = 0;
1785
1786 // create object
1787 {
1788 bufferlist bl;
1789 bl.append("hi there");
1790 ObjectWriteOperation op;
1791 op.write_full(bl);
1792 ASSERT_EQ(0, ioctx.operate("foo", &op));
1793 }
1794
1795 // verify the object is present in the cache tier
1796 {
1797 NObjectIterator it = cache_ioctx.nobjects_begin();
1798 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1799 ASSERT_TRUE(it->get_oid() == string("foo"));
1800 ++it;
1801 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1802 }
1803
1804 // verify the object is NOT present in the base tier
1805 {
1806 NObjectIterator it = ioctx.nobjects_begin();
1807 ASSERT_TRUE(it == ioctx.nobjects_end());
1808 }
1809
1810 // verify dirty
1811 {
1812 bool dirty = false;
1813 int r = -1;
1814 ObjectReadOperation op;
1815 op.is_dirty(&dirty, &r);
1816 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1817 ASSERT_TRUE(dirty);
1818 ASSERT_EQ(0, r);
1819 user_version = cache_ioctx.get_last_version();
1820 }
1821
1822 // pin
1823 {
1824 ObjectWriteOperation op;
1825 op.cache_pin();
1826 librados::AioCompletion *completion = cluster.aio_create_completion();
1827 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1828 completion->wait_for_complete();
7c673cae
FG		 1829 ASSERT_EQ(0, completion->get_return_value());
1830 completion->release();
1831 }
1832
1833 // flush the pinned object with -EPERM
1834 {
1835 ObjectReadOperation op;
1836 op.cache_try_flush();
1837 librados::AioCompletion *completion = cluster.aio_create_completion();
1838 ASSERT_EQ(0, cache_ioctx.aio_operate(
1839 "foo", completion, &op,
1840 librados::OPERATION_IGNORE_OVERLAY |
1841 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1842 completion->wait_for_complete();
7c673cae
FG		 1843 ASSERT_EQ(-EPERM, completion->get_return_value());
1844 completion->release();
1845 }
1846
1847 // unpin
1848 {
1849 ObjectWriteOperation op;
1850 op.cache_unpin();
1851 librados::AioCompletion *completion = cluster.aio_create_completion();
1852 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1853 completion->wait_for_complete();
7c673cae
FG		 1854 ASSERT_EQ(0, completion->get_return_value());
1855 completion->release();
1856 }
1857
1858 // flush
1859 {
1860 ObjectReadOperation op;
1861 op.cache_flush();
1862 librados::AioCompletion *completion = cluster.aio_create_completion();
1863 ASSERT_EQ(0, cache_ioctx.aio_operate(
1864 "foo", completion, &op,
1865 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 1866 completion->wait_for_complete();
7c673cae
FG		 1867 ASSERT_EQ(0, completion->get_return_value());
1868 completion->release();
1869 }
1870
1871 // verify clean
1872 {
1873 bool dirty = false;
1874 int r = -1;
1875 ObjectReadOperation op;
1876 op.is_dirty(&dirty, &r);
1877 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1878 ASSERT_FALSE(dirty);
1879 ASSERT_EQ(0, r);
1880 }
1881
1882 // verify in base tier
1883 {
1884 NObjectIterator it = ioctx.nobjects_begin();
1885 ASSERT_TRUE(it != ioctx.nobjects_end());
1886 ASSERT_TRUE(it->get_oid() == string("foo"));
1887 ++it;
1888 ASSERT_TRUE(it == ioctx.nobjects_end());
1889 }
1890
1891 // evict it
1892 {
1893 ObjectReadOperation op;
1894 op.cache_evict();
1895 librados::AioCompletion *completion = cluster.aio_create_completion();
1896 ASSERT_EQ(0, cache_ioctx.aio_operate(
1897 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1898 completion->wait_for_complete();
7c673cae
FG		 1899 ASSERT_EQ(0, completion->get_return_value());
1900 completion->release();
1901 }
1902
1903 // verify no longer in cache tier
1904 {
1905 NObjectIterator it = cache_ioctx.nobjects_begin();
1906 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1907 }
1908
1909 // read it again and verify the version is consistent
1910 {
1911 bufferlist bl;
1912 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
1913 ASSERT_EQ(user_version, cache_ioctx.get_last_version());
1914 }
1915
1916 // erase it
1917 {
1918 ObjectWriteOperation op;
1919 op.remove();
1920 ASSERT_EQ(0, ioctx.operate("foo", &op));
1921 }
1922
1923 // flush whiteout
1924 {
1925 ObjectReadOperation op;
1926 op.cache_flush();
1927 librados::AioCompletion *completion = cluster.aio_create_completion();
1928 ASSERT_EQ(0, cache_ioctx.aio_operate(
1929 "foo", completion, &op,
1930 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 1931 completion->wait_for_complete();
7c673cae
FG		 1932 ASSERT_EQ(0, completion->get_return_value());
1933 completion->release();
1934 }
1935
1936 // evict
1937 {
1938 ObjectReadOperation op;
1939 op.cache_evict();
1940 librados::AioCompletion *completion = cluster.aio_create_completion();
1941 ASSERT_EQ(0, cache_ioctx.aio_operate(
1942 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1943 completion->wait_for_complete();
7c673cae
FG		 1944 ASSERT_EQ(0, completion->get_return_value());
1945 completion->release();
1946 }
1947
1948 // verify no longer in cache tier
1949 {
1950 NObjectIterator it = cache_ioctx.nobjects_begin();
1951 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1952 }
1953 // or base tier
1954 {
1955 NObjectIterator it = ioctx.nobjects_begin();
1956 ASSERT_TRUE(it == ioctx.nobjects_end());
1957 }
1958}
1959
1960TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
1961 // configure cache
1962 bufferlist inbl;
1963 ASSERT_EQ(0, cluster.mon_command(
1964 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1965 "\", \"tierpool\": \"" + cache_pool_name +
1966 "\", \"force_nonempty\": \"--force-nonempty\" }",
1967 inbl, NULL, NULL));
1968 ASSERT_EQ(0, cluster.mon_command(
1969 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1970 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1971 inbl, NULL, NULL));
1972 ASSERT_EQ(0, cluster.mon_command(
1973 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1974 "\", \"mode\": \"writeback\"}",
1975 inbl, NULL, NULL));
1976
1977 // wait for maps to settle
1978 cluster.wait_for_latest_osdmap();
1979
1980 // create object
1981 {
1982 bufferlist bl;
1983 bl.append("a");
1984 ObjectWriteOperation op;
1985 op.write_full(bl);
1986 ASSERT_EQ(0, ioctx.operate("foo", &op));
1987 }
1988
1989 // create a snapshot, clone
1990 vector<uint64_t> my_snaps(1);
1991 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1992 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1993 my_snaps));
1994 {
1995 bufferlist bl;
1996 bl.append("b");
1997 ObjectWriteOperation op;
1998 op.write_full(bl);
1999 ASSERT_EQ(0, ioctx.operate("foo", &op));
2000 }
2001
2002 // and another
2003 my_snaps.resize(2);
2004 my_snaps[1] = my_snaps[0];
2005 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
2006 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
2007 my_snaps));
2008 {
2009 bufferlist bl;
2010 bl.append("c");
2011 ObjectWriteOperation op;
2012 op.write_full(bl);
2013 ASSERT_EQ(0, ioctx.operate("foo", &op));
2014 }
2015
2016 // verify the object is present in the cache tier
2017 {
2018 NObjectIterator it = cache_ioctx.nobjects_begin();
2019 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
2020 ASSERT_TRUE(it->get_oid() == string("foo"));
2021 ++it;
2022 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
2023 }
2024
2025 // verify the object is NOT present in the base tier
2026 {
2027 NObjectIterator it = ioctx.nobjects_begin();
2028 ASSERT_TRUE(it == ioctx.nobjects_end());
2029 }
2030
2031 // flush on head (should fail)
2032 ioctx.snap_set_read(librados::SNAP_HEAD);
2033 {
2034 ObjectReadOperation op;
2035 op.cache_flush();
2036 librados::AioCompletion *completion = cluster.aio_create_completion();
2037 ASSERT_EQ(0, ioctx.aio_operate(
2038 "foo", completion, &op,
2039 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 2040 completion->wait_for_complete();
7c673cae
FG		 2041 ASSERT_EQ(-EBUSY, completion->get_return_value());
2042 completion->release();
2043 }
2044 // flush on recent snap (should fail)
2045 ioctx.snap_set_read(my_snaps[0]);
2046 {
2047 ObjectReadOperation op;
2048 op.cache_flush();
2049 librados::AioCompletion *completion = cluster.aio_create_completion();
2050 ASSERT_EQ(0, ioctx.aio_operate(
2051 "foo", completion, &op,
2052 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 2053 completion->wait_for_complete();
7c673cae
FG		 2054 ASSERT_EQ(-EBUSY, completion->get_return_value());
2055 completion->release();
2056 }
2057 // flush on oldest snap
2058 ioctx.snap_set_read(my_snaps[1]);
2059 {
2060 ObjectReadOperation op;
2061 op.cache_flush();
2062 librados::AioCompletion *completion = cluster.aio_create_completion();
2063 ASSERT_EQ(0, ioctx.aio_operate(
2064 "foo", completion, &op,
2065 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 2066 completion->wait_for_complete();
7c673cae
FG		 2067 ASSERT_EQ(0, completion->get_return_value());
2068 completion->release();
2069 }
2070 // flush on next oldest snap
2071 ioctx.snap_set_read(my_snaps[0]);
2072 {
2073 ObjectReadOperation op;
2074 op.cache_flush();
2075 librados::AioCompletion *completion = cluster.aio_create_completion();
2076 ASSERT_EQ(0, ioctx.aio_operate(
2077 "foo", completion, &op,
2078 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 2079 completion->wait_for_complete();
7c673cae
FG		 2080 ASSERT_EQ(0, completion->get_return_value());
2081 completion->release();
2082 }
2083 // flush on head
2084 ioctx.snap_set_read(librados::SNAP_HEAD);
2085 {
2086 ObjectReadOperation op;
2087 op.cache_flush();
2088 librados::AioCompletion *completion = cluster.aio_create_completion();
2089 ASSERT_EQ(0, ioctx.aio_operate(
2090 "foo", completion, &op,
2091 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 2092 completion->wait_for_complete();
7c673cae
FG		 2093 ASSERT_EQ(0, completion->get_return_value());
2094 completion->release();
2095 }
2096
2097 // verify i can read the snaps from the cache pool
2098 ioctx.snap_set_read(librados::SNAP_HEAD);
2099 {
2100 bufferlist bl;
2101 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2102 ASSERT_EQ('c', bl[0]);
2103 }
2104 ioctx.snap_set_read(my_snaps[0]);
2105 {
2106 bufferlist bl;
2107 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2108 ASSERT_EQ('b', bl[0]);
2109 }
2110 ioctx.snap_set_read(my_snaps[1]);
2111 {
2112 bufferlist bl;
2113 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2114 ASSERT_EQ('a', bl[0]);
2115 }
2116
2117 // remove overlay
2118 ASSERT_EQ(0, cluster.mon_command(
2119 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2120 "\"}",
2121 inbl, NULL, NULL));
2122
2123 // wait for maps to settle
2124 cluster.wait_for_latest_osdmap();
2125
2126 // verify i can read the snaps from the base pool
2127 ioctx.snap_set_read(librados::SNAP_HEAD);
2128 {
2129 bufferlist bl;
2130 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2131 ASSERT_EQ('c', bl[0]);
2132 }
2133 ioctx.snap_set_read(my_snaps[0]);
2134 {
2135 bufferlist bl;
2136 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2137 ASSERT_EQ('b', bl[0]);
2138 }
2139 ioctx.snap_set_read(my_snaps[1]);
2140 {
2141 bufferlist bl;
2142 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2143 ASSERT_EQ('a', bl[0]);
2144 }
2145
2146 ASSERT_EQ(0, cluster.mon_command(
2147 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2148 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2149 inbl, NULL, NULL));
2150
2151 // cleanup
2152 ioctx.selfmanaged_snap_remove(my_snaps[0]);
2153}
2154
2155TEST_F(LibRadosTierPP, FlushWriteRaces) {
2156 Rados cluster;
2157 std::string pool_name = get_temp_pool_name();
2158 std::string cache_pool_name = pool_name + "-cache";
2159 ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
2160 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
2161 IoCtx cache_ioctx;
2162 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 2163 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 2164 IoCtx ioctx;
2165 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
2166
2167 // configure cache
2168 bufferlist inbl;
2169 ASSERT_EQ(0, cluster.mon_command(
2170 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2171 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2172 inbl, NULL, NULL));
2173 ASSERT_EQ(0, cluster.mon_command(
2174 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2175 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2176 inbl, NULL, NULL));
2177 ASSERT_EQ(0, cluster.mon_command(
2178 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2179 "\", \"mode\": \"writeback\"}",
2180 inbl, NULL, NULL));
2181
2182 // wait for maps to settle
2183 cluster.wait_for_latest_osdmap();
2184
2185 // create/dirty object
2186 bufferlist bl;
2187 bl.append("hi there");
2188 {
2189 ObjectWriteOperation op;
2190 op.write_full(bl);
2191 ASSERT_EQ(0, ioctx.operate("foo", &op));
2192 }
2193
2194 // flush + write
2195 {
2196 ObjectReadOperation op;
2197 op.cache_flush();
2198 librados::AioCompletion *completion = cluster.aio_create_completion();
2199 ASSERT_EQ(0, cache_ioctx.aio_operate(
2200 "foo", completion, &op,
2201 librados::OPERATION_IGNORE_OVERLAY, NULL));
2202
2203 ObjectWriteOperation op2;
2204 op2.write_full(bl);
2205 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2206 ASSERT_EQ(0, ioctx.aio_operate(
2207 "foo", completion2, &op2, 0));
2208
9f95a23c
TL		 2209 completion->wait_for_complete();
2210 completion2->wait_for_complete();
7c673cae
FG		 2211 ASSERT_EQ(0, completion->get_return_value());
2212 ASSERT_EQ(0, completion2->get_return_value());
2213 completion->release();
2214 completion2->release();
2215 }
2216
2217 int tries = 1000;
2218 do {
2219 // create/dirty object
2220 {
2221 bufferlist bl;
2222 bl.append("hi there");
2223 ObjectWriteOperation op;
2224 op.write_full(bl);
2225 ASSERT_EQ(0, ioctx.operate("foo", &op));
2226 }
2227
2228 // try-flush + write
2229 {
2230 ObjectReadOperation op;
2231 op.cache_try_flush();
2232 librados::AioCompletion *completion = cluster.aio_create_completion();
2233 ASSERT_EQ(0, cache_ioctx.aio_operate(
2234 "foo", completion, &op,
2235 librados::OPERATION_IGNORE_OVERLAY |
2236 librados::OPERATION_SKIPRWLOCKS, NULL));
2237
2238 ObjectWriteOperation op2;
2239 op2.write_full(bl);
2240 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2241 ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
2242
9f95a23c
TL		 2243 completion->wait_for_complete();
2244 completion2->wait_for_complete();
7c673cae
FG		 2245 int r = completion->get_return_value();
2246 ASSERT_TRUE(r == -EBUSY || r == 0);
2247 ASSERT_EQ(0, completion2->get_return_value());
2248 completion->release();
2249 completion2->release();
2250 if (r == -EBUSY)
2251 break;
2252 cout << "didn't get EBUSY, trying again" << std::endl;
2253 }
2254 ASSERT_TRUE(--tries);
2255 } while (true);
2256
2257 // tear down tiers
2258 ASSERT_EQ(0, cluster.mon_command(
2259 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2260 "\"}",
2261 inbl, NULL, NULL));
2262 ASSERT_EQ(0, cluster.mon_command(
2263 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2264 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2265 inbl, NULL, NULL));
2266
2267 // wait for maps to settle before next test
2268 cluster.wait_for_latest_osdmap();
2269
2270 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
2271 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
2272}
2273
2274TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
2275 // configure cache
2276 bufferlist inbl;
2277 ASSERT_EQ(0, cluster.mon_command(
2278 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2279 "\", \"tierpool\": \"" + cache_pool_name +
2280 "\", \"force_nonempty\": \"--force-nonempty\" }",
2281 inbl, NULL, NULL));
2282 ASSERT_EQ(0, cluster.mon_command(
2283 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2284 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2285 inbl, NULL, NULL));
2286 ASSERT_EQ(0, cluster.mon_command(
2287 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2288 "\", \"mode\": \"writeback\"}",
2289 inbl, NULL, NULL));
2290
2291 // wait for maps to settle
2292 cluster.wait_for_latest_osdmap();
2293
2294 // create/dirty object
2295 {
2296 bufferlist bl;
2297 bl.append("hi there");
2298 ObjectWriteOperation op;
2299 op.write_full(bl);
2300 ASSERT_EQ(0, ioctx.operate("foo", &op));
2301 }
2302
2303 // flush + flush
2304 {
2305 ObjectReadOperation op;
2306 op.cache_flush();
2307 librados::AioCompletion *completion = cluster.aio_create_completion();
2308 ASSERT_EQ(0, cache_ioctx.aio_operate(
2309 "foo", completion, &op,
2310 librados::OPERATION_IGNORE_OVERLAY, NULL));
2311
2312 ObjectReadOperation op2;
2313 op2.cache_flush();
2314 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2315 ASSERT_EQ(0, cache_ioctx.aio_operate(
2316 "foo", completion2, &op2,
2317 librados::OPERATION_IGNORE_OVERLAY, NULL));
2318
9f95a23c
TL		 2319 completion->wait_for_complete();
2320 completion2->wait_for_complete();
7c673cae
FG		 2321 ASSERT_EQ(0, completion->get_return_value());
2322 ASSERT_EQ(0, completion2->get_return_value());
2323 completion->release();
2324 completion2->release();
2325 }
2326
2327 // create/dirty object
2328 {
2329 bufferlist bl;
2330 bl.append("hi there");
2331 ObjectWriteOperation op;
2332 op.write_full(bl);
2333 ASSERT_EQ(0, ioctx.operate("foo", &op));
2334 }
2335
2336 // flush + try-flush
2337 {
2338 ObjectReadOperation op;
2339 op.cache_flush();
2340 librados::AioCompletion *completion = cluster.aio_create_completion();
2341 ASSERT_EQ(0, cache_ioctx.aio_operate(
2342 "foo", completion, &op,
2343 librados::OPERATION_IGNORE_OVERLAY, NULL));
2344
2345 ObjectReadOperation op2;
2346 op2.cache_try_flush();
2347 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2348 ASSERT_EQ(0, cache_ioctx.aio_operate(
2349 "foo", completion2, &op2,
2350 librados::OPERATION_IGNORE_OVERLAY |
2351 librados::OPERATION_SKIPRWLOCKS, NULL));
2352
9f95a23c
TL		 2353 completion->wait_for_complete();
2354 completion2->wait_for_complete();
7c673cae
FG		 2355 ASSERT_EQ(0, completion->get_return_value());
2356 ASSERT_EQ(0, completion2->get_return_value());
2357 completion->release();
2358 completion2->release();
2359 }
2360
2361 // create/dirty object
2362 int tries = 1000;
2363 do {
2364 {
2365 bufferlist bl;
2366 bl.append("hi there");
2367 ObjectWriteOperation op;
2368 op.write_full(bl);
2369 ASSERT_EQ(0, ioctx.operate("foo", &op));
2370 }
2371
2372 // try-flush + flush
2373 // (flush will not piggyback on try-flush)
2374 {
2375 ObjectReadOperation op;
2376 op.cache_try_flush();
2377 librados::AioCompletion *completion = cluster.aio_create_completion();
2378 ASSERT_EQ(0, cache_ioctx.aio_operate(
2379 "foo", completion, &op,
2380 librados::OPERATION_IGNORE_OVERLAY |
2381 librados::OPERATION_SKIPRWLOCKS, NULL));
2382
2383 ObjectReadOperation op2;
2384 op2.cache_flush();
2385 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2386 ASSERT_EQ(0, cache_ioctx.aio_operate(
2387 "foo", completion2, &op2,
2388 librados::OPERATION_IGNORE_OVERLAY, NULL));
2389
9f95a23c
TL		 2390 completion->wait_for_complete();
2391 completion2->wait_for_complete();
7c673cae
FG		 2392 int r = completion->get_return_value();
2393 ASSERT_TRUE(r == -EBUSY || r == 0);
2394 ASSERT_EQ(0, completion2->get_return_value());
2395 completion->release();
2396 completion2->release();
2397 if (r == -EBUSY)
2398 break;
2399 cout << "didn't get EBUSY, trying again" << std::endl;
2400 }
2401 ASSERT_TRUE(--tries);
2402 } while (true);
2403
2404 // create/dirty object
2405 {
2406 bufferlist bl;
2407 bl.append("hi there");
2408 ObjectWriteOperation op;
2409 op.write_full(bl);
2410 ASSERT_EQ(0, ioctx.operate("foo", &op));
2411 }
2412
2413 // try-flush + try-flush
2414 {
2415 ObjectReadOperation op;
2416 op.cache_try_flush();
2417 librados::AioCompletion *completion = cluster.aio_create_completion();
2418 ASSERT_EQ(0, cache_ioctx.aio_operate(
2419 "foo", completion, &op,
2420 librados::OPERATION_IGNORE_OVERLAY |
2421 librados::OPERATION_SKIPRWLOCKS, NULL));
2422
2423 ObjectReadOperation op2;
2424 op2.cache_try_flush();
2425 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2426 ASSERT_EQ(0, cache_ioctx.aio_operate(
2427 "foo", completion2, &op2,
2428 librados::OPERATION_IGNORE_OVERLAY |
2429 librados::OPERATION_SKIPRWLOCKS, NULL));
2430
9f95a23c
TL		 2431 completion->wait_for_complete();
2432 completion2->wait_for_complete();
7c673cae
FG		 2433 ASSERT_EQ(0, completion->get_return_value());
2434 ASSERT_EQ(0, completion2->get_return_value());
2435 completion->release();
2436 completion2->release();
2437 }
2438}
2439
2440
2441IoCtx *read_ioctx = 0;
9f95a23c
TL		 2442ceph::mutex test_lock = ceph::make_mutex("FlushReadRaces::lock");
2443ceph::condition_variable cond;
7c673cae
FG		 2444int max_reads = 100;
2445int num_reads = 0; // in progress
2446
2447void flush_read_race_cb(completion_t cb, void *arg);
2448
2449void start_flush_read()
2450{
2451 //cout << " starting read" << std::endl;
2452 ObjectReadOperation op;
2453 op.stat(NULL, NULL, NULL);
2454 librados::AioCompletion *completion = completions.getCompletion();
2455 completion->set_complete_callback(0, flush_read_race_cb);
2456 read_ioctx->aio_operate("foo", completion, &op, NULL);
2457}
2458
2459void flush_read_race_cb(completion_t cb, void *arg)
2460{
2461 //cout << " finished read" << std::endl;
9f95a23c 2462 std::lock_guard l{test_lock};
7c673cae
FG		 2463 if (num_reads > max_reads) {
2464 num_reads--;
9f95a23c 2465 cond.notify_all();
7c673cae
FG		 2466 } else {
2467 start_flush_read();
2468 }
7c673cae
FG		 2469}
2470
2471TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
2472 // configure cache
2473 bufferlist inbl;
2474 ASSERT_EQ(0, cluster.mon_command(
2475 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2476 "\", \"tierpool\": \"" + cache_pool_name +
2477 "\", \"force_nonempty\": \"--force-nonempty\" }",
2478 inbl, NULL, NULL));
2479 ASSERT_EQ(0, cluster.mon_command(
2480 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2481 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2482 inbl, NULL, NULL));
2483 ASSERT_EQ(0, cluster.mon_command(
2484 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2485 "\", \"mode\": \"writeback\"}",
2486 inbl, NULL, NULL));
2487
2488 // wait for maps to settle
2489 cluster.wait_for_latest_osdmap();
2490
2491 // create/dirty object
2492 {
2493 bufferlist bl;
2494 bl.append("hi there");
2495 bufferptr bp(4000000); // make it big!
2496 bp.zero();
2497 bl.append(bp);
2498 ObjectWriteOperation op;
2499 op.write_full(bl);
2500 ASSERT_EQ(0, ioctx.operate("foo", &op));
2501 }
2502
2503 // start a continuous stream of reads
2504 read_ioctx = &ioctx;
9f95a23c 2505 test_lock.lock();
7c673cae
FG		 2506 for (int i = 0; i < max_reads; ++i) {
2507 start_flush_read();
2508 num_reads++;
2509 }
9f95a23c 2510 test_lock.unlock();
7c673cae
FG		 2511
2512 // try-flush
2513 ObjectReadOperation op;
2514 op.cache_try_flush();
2515 librados::AioCompletion *completion = cluster.aio_create_completion();
2516 ASSERT_EQ(0, cache_ioctx.aio_operate(
2517 "foo", completion, &op,
2518 librados::OPERATION_IGNORE_OVERLAY |
2519 librados::OPERATION_SKIPRWLOCKS, NULL));
2520
9f95a23c 2521 completion->wait_for_complete();
7c673cae
FG		 2522 ASSERT_EQ(0, completion->get_return_value());
2523 completion->release();
2524
2525 // stop reads
9f95a23c
TL		 2526 std::unique_lock locker{test_lock};
2527 max_reads = 0;
2528 cond.wait(locker, [] { return num_reads == 0;});
7c673cae
FG		 2529}
2530
2531TEST_F(LibRadosTierPP, HitSetNone) {
2532 {
2533 list< pair<time_t,time_t> > ls;
2534 AioCompletion *c = librados::Rados::aio_create_completion();
2535 ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
2536 c->wait_for_complete();
2537 ASSERT_EQ(0, c->get_return_value());
2538 ASSERT_TRUE(ls.empty());
2539 c->release();
2540 }
2541 {
2542 bufferlist bl;
2543 AioCompletion *c = librados::Rados::aio_create_completion();
2544 ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
2545 c->wait_for_complete();
2546 ASSERT_EQ(-ENOENT, c->get_return_value());
2547 c->release();
2548 }
2549}
2550
2551string set_pool_str(string pool, string var, string val)
2552{
2553 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2554 + string("\",\"var\": \"") + var + string("\",\"val\": \"")
2555 + val + string("\"}");
2556}
2557
2558string set_pool_str(string pool, string var, int val)
2559{
2560 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2561 + string("\",\"var\": \"") + var + string("\",\"val\": \"")
2562 + stringify(val) + string("\"}");
2563}
2564
2565TEST_F(LibRadosTwoPoolsPP, HitSetRead) {
2566 // make it a tier
2567 bufferlist inbl;
2568 ASSERT_EQ(0, cluster.mon_command(
2569 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2570 "\", \"tierpool\": \"" + cache_pool_name +
2571 "\", \"force_nonempty\": \"--force-nonempty\" }",
2572 inbl, NULL, NULL));
2573
2574 // enable hitset tracking for this pool
2575 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
2576 inbl, NULL, NULL));
2577 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
2578 inbl, NULL, NULL));
2579 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
2580 "explicit_object"),
2581 inbl, NULL, NULL));
2582
2583 // wait for maps to settle
2584 cluster.wait_for_latest_osdmap();
2585
2586 cache_ioctx.set_namespace("");
2587
2588 // keep reading until we see our object appear in the HitSet
2589 utime_t start = ceph_clock_now();
2590 utime_t hard_stop = start + utime_t(600, 0);
2591
2592 while (true) {
2593 utime_t now = ceph_clock_now();
2594 ASSERT_TRUE(now < hard_stop);
2595
2596 string name = "foo";
2597 uint32_t hash;
2598 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
2599 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
2600 cluster.pool_lookup(cache_pool_name.c_str()), "");
2601
2602 bufferlist bl;
2603 ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
2604
2605 bufferlist hbl;
2606 AioCompletion *c = librados::Rados::aio_create_completion();
2607 ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
2608 c->wait_for_complete();
2609 c->release();
2610
2611 if (hbl.length()) {
11fdf7f2 2612 auto p = hbl.cbegin();
7c673cae 2613 HitSet hs;
11fdf7f2 2614 decode(hs, p);
7c673cae
FG		 2615 if (hs.contains(oid)) {
2616 cout << "ok, hit_set contains " << oid << std::endl;
2617 break;
2618 }
2619 cout << "hmm, not in HitSet yet" << std::endl;
2620 } else {
2621 cout << "hmm, no HitSet yet" << std::endl;
2622 }
2623
2624 sleep(1);
2625 }
2626}
2627
2628static int _get_pg_num(Rados& cluster, string pool_name)
2629{
2630 bufferlist inbl;
2631 string cmd = string("{\"prefix\": \"osd pool get\",\"pool\":\"")
2632 + pool_name
2633 + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
2634 bufferlist outbl;
2635 int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
11fdf7f2 2636 ceph_assert(r >= 0);
7c673cae
FG		 2637 string outstr(outbl.c_str(), outbl.length());
2638 json_spirit::Value v;
2639 if (!json_spirit::read(outstr, v)) {
2640 cerr <<" unable to parse json " << outstr << std::endl;
2641 return -1;
2642 }
2643
2644 json_spirit::Object& o = v.get_obj();
2645 for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
2646 json_spirit::Pair& p = o[i];
2647 if (p.name_ == "pg_num") {
2648 cout << "pg_num = " << p.value_.get_int() << std::endl;
2649 return p.value_.get_int();
2650 }
2651 }
2652 cerr << "didn't find pg_num in " << outstr << std::endl;
2653 return -1;
2654}
2655
20effc67
TL		 2656int make_hitset(Rados& cluster, librados::IoCtx& cache_ioctx, int num_pg,
2657 int num, std::map<int, HitSet>& hitsets, std::string& cache_pool_name)
2658{
2659 int pg = num_pg;
7c673cae
FG		 2660 // do a bunch of writes
2661 for (int i=0; i<num; ++i) {
2662 bufferlist bl;
2663 bl.append("a");
20effc67 2664 ceph_assert(0 == cache_ioctx.write(stringify(i), bl, 1, 0));
7c673cae
FG		 2665 }
2666
2667 // get HitSets
20effc67 2668 for (int i=0; i<pg; ++i) {
7c673cae
FG		 2669 list< pair<time_t,time_t> > ls;
2670 AioCompletion *c = librados::Rados::aio_create_completion();
20effc67 2671 ceph_assert(0 == cache_ioctx.hit_set_list(i, c, &ls));
7c673cae
FG		 2672 c->wait_for_complete();
2673 c->release();
2674 std::cout << "pg " << i << " ls " << ls << std::endl;
20effc67 2675 ceph_assert(!ls.empty());
7c673cae
FG		 2676
2677 // get the latest
2678 c = librados::Rados::aio_create_completion();
2679 bufferlist bl;
20effc67 2680 ceph_assert(0 == cache_ioctx.hit_set_get(i, c, ls.back().first, &bl));
7c673cae
FG		 2681 c->wait_for_complete();
2682 c->release();
2683
2684 try {
11fdf7f2
TL		 2685 auto p = bl.cbegin();
2686 decode(hitsets[i], p);
7c673cae
FG		 2687 }
2688 catch (buffer::error& e) {
2689 std::cout << "failed to decode hit set; bl len is " << bl.length() << "\n";
2690 bl.hexdump(std::cout);
2691 std::cout << std::endl;
2692 throw e;
2693 }
2694
2695 // cope with racing splits by refreshing pg_num
20effc67
TL		 2696 if (i == pg - 1)
2697 pg = _get_pg_num(cluster, cache_pool_name);
7c673cae 2698 }
20effc67
TL		 2699 return pg;
2700}
2701
2702TEST_F(LibRadosTwoPoolsPP, HitSetWrite) {
2703 int num_pg = _get_pg_num(cluster, pool_name);
2704 ceph_assert(num_pg > 0);
2705
2706 // make it a tier
2707 bufferlist inbl;
2708 ASSERT_EQ(0, cluster.mon_command(
2709 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2710 "\", \"tierpool\": \"" + cache_pool_name +
2711 "\", \"force_nonempty\": \"--force-nonempty\" }",
2712 inbl, NULL, NULL));
2713
2714 // enable hitset tracking for this pool
2715 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 8),
2716 inbl, NULL, NULL));
2717 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
2718 inbl, NULL, NULL));
2719 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
2720 "explicit_hash"),
2721 inbl, NULL, NULL));
2722
2723 // wait for maps to settle
2724 cluster.wait_for_latest_osdmap();
2725
2726 cache_ioctx.set_namespace("");
2727
2728 int num = 200;
2729
2730 std::map<int,HitSet> hitsets;
2731
2732 num_pg = make_hitset(cluster, cache_ioctx, num_pg, num, hitsets, cache_pool_name);
2733
2734 int retry = 0;
7c673cae
FG		 2735
2736 for (int i=0; i<num; ++i) {
2737 string n = stringify(i);
2738 uint32_t hash;
2739 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(n, &hash));
2740 hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
2741 cluster.pool_lookup(cache_pool_name.c_str()), "");
2742 std::cout << "checking for " << oid << std::endl;
2743 bool found = false;
2744 for (int p=0; p<num_pg; ++p) {
2745 if (hitsets[p].contains(oid)) {
2746 found = true;
2747 break;
2748 }
2749 }
20effc67
TL		 2750 if (!found && retry < 5) {
2751 num_pg = make_hitset(cluster, cache_ioctx, num_pg, num, hitsets, cache_pool_name);
2752 i--;
2753 retry++;
2754 continue;
2755 }
7c673cae
FG		 2756 ASSERT_TRUE(found);
2757 }
2758}
2759
2760TEST_F(LibRadosTwoPoolsPP, HitSetTrim) {
2761 unsigned count = 3;
2762 unsigned period = 3;
2763
2764 // make it a tier
2765 bufferlist inbl;
2766 ASSERT_EQ(0, cluster.mon_command(
2767 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2768 "\", \"tierpool\": \"" + cache_pool_name +
2769 "\", \"force_nonempty\": \"--force-nonempty\" }",
2770 inbl, NULL, NULL));
2771
2772 // enable hitset tracking for this pool
2773 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
2774 inbl, NULL, NULL));
2775 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
2776 inbl, NULL, NULL));
2777 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2778 inbl, NULL, NULL));
2779 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
2780 inbl, NULL, NULL));
2781
2782 // wait for maps to settle
2783 cluster.wait_for_latest_osdmap();
2784
2785 cache_ioctx.set_namespace("");
2786
2787 // do a bunch of writes and make sure the hitsets rotate
2788 utime_t start = ceph_clock_now();
2789 utime_t hard_stop = start + utime_t(count * period * 50, 0);
2790
2791 time_t first = 0;
2792 while (true) {
2793 string name = "foo";
2794 uint32_t hash;
2795 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
2796 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
2797
2798 bufferlist bl;
2799 bl.append("f");
2800 ASSERT_EQ(0, cache_ioctx.write("foo", bl, 1, 0));
2801
2802 list<pair<time_t, time_t> > ls;
2803 AioCompletion *c = librados::Rados::aio_create_completion();
2804 ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
2805 c->wait_for_complete();
2806 c->release();
2807
2808 cout << " got ls " << ls << std::endl;
2809 if (!ls.empty()) {
2810 if (!first) {
2811 first = ls.front().first;
2812 cout << "first is " << first << std::endl;
2813 } else {
2814 if (ls.front().first != first) {
2815 cout << "first now " << ls.front().first << ", trimmed" << std::endl;
2816 break;
2817 }
2818 }
2819 }
2820
2821 utime_t now = ceph_clock_now();
2822 ASSERT_TRUE(now < hard_stop);
2823
2824 sleep(1);
2825 }
2826}
2827
2828TEST_F(LibRadosTwoPoolsPP, PromoteOn2ndRead) {
2829 // create object
2830 for (int i=0; i<20; ++i) {
2831 bufferlist bl;
2832 bl.append("hi there");
2833 ObjectWriteOperation op;
2834 op.write_full(bl);
2835 ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
2836 }
2837
2838 // configure cache
2839 bufferlist inbl;
2840 ASSERT_EQ(0, cluster.mon_command(
2841 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2842 "\", \"tierpool\": \"" + cache_pool_name +
2843 "\", \"force_nonempty\": \"--force-nonempty\" }",
2844 inbl, NULL, NULL));
2845 ASSERT_EQ(0, cluster.mon_command(
2846 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2847 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2848 inbl, NULL, NULL));
2849 ASSERT_EQ(0, cluster.mon_command(
2850 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2851 "\", \"mode\": \"writeback\"}",
2852 inbl, NULL, NULL));
2853
2854 // enable hitset tracking for this pool
2855 ASSERT_EQ(0, cluster.mon_command(
2856 set_pool_str(cache_pool_name, "hit_set_count", 2),
2857 inbl, NULL, NULL));
2858 ASSERT_EQ(0, cluster.mon_command(
2859 set_pool_str(cache_pool_name, "hit_set_period", 600),
2860 inbl, NULL, NULL));
2861 ASSERT_EQ(0, cluster.mon_command(
2862 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2863 inbl, NULL, NULL));
2864 ASSERT_EQ(0, cluster.mon_command(
2865 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
2866 inbl, NULL, NULL));
2867 ASSERT_EQ(0, cluster.mon_command(
2868 set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
2869 inbl, NULL, NULL));
2870 ASSERT_EQ(0, cluster.mon_command(
2871 set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
2872 inbl, NULL, NULL));
2873
2874 // wait for maps to settle
2875 cluster.wait_for_latest_osdmap();
2876
2877 int fake = 0; // set this to non-zero to test spurious promotion,
2878 // e.g. from thrashing
2879 int attempt = 0;
2880 string obj;
2881 while (true) {
2882 // 1st read, don't trigger a promote
2883 obj = "foo" + stringify(attempt);
2884 cout << obj << std::endl;
2885 {
2886 bufferlist bl;
2887 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2888 if (--fake >= 0) {
2889 sleep(1);
2890 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2891 sleep(1);
2892 }
2893 }
2894
2895 // verify the object is NOT present in the cache tier
2896 {
2897 bool found = false;
2898 NObjectIterator it = cache_ioctx.nobjects_begin();
2899 while (it != cache_ioctx.nobjects_end()) {
2900 cout << " see " << it->get_oid() << std::endl;
2901 if (it->get_oid() == string(obj.c_str())) {
2902 found = true;
2903 break;
2904 }
2905 ++it;
2906 }
2907 if (!found)
2908 break;
2909 }
2910
2911 ++attempt;
2912 ASSERT_LE(attempt, 20);
2913 cout << "hrm, object is present in cache on attempt " << attempt
2914 << ", retrying" << std::endl;
2915 }
2916
2917 // Read until the object is present in the cache tier
2918 cout << "verifying " << obj << " is eventually promoted" << std::endl;
2919 while (true) {
2920 bufferlist bl;
2921 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2922
2923 bool there = false;
2924 NObjectIterator it = cache_ioctx.nobjects_begin();
2925 while (it != cache_ioctx.nobjects_end()) {
2926 if (it->get_oid() == string(obj.c_str())) {
2927 there = true;
2928 break;
2929 }
2930 ++it;
2931 }
2932 if (there)
2933 break;
2934
2935 sleep(1);
2936 }
2937
2938 // tear down tiers
2939 ASSERT_EQ(0, cluster.mon_command(
2940 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2941 "\"}",
2942 inbl, NULL, NULL));
2943 ASSERT_EQ(0, cluster.mon_command(
2944 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2945 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2946 inbl, NULL, NULL));
2947
2948 // wait for maps to settle before next test
2949 cluster.wait_for_latest_osdmap();
2950}
2951
2952TEST_F(LibRadosTwoPoolsPP, ProxyRead) {
2953 // create object
2954 {
2955 bufferlist bl;
2956 bl.append("hi there");
2957 ObjectWriteOperation op;
2958 op.write_full(bl);
2959 ASSERT_EQ(0, ioctx.operate("foo", &op));
2960 }
2961
2962 // configure cache
2963 bufferlist inbl;
2964 ASSERT_EQ(0, cluster.mon_command(
2965 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2966 "\", \"tierpool\": \"" + cache_pool_name +
2967 "\", \"force_nonempty\": \"--force-nonempty\" }",
2968 inbl, NULL, NULL));
2969 ASSERT_EQ(0, cluster.mon_command(
2970 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2971 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2972 inbl, NULL, NULL));
2973 ASSERT_EQ(0, cluster.mon_command(
2974 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2975 "\", \"mode\": \"readproxy\"}",
2976 inbl, NULL, NULL));
2977
2978 // wait for maps to settle
2979 cluster.wait_for_latest_osdmap();
2980
2981 // read and verify the object
2982 {
2983 bufferlist bl;
2984 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2985 ASSERT_EQ('h', bl[0]);
2986 }
2987
2988 // Verify 10 times the object is NOT present in the cache tier
2989 uint32_t i = 0;
2990 while (i++ < 10) {
2991 NObjectIterator it = cache_ioctx.nobjects_begin();
2992 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
2993 sleep(1);
2994 }
2995
2996 // tear down tiers
2997 ASSERT_EQ(0, cluster.mon_command(
2998 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2999 "\"}",
3000 inbl, NULL, NULL));
3001 ASSERT_EQ(0, cluster.mon_command(
3002 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
3003 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
3004 inbl, NULL, NULL));
3005
3006 // wait for maps to settle before next test
3007 cluster.wait_for_latest_osdmap();
3008}
3009
3010TEST_F(LibRadosTwoPoolsPP, CachePin) {
3011 // create object
3012 {
3013 bufferlist bl;
3014 bl.append("hi there");
3015 ObjectWriteOperation op;
3016 op.write_full(bl);
3017 ASSERT_EQ(0, ioctx.operate("foo", &op));
3018 }
3019 {
3020 bufferlist bl;
3021 bl.append("hi there");
3022 ObjectWriteOperation op;
3023 op.write_full(bl);
3024 ASSERT_EQ(0, ioctx.operate("bar", &op));
3025 }
3026 {
3027 bufferlist bl;
3028 bl.append("hi there");
3029 ObjectWriteOperation op;
3030 op.write_full(bl);
3031 ASSERT_EQ(0, ioctx.operate("baz", &op));
3032 }
3033 {
3034 bufferlist bl;
3035 bl.append("hi there");
3036 ObjectWriteOperation op;
3037 op.write_full(bl);
3038 ASSERT_EQ(0, ioctx.operate("bam", &op));
3039 }
3040
3041 // configure cache
3042 bufferlist inbl;
3043 ASSERT_EQ(0, cluster.mon_command(
3044 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3045 "\", \"tierpool\": \"" + cache_pool_name +
3046 "\", \"force_nonempty\": \"--force-nonempty\" }",
3047 inbl, NULL, NULL));
3048 ASSERT_EQ(0, cluster.mon_command(
3049 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3050 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3051 inbl, NULL, NULL));
3052 ASSERT_EQ(0, cluster.mon_command(
3053 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3054 "\", \"mode\": \"writeback\"}",
3055 inbl, NULL, NULL));
3056
3057 // wait for maps to settle
3058 cluster.wait_for_latest_osdmap();
3059
3060 // read, trigger promote
3061 {
3062 bufferlist bl;
3063 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3064 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
3065 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
3066 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
3067 }
3068
3069 // verify the objects are present in the cache tier
3070 {
3071 NObjectIterator it = cache_ioctx.nobjects_begin();
3072 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
3073 for (uint32_t i = 0; i < 4; i++) {
3074 ASSERT_TRUE(it->get_oid() == string("foo") ||
3075 it->get_oid() == string("bar") ||
3076 it->get_oid() == string("baz") ||
3077 it->get_oid() == string("bam"));
3078 ++it;
3079 }
3080 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3081 }
3082
3083 // pin objects
3084 {
3085 ObjectWriteOperation op;
3086 op.cache_pin();
3087 librados::AioCompletion *completion = cluster.aio_create_completion();
3088 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 3089 completion->wait_for_complete();
7c673cae
FG		 3090 ASSERT_EQ(0, completion->get_return_value());
3091 completion->release();
3092 }
3093 {
3094 ObjectWriteOperation op;
3095 op.cache_pin();
3096 librados::AioCompletion *completion = cluster.aio_create_completion();
3097 ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
9f95a23c 3098 completion->wait_for_complete();
7c673cae
FG		 3099 ASSERT_EQ(0, completion->get_return_value());
3100 completion->release();
3101 }
3102
3103 // enable agent
3104 ASSERT_EQ(0, cluster.mon_command(
3105 set_pool_str(cache_pool_name, "hit_set_count", 2),
3106 inbl, NULL, NULL));
3107 ASSERT_EQ(0, cluster.mon_command(
3108 set_pool_str(cache_pool_name, "hit_set_period", 600),
3109 inbl, NULL, NULL));
3110 ASSERT_EQ(0, cluster.mon_command(
3111 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
3112 inbl, NULL, NULL));
3113 ASSERT_EQ(0, cluster.mon_command(
3114 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
3115 inbl, NULL, NULL));
3116 ASSERT_EQ(0, cluster.mon_command(
3117 set_pool_str(cache_pool_name, "target_max_objects", 1),
3118 inbl, NULL, NULL));
3119
3120 sleep(10);
3121
3122 // Verify the pinned object 'foo' is not flushed/evicted
3123 uint32_t count = 0;
3124 while (true) {
3125 bufferlist bl;
3126 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
3127
3128 count = 0;
3129 NObjectIterator it = cache_ioctx.nobjects_begin();
3130 while (it != cache_ioctx.nobjects_end()) {
3131 ASSERT_TRUE(it->get_oid() == string("foo") ||
3132 it->get_oid() == string("bar") ||
3133 it->get_oid() == string("baz") ||
3134 it->get_oid() == string("bam"));
3135 ++count;
3136 ++it;
3137 }
3138 if (count == 2) {
3139 ASSERT_TRUE(it->get_oid() == string("foo") ||
3140 it->get_oid() == string("baz"));
3141 break;
3142 }
3143
3144 sleep(1);
3145 }
3146
3147 // tear down tiers
3148 ASSERT_EQ(0, cluster.mon_command(
3149 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
3150 "\"}",
3151 inbl, NULL, NULL));
3152 ASSERT_EQ(0, cluster.mon_command(
3153 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
3154 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
3155 inbl, NULL, NULL));
3156
3157 // wait for maps to settle before next test
3158 cluster.wait_for_latest_osdmap();
3159}
3160
11fdf7f2
TL		 3161TEST_F(LibRadosTwoPoolsPP, SetRedirectRead) {
3162 // create object
3163 {
3164 bufferlist bl;
3165 bl.append("hi there");
3166 ObjectWriteOperation op;
3167 op.write_full(bl);
3168 ASSERT_EQ(0, ioctx.operate("foo", &op));
3169 }
3170 {
3171 bufferlist bl;
3172 bl.append("there");
3173 ObjectWriteOperation op;
3174 op.write_full(bl);
3175 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3176 }
3177
11fdf7f2
TL		 3178 // wait for maps to settle
3179 cluster.wait_for_latest_osdmap();
3180
3181 {
3182 ObjectWriteOperation op;
3183 op.set_redirect("bar", cache_ioctx, 0);
3184 librados::AioCompletion *completion = cluster.aio_create_completion();
3185 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3186 completion->wait_for_complete();
11fdf7f2
TL		 3187 ASSERT_EQ(0, completion->get_return_value());
3188 completion->release();
3189 }
3190 // read and verify the object
3191 {
3192 bufferlist bl;
3193 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3194 ASSERT_EQ('t', bl[0]);
3195 }
3196
11fdf7f2
TL		 3197 // wait for maps to settle before next test
3198 cluster.wait_for_latest_osdmap();
3199}
3200
3201TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
3202 // skip test if not yet mimic
9f95a23c
TL		 3203 if (_get_required_osd_release(cluster) < "mimic") {
3204 GTEST_SKIP() << "cluster is not yet mimic, skipping test";
11fdf7f2
TL		 3205 }
3206
3207 // create object
3208 {
3209 bufferlist bl;
3210 bl.append("hi there");
3211 ObjectWriteOperation op;
3212 op.write_full(bl);
3213 ASSERT_EQ(0, ioctx.operate("foo", &op));
3214 }
3215 {
3216 bufferlist bl;
3217 bl.append("base chunk");
3218 ObjectWriteOperation op;
3219 op.write_full(bl);
3220 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3221 }
3222 {
3223 bufferlist bl;
3224 bl.append("there");
3225 ObjectWriteOperation op;
3226 op.write_full(bl);
3227 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3228 }
3229 {
3230 bufferlist bl;
3231 bl.append("CHUNK");
3232 ObjectWriteOperation op;
3233 op.write_full(bl);
3234 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3235 }
3236
11fdf7f2
TL		 3237 // wait for maps to settle
3238 cluster.wait_for_latest_osdmap();
3239
3240 // set-redirect
3241 {
3242 ObjectWriteOperation op;
3243 op.set_redirect("bar", cache_ioctx, 0);
3244 librados::AioCompletion *completion = cluster.aio_create_completion();
3245 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3246 completion->wait_for_complete();
11fdf7f2
TL		 3247 ASSERT_EQ(0, completion->get_return_value());
3248 completion->release();
3249 }
3250 // set-chunk
f67539c2
TL		 3251 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, "bar-chunk", "foo-chunk");
3252
11fdf7f2
TL		 3253 // promote
3254 {
3255 ObjectWriteOperation op;
3256 op.tier_promote();
3257 librados::AioCompletion *completion = cluster.aio_create_completion();
3258 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3259 completion->wait_for_complete();
11fdf7f2
TL		 3260 ASSERT_EQ(0, completion->get_return_value());
3261 completion->release();
3262 }
3263 // read and verify the object (redirect)
3264 {
3265 bufferlist bl;
3266 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3267 ASSERT_EQ('t', bl[0]);
3268 }
3269 // promote
3270 {
3271 ObjectWriteOperation op;
3272 op.tier_promote();
3273 librados::AioCompletion *completion = cluster.aio_create_completion();
3274 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3275 completion->wait_for_complete();
11fdf7f2
TL		 3276 ASSERT_EQ(0, completion->get_return_value());
3277 completion->release();
3278 }
3279 // read and verify the object
3280 {
3281 bufferlist bl;
3282 ASSERT_EQ(1, ioctx.read("foo-chunk", bl, 1, 0));
3283 ASSERT_EQ('C', bl[0]);
3284 }
3285
11fdf7f2
TL		 3286 // wait for maps to settle before next test
3287 cluster.wait_for_latest_osdmap();
3288}
3289
3290TEST_F(LibRadosTwoPoolsPP, ManifestRefRead) {
3291 // note: require >= mimic
3292
3293 // create object
3294 {
3295 bufferlist bl;
3296 bl.append("hi there");
3297 ObjectWriteOperation op;
3298 op.write_full(bl);
3299 ASSERT_EQ(0, ioctx.operate("foo", &op));
3300 }
3301 {
3302 bufferlist bl;
3303 bl.append("base chunk");
3304 ObjectWriteOperation op;
3305 op.write_full(bl);
3306 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3307 }
3308 {
3309 bufferlist bl;
3310 bl.append("there");
3311 ObjectWriteOperation op;
3312 op.write_full(bl);
3313 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3314 }
3315 {
3316 bufferlist bl;
3317 bl.append("CHUNK");
3318 ObjectWriteOperation op;
3319 op.write_full(bl);
3320 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3321 }
3322
3323 // wait for maps to settle
3324 cluster.wait_for_latest_osdmap();
3325
3326 // set-redirect
3327 {
3328 ObjectWriteOperation op;
3329 op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3330 librados::AioCompletion *completion = cluster.aio_create_completion();
3331 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3332 completion->wait_for_complete();
11fdf7f2
TL		 3333 ASSERT_EQ(0, completion->get_return_value());
3334 completion->release();
3335 }
3336 // set-chunk
3337 {
f67539c2 3338 ObjectReadOperation op;
11fdf7f2
TL		 3339 op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3340 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2
TL		 3341 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op,
3342 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 3343 completion->wait_for_complete();
11fdf7f2
TL		 3344 ASSERT_EQ(0, completion->get_return_value());
3345 completion->release();
3346 }
3347 // redirect's refcount
3348 {
f67539c2
TL		 3349 bufferlist t;
3350 cache_ioctx.getxattr("bar", CHUNK_REFCOUNT_ATTR, t);
3351 chunk_refs_t refs;
11fdf7f2 3352 try {
f67539c2
TL		 3353 auto iter = t.cbegin();
3354 decode(refs, iter);
11fdf7f2
TL		 3355 } catch (buffer::error& err) {
3356 ASSERT_TRUE(0);
3357 }
f67539c2 3358 ASSERT_EQ(1U, refs.count());
11fdf7f2
TL		 3359 }
3360 // chunk's refcount
3361 {
f67539c2
TL		 3362 bufferlist t;
3363 cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
3364 chunk_refs_t refs;
11fdf7f2 3365 try {
f67539c2
TL		 3366 auto iter = t.cbegin();
3367 decode(refs, iter);
11fdf7f2
TL		 3368 } catch (buffer::error& err) {
3369 ASSERT_TRUE(0);
3370 }
f67539c2 3371 ASSERT_EQ(1u, refs.count());
11fdf7f2
TL		 3372 }
3373
3374 // wait for maps to settle before next test
3375 cluster.wait_for_latest_osdmap();
3376}
3377
3378TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
3379 // skip test if not yet nautilus
9f95a23c
TL		 3380 if (_get_required_osd_release(cluster) < "nautilus") {
3381 GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
11fdf7f2
TL		 3382 }
3383
3384 // create object
3385 {
3386 bufferlist bl;
3387 bl.append("hi there");
3388 ObjectWriteOperation op;
3389 op.write_full(bl);
3390 ASSERT_EQ(0, ioctx.operate("foo", &op));
3391 }
3392 {
3393 bufferlist bl;
3394 bl.append("base chunk");
3395 ObjectWriteOperation op;
3396 op.write_full(bl);
3397 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3398 }
3399 {
3400 bufferlist bl;
3401 bl.append("there");
3402 ObjectWriteOperation op;
3403 op.write_full(bl);
3404 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3405 }
3406 {
3407 bufferlist bl;
3408 bl.append("CHUNK");
3409 ObjectWriteOperation op;
3410 op.write_full(bl);
3411 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3412 }
3413
3414 // wait for maps to settle
3415 cluster.wait_for_latest_osdmap();
3416
3417 // set-redirect
3418 {
3419 ObjectWriteOperation op;
3420 op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3421 librados::AioCompletion *completion = cluster.aio_create_completion();
3422 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3423 completion->wait_for_complete();
11fdf7f2
TL		 3424 ASSERT_EQ(0, completion->get_return_value());
3425 completion->release();
3426 }
3427 // set-chunk
3428 {
f67539c2 3429 ObjectReadOperation op;
11fdf7f2
TL		 3430 op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3431 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2
TL		 3432 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op,
3433 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 3434 completion->wait_for_complete();
11fdf7f2
TL		 3435 ASSERT_EQ(0, completion->get_return_value());
3436 completion->release();
3437 }
3438 // redirect's refcount
3439 {
f67539c2
TL		 3440 bufferlist t;
3441 cache_ioctx.getxattr("bar", CHUNK_REFCOUNT_ATTR, t);
3442 chunk_refs_t refs;
11fdf7f2 3443 try {
f67539c2
TL		 3444 auto iter = t.cbegin();
3445 decode(refs, iter);
11fdf7f2
TL		 3446 } catch (buffer::error& err) {
3447 ASSERT_TRUE(0);
3448 }
f67539c2 3449 ASSERT_EQ(1u, refs.count());
11fdf7f2
TL		 3450 }
3451 // chunk's refcount
3452 {
f67539c2
TL		 3453 bufferlist t;
3454 cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
3455 chunk_refs_t refs;
11fdf7f2 3456 try {
f67539c2
TL		 3457 auto iter = t.cbegin();
3458 decode(refs, iter);
11fdf7f2
TL		 3459 } catch (buffer::error& err) {
3460 ASSERT_TRUE(0);
3461 }
f67539c2 3462 ASSERT_EQ(1u, refs.count());
11fdf7f2
TL		 3463 }
3464
3465 // unset-manifest for set-redirect
3466 {
3467 ObjectWriteOperation op;
3468 op.unset_manifest();
3469 librados::AioCompletion *completion = cluster.aio_create_completion();
3470 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3471 completion->wait_for_complete();
11fdf7f2
TL		 3472 ASSERT_EQ(0, completion->get_return_value());
3473 completion->release();
3474 }
3475
3476 // unset-manifest for set-chunk
3477 {
3478 ObjectWriteOperation op;
3479 op.unset_manifest();
3480 librados::AioCompletion *completion = cluster.aio_create_completion();
3481 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3482 completion->wait_for_complete();
11fdf7f2
TL		 3483 ASSERT_EQ(0, completion->get_return_value());
3484 completion->release();
3485 }
3486 // redirect's refcount
3487 {
f67539c2
TL		 3488 bufferlist t;
3489 cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
3490 if (t.length() != 0U) {
11fdf7f2
TL		 3491 ObjectWriteOperation op;
3492 op.unset_manifest();
3493 librados::AioCompletion *completion = cluster.aio_create_completion();
3494 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3495 completion->wait_for_complete();
11fdf7f2
TL		 3496 ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
3497 completion->release();
3498 }
3499 }
3500 // chunk's refcount
3501 {
f67539c2
TL		 3502 bufferlist t;
3503 cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
3504 if (t.length() != 0U) {
11fdf7f2
TL		 3505 ObjectWriteOperation op;
3506 op.unset_manifest();
3507 librados::AioCompletion *completion = cluster.aio_create_completion();
3508 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3509 completion->wait_for_complete();
11fdf7f2
TL		 3510 ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
3511 completion->release();
3512 }
3513 }
3514
3515 // wait for maps to settle before next test
3516 cluster.wait_for_latest_osdmap();
3517}
3518
3519#include "common/ceph_crypto.h"
3520using ceph::crypto::SHA1;
3521#include "rgw/rgw_common.h"
3522TEST_F(LibRadosTwoPoolsPP, ManifestDedupRefRead) {
3523 // skip test if not yet nautilus
9f95a23c
TL		 3524 if (_get_required_osd_release(cluster) < "nautilus") {
3525 GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
31f18b77 3526 }
9f95a23c 3527
11fdf7f2
TL		 3528 bufferlist inbl;
3529 ASSERT_EQ(0, cluster.mon_command(
3530 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
3531 inbl, NULL, NULL));
3532 cluster.wait_for_latest_osdmap();
f67539c2
TL		 3533 string tgt_oid;
3534
3535 // get fp_oid
3536 tgt_oid = get_fp_oid("There hi", "sha1");
31f18b77
FG		 3537
3538 // create object
3539 {
3540 bufferlist bl;
f67539c2 3541 bl.append("There hi");
31f18b77
FG		 3542 ObjectWriteOperation op;
3543 op.write_full(bl);
3544 ASSERT_EQ(0, ioctx.operate("foo", &op));
3545 }
11fdf7f2
TL		 3546 {
3547 bufferlist bl;
f67539c2 3548 bl.append("There hi");
11fdf7f2
TL		 3549 ObjectWriteOperation op;
3550 op.write_full(bl);
3551 ASSERT_EQ(0, ioctx.operate("foo-dedup", &op));
3552 }
f67539c2
TL		 3553
3554 // write
31f18b77 3555 {
31f18b77 3556 ObjectWriteOperation op;
f67539c2
TL		 3557 bufferlist bl;
3558 bl.append("There hi");
31f18b77 3559 op.write_full(bl);
f67539c2 3560 ASSERT_EQ(0, cache_ioctx.operate(tgt_oid, &op));
31f18b77 3561 }
f67539c2
TL		 3562
3563 // set-chunk (dedup)
3564 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 8, tgt_oid, "foo-dedup");
3565 // set-chunk (dedup)
3566 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 8, tgt_oid, "foo");
3567 // chunk's refcount
11fdf7f2 3568 {
f67539c2
TL		 3569 bufferlist t;
3570 cache_ioctx.getxattr(tgt_oid, CHUNK_REFCOUNT_ATTR, t);
3571 chunk_refs_t refs;
3572 try {
3573 auto iter = t.cbegin();
3574 decode(refs, iter);
3575 } catch (buffer::error& err) {
3576 ASSERT_TRUE(0);
3577 }
a4b75251 3578 ASSERT_LE(2u, refs.count());
11fdf7f2 3579 }
31f18b77 3580
f67539c2 3581 // wait for maps to settle before next test
31f18b77 3582 cluster.wait_for_latest_osdmap();
f67539c2 3583}
31f18b77 3584
f67539c2
TL		 3585TEST_F(LibRadosTwoPoolsPP, ManifestSnapRefcount) {
3586 // skip test if not yet octopus
3587 if (_get_required_osd_release(cluster) < "octopus") {
3588 cout << "cluster is not yet octopus, skipping test" << std::endl;
3589 return;
11fdf7f2 3590 }
f67539c2
TL		 3591
3592 bufferlist inbl;
3593 ASSERT_EQ(0, cluster.mon_command(
3594 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
3595 inbl, NULL, NULL));
3596 cluster.wait_for_latest_osdmap();
3597
3598 // create object
11fdf7f2 3599 {
f67539c2
TL		 3600 bufferlist bl;
3601 bl.append("there hi");
11fdf7f2 3602 ObjectWriteOperation op;
f67539c2
TL		 3603 op.write_full(bl);
3604 ASSERT_EQ(0, ioctx.operate("foo", &op));
31f18b77 3605 }
31f18b77
FG		 3606 {
3607 bufferlist bl;
f67539c2 3608 bl.append("there hi");
11fdf7f2
TL		 3609 ObjectWriteOperation op;
3610 op.write_full(bl);
f67539c2 3611 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
11fdf7f2 3612 }
f67539c2
TL		 3613
3614 // wait for maps to settle
3615 cluster.wait_for_latest_osdmap();
3616
3617 string er_fp_oid, hi_fp_oid, bb_fp_oid;
3618
3619 // get fp_oid
3620 er_fp_oid = get_fp_oid("er", "sha1");
3621 hi_fp_oid = get_fp_oid("hi", "sha1");
3622 bb_fp_oid = get_fp_oid("bb", "sha1");
3623
3624 // write
11fdf7f2 3625 {
11fdf7f2 3626 ObjectWriteOperation op;
f67539c2
TL		 3627 bufferlist bl;
3628 bl.append("er");
11fdf7f2 3629 op.write_full(bl);
f67539c2 3630 ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
11fdf7f2 3631 }
f67539c2 3632 // write
11fdf7f2 3633 {
11fdf7f2 3634 ObjectWriteOperation op;
f67539c2
TL		 3635 bufferlist bl;
3636 bl.append("hi");
11fdf7f2 3637 op.write_full(bl);
f67539c2 3638 ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
11fdf7f2 3639 }
f67539c2 3640 // write
11fdf7f2 3641 {
11fdf7f2 3642 ObjectWriteOperation op;
f67539c2
TL		 3643 bufferlist bl;
3644 bl.append("bb");
11fdf7f2 3645 op.write_full(bl);
f67539c2 3646 ASSERT_EQ(0, cache_ioctx.operate(bb_fp_oid, &op));
11fdf7f2 3647 }
f67539c2
TL		 3648
3649 // set-chunk (dedup)
3650 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
3651 // set-chunk (dedup)
3652 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, hi_fp_oid, "foo");
3653
3654 // make all chunks dirty --> flush
3655 // foo: [er] [hi]
3656
3657 // check chunk's refcount
11fdf7f2 3658 {
f67539c2 3659 bufferlist t;
11fdf7f2 3660 SHA1 sha1_gen;
f67539c2 3661 int size = strlen("er");
11fdf7f2
TL		 3662 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3663 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
f67539c2 3664 sha1_gen.Update((const unsigned char *)"er", size);
11fdf7f2
TL		 3665 sha1_gen.Final(fingerprint);
3666 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
f67539c2
TL		 3667 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3668 chunk_refs_t refs;
11fdf7f2 3669 try {
f67539c2
TL		 3670 auto iter = t.cbegin();
3671 decode(refs, iter);
11fdf7f2
TL		 3672 } catch (buffer::error& err) {
3673 ASSERT_TRUE(0);
3674 }
a4b75251 3675 ASSERT_LE(1u, refs.count());
31f18b77 3676 }
31f18b77 3677
f67539c2
TL		 3678 // create a snapshot, clone
3679 vector<uint64_t> my_snaps(1);
3680 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3681 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3682 my_snaps));
9f95a23c 3683
f67539c2
TL		 3684 // foo: [bb] [hi]
3685 // create a clone
9f95a23c
TL		 3686 {
3687 bufferlist bl;
f67539c2
TL		 3688 bl.append("Thbbe");
3689 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
9f95a23c 3690 }
f67539c2 3691 // make clean
9f95a23c
TL		 3692 {
3693 bufferlist bl;
f67539c2
TL		 3694 bl.append("Thbbe");
3695 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3696 }
3697 // set-chunk (dedup)
3698 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, bb_fp_oid, "foo");
3699
3700 // and another
3701 my_snaps.resize(2);
3702 my_snaps[1] = my_snaps[0];
3703 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3704 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3705 my_snaps));
3706
3707 // foo: [er] [hi]
3708 // create a clone
3709 {
3710 bufferlist bl;
3711 bl.append("There");
3712 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3713 }
3714 // make clean
3715 {
3716 bufferlist bl;
3717 bl.append("There");
3718 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3719 }
3720 // set-chunk (dedup)
3721 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
3722
3723 // check chunk's refcount
3724 {
3725 bufferlist t;
3726 SHA1 sha1_gen;
3727 int size = strlen("er");
3728 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3729 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3730 sha1_gen.Update((const unsigned char *)"er", size);
3731 sha1_gen.Final(fingerprint);
3732 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3733 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3734 chunk_refs_t refs;
3735 try {
3736 auto iter = t.cbegin();
3737 decode(refs, iter);
3738 } catch (buffer::error& err) {
3739 ASSERT_TRUE(0);
3740 }
a4b75251 3741 ASSERT_LE(2u, refs.count());
f67539c2
TL		 3742 }
3743
3744 // and another
3745 my_snaps.resize(3);
3746 my_snaps[2] = my_snaps[1];
3747 my_snaps[1] = my_snaps[0];
3748 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3749 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3750 my_snaps));
3751
3752 // foo: [bb] [hi]
3753 // create a clone
3754 {
3755 bufferlist bl;
3756 bl.append("Thbbe");
3757 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3758 }
3759 // make clean
3760 {
3761 bufferlist bl;
3762 bl.append("Thbbe");
3763 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3764 }
3765 // set-chunk (dedup)
3766 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, bb_fp_oid, "foo");
3767
3768 /*
3769 * snap[2]: [er] [hi]
3770 * snap[1]: [bb] [hi]
3771 * snap[0]: [er] [hi]
3772 * head: [bb] [hi]
3773 */
3774
3775 // check chunk's refcount
3776 {
3777 bufferlist t;
3778 SHA1 sha1_gen;
3779 int size = strlen("hi");
3780 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3781 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3782 sha1_gen.Update((const unsigned char *)"hi", size);
3783 sha1_gen.Final(fingerprint);
3784 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3785 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3786 }
3787
3788 // check chunk's refcount
3789 {
3790 bufferlist t;
3791 SHA1 sha1_gen;
3792 int size = strlen("er");
3793 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3794 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3795 sha1_gen.Update((const unsigned char *)"er", size);
3796 sha1_gen.Final(fingerprint);
3797 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3798 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
3799 chunk_refs_t refs;
3800 try {
3801 auto iter = t.cbegin();
3802 decode(refs, iter);
3803 } catch (buffer::error& err) {
3804 ASSERT_TRUE(0);
3805 }
a4b75251 3806 ASSERT_LE(2u, refs.count());
f67539c2
TL		 3807 }
3808
3809 // remove snap
3810 ioctx.selfmanaged_snap_remove(my_snaps[2]);
3811
3812 /*
3813 * snap[1]: [bb] [hi]
3814 * snap[0]: [er] [hi]
3815 * head: [bb] [hi]
3816 */
3817
3818 sleep(10);
3819
3820 // check chunk's refcount
3821 {
3822 bufferlist t;
3823 SHA1 sha1_gen;
3824 int size = strlen("hi");
3825 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3826 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3827 sha1_gen.Update((const unsigned char *)"hi", size);
3828 sha1_gen.Final(fingerprint);
3829 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3830 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3831 }
3832
3833 // remove snap
3834 ioctx.selfmanaged_snap_remove(my_snaps[0]);
3835
3836 /*
3837 * snap[1]: [bb] [hi]
3838 * head: [bb] [hi]
3839 */
3840
3841 sleep(10);
3842
3843 // check chunk's refcount
3844 {
3845 bufferlist t;
3846 SHA1 sha1_gen;
3847 int size = strlen("bb");
3848 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3849 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3850 sha1_gen.Update((const unsigned char *)"bb", size);
3851 sha1_gen.Final(fingerprint);
3852 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3853 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3854 }
3855
3856 // remove snap
3857 ioctx.selfmanaged_snap_remove(my_snaps[1]);
3858
3859 /*
3860 * snap[1]: [bb] [hi]
3861 */
3862
3863 sleep(10);
3864
3865 // check chunk's refcount
3866 {
3867 bufferlist t;
3868 SHA1 sha1_gen;
3869 int size = strlen("bb");
3870 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3871 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3872 sha1_gen.Update((const unsigned char *)"bb", size);
3873 sha1_gen.Final(fingerprint);
3874 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3875 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3876 }
3877
3878 // check chunk's refcount
3879 {
3880 bufferlist t;
3881 SHA1 sha1_gen;
3882 int size = strlen("hi");
3883 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3884 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3885 sha1_gen.Update((const unsigned char *)"hi", size);
3886 sha1_gen.Final(fingerprint);
3887 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3888 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
3889 }
3890}
3891
3892TEST_F(LibRadosTwoPoolsPP, ManifestSnapRefcount2) {
3893 // skip test if not yet octopus
3894 if (_get_required_osd_release(cluster) < "octopus") {
3895 cout << "cluster is not yet octopus, skipping test" << std::endl;
3896 return;
3897 }
3898
3899 bufferlist inbl;
3900 ASSERT_EQ(0, cluster.mon_command(
3901 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
3902 inbl, NULL, NULL));
3903 cluster.wait_for_latest_osdmap();
3904
3905 // create object
3906 {
3907 bufferlist bl;
3908 bl.append("Thabe cdHI");
3909 ObjectWriteOperation op;
3910 op.write_full(bl);
3911 ASSERT_EQ(0, ioctx.operate("foo", &op));
3912 }
3913 {
3914 bufferlist bl;
3915 bl.append("there hiHI");
3916 ObjectWriteOperation op;
3917 op.write_full(bl);
3918 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3919 }
3920
3921 string ab_fp_oid, cd_fp_oid, ef_fp_oid, BB_fp_oid;
3922
3923 // get fp_oid
3924 ab_fp_oid = get_fp_oid("ab", "sha1");
3925 cd_fp_oid = get_fp_oid("cd", "sha1");
3926 ef_fp_oid = get_fp_oid("ef", "sha1");
3927 BB_fp_oid = get_fp_oid("BB", "sha1");
3928
3929 // write
3930 {
3931 ObjectWriteOperation op;
3932 bufferlist bl;
3933 bl.append("ab");
3934 op.write_full(bl);
3935 ASSERT_EQ(0, cache_ioctx.operate(ab_fp_oid, &op));
3936 }
3937 // write
3938 {
3939 ObjectWriteOperation op;
3940 bufferlist bl;
3941 bl.append("cd");
3942 op.write_full(bl);
3943 ASSERT_EQ(0, cache_ioctx.operate(cd_fp_oid, &op));
3944 }
3945 // write
3946 {
3947 ObjectWriteOperation op;
3948 bufferlist bl;
3949 bl.append("ef");
3950 op.write_full(bl);
3951 ASSERT_EQ(0, cache_ioctx.operate(ef_fp_oid, &op));
3952 }
3953 // write
3954 {
3955 ObjectWriteOperation op;
3956 bufferlist bl;
3957 bl.append("BB");
3958 op.write_full(bl);
3959 ASSERT_EQ(0, cache_ioctx.operate(BB_fp_oid, &op));
3960 }
3961
3962 // set-chunk (dedup)
3963 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, ab_fp_oid, "foo");
3964 // set-chunk (dedup)
3965 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, cd_fp_oid, "foo");
3966 // set-chunk (dedup)
3967 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, ef_fp_oid, "foo");
3968
3969
3970 // make all chunks dirty --> flush
3971 // foo: [ab] [cd] [ef]
3972
3973 // create a snapshot, clone
3974 vector<uint64_t> my_snaps(1);
3975 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3976 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3977 my_snaps));
3978
3979 // foo: [BB] [BB] [ef]
3980 // create a clone
3981 {
3982 bufferlist bl;
3983 bl.append("ThBBe BB");
3984 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3985 }
3986 // make clean
3987 {
3988 bufferlist bl;
3989 bl.append("ThBBe BB");
3990 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
3991 }
3992 // set-chunk (dedup)
3993 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, BB_fp_oid, "foo");
3994 // set-chunk (dedup)
3995 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, BB_fp_oid, "foo");
3996
3997 // and another
3998 my_snaps.resize(2);
3999 my_snaps[1] = my_snaps[0];
4000 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4001 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4002 my_snaps));
4003
4004 // foo: [ab] [cd] [ef]
4005 // create a clone
4006 {
4007 bufferlist bl;
4008 bl.append("Thabe cd");
4009 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4010 }
4011 // make clean
4012 {
4013 bufferlist bl;
4014 bl.append("Thabe cd");
4015 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4016 }
4017 // set-chunk (dedup)
4018 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, ab_fp_oid, "foo");
4019 // set-chunk (dedup)
4020 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, cd_fp_oid, "foo");
4021
4022 /*
4023 * snap[1]: [ab] [cd] [ef]
4024 * snap[0]: [BB] [BB] [ef]
4025 * head: [ab] [cd] [ef]
4026 */
4027
4028 // check chunk's refcount
4029 {
4030 bufferlist t;
4031 SHA1 sha1_gen;
4032 int size = strlen("ab");
4033 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
4034 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4035 sha1_gen.Update((const unsigned char *)"ab", size);
4036 sha1_gen.Final(fingerprint);
4037 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4038 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
4039 chunk_refs_t refs;
4040 try {
4041 auto iter = t.cbegin();
4042 decode(refs, iter);
4043 } catch (buffer::error& err) {
4044 ASSERT_TRUE(0);
4045 }
a4b75251 4046 ASSERT_LE(2u, refs.count());
f67539c2
TL		 4047 }
4048
4049 // check chunk's refcount
4050 {
4051 bufferlist t;
4052 SHA1 sha1_gen;
4053 int size = strlen("cd");
4054 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
4055 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4056 sha1_gen.Update((const unsigned char *)"cd", size);
4057 sha1_gen.Final(fingerprint);
4058 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4059 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
4060 chunk_refs_t refs;
4061 try {
4062 auto iter = t.cbegin();
4063 decode(refs, iter);
4064 } catch (buffer::error& err) {
4065 ASSERT_TRUE(0);
4066 }
a4b75251 4067 ASSERT_LE(2u, refs.count());
f67539c2
TL		 4068 }
4069
4070 // check chunk's refcount
4071 {
4072 bufferlist t;
4073 SHA1 sha1_gen;
4074 int size = strlen("BB");
4075 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
4076 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4077 sha1_gen.Update((const unsigned char *)"BB", size);
4078 sha1_gen.Final(fingerprint);
4079 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4080 cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
4081 chunk_refs_t refs;
4082 try {
4083 auto iter = t.cbegin();
4084 decode(refs, iter);
4085 } catch (buffer::error& err) {
4086 ASSERT_TRUE(0);
4087 }
a4b75251 4088 ASSERT_LE(2u, refs.count());
f67539c2
TL		 4089 }
4090
4091 // remove snap
4092 ioctx.selfmanaged_snap_remove(my_snaps[0]);
4093
4094 /*
4095 * snap[1]: [ab] [cd] [ef]
4096 * head: [ab] [cd] [ef]
4097 */
4098
4099 sleep(10);
4100
4101 // check chunk's refcount
4102 {
4103 bufferlist t;
4104 SHA1 sha1_gen;
4105 int size = strlen("BB");
4106 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
4107 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4108 sha1_gen.Update((const unsigned char *)"BB", size);
4109 sha1_gen.Final(fingerprint);
4110 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4111 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
4112 }
4113}
4114
4115TEST_F(LibRadosTwoPoolsPP, ManifestTestSnapCreate) {
4116 // skip test if not yet octopus
4117 if (_get_required_osd_release(cluster) < "octopus") {
4118 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4119 }
4120
4121 // create object
4122 {
4123 bufferlist bl;
4124 bl.append("base chunk");
4125 ObjectWriteOperation op;
4126 op.write_full(bl);
4127 ASSERT_EQ(0, ioctx.operate("foo", &op));
4128 }
4129 {
4130 bufferlist bl;
4131 bl.append("CHUNKS CHUNKS");
4132 ObjectWriteOperation op;
4133 op.write_full(bl);
4134 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
4135 }
4136
4137 string ba_fp_oid, se_fp_oid, ch_fp_oid;
4138
4139 // get fp_oid
4140 ba_fp_oid = get_fp_oid("ba", "sha1");
4141 se_fp_oid = get_fp_oid("se", "sha1");
4142 ch_fp_oid = get_fp_oid("ch", "sha1");
4143
4144 // write
4145 {
4146 ObjectWriteOperation op;
4147 bufferlist bl;
4148 bl.append("ba");
4149 op.write_full(bl);
4150 ASSERT_EQ(0, cache_ioctx.operate(ba_fp_oid, &op));
4151 }
4152 // write
4153 {
4154 ObjectWriteOperation op;
4155 bufferlist bl;
4156 bl.append("se");
4157 op.write_full(bl);
4158 ASSERT_EQ(0, cache_ioctx.operate(se_fp_oid, &op));
4159 }
4160 // write
4161 {
4162 ObjectWriteOperation op;
4163 bufferlist bl;
4164 bl.append("ch");
4165 op.write_full(bl);
4166 ASSERT_EQ(0, cache_ioctx.operate(ch_fp_oid, &op));
4167 }
4168
4169 // set-chunk (dedup)
4170 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, ba_fp_oid, "foo");
4171
4172 // try to create a snapshot, clone
4173 vector<uint64_t> my_snaps(1);
4174 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4175 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4176 my_snaps));
4177
4178 // set-chunk (dedup)
4179 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, se_fp_oid, "foo");
4180
4181 // check whether clone is created
4182 ioctx.snap_set_read(librados::SNAP_DIR);
4183 {
4184 snap_set_t snap_set;
4185 int snap_ret;
4186 ObjectReadOperation op;
4187 op.list_snaps(&snap_set, &snap_ret);
4188 librados::AioCompletion *completion = cluster.aio_create_completion();
4189 ASSERT_EQ(0, ioctx.aio_operate(
4190 "foo", completion, &op,
4191 0, NULL));
4192 completion->wait_for_complete();
4193 ASSERT_EQ(0, snap_ret);
4194 ASSERT_LT(0u, snap_set.clones.size());
4195 ASSERT_EQ(1, snap_set.clones.size());
4196 }
4197
4198 // create a clone
4199 ioctx.snap_set_read(librados::SNAP_HEAD);
4200 {
4201 bufferlist bl;
4202 bl.append("B");
4203 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 0));
4204 }
4205
4206 ioctx.snap_set_read(my_snaps[0]);
4207 // set-chunk to clone
4208 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ch_fp_oid, "foo");
4209}
4210
4211TEST_F(LibRadosTwoPoolsPP, ManifestRedirectAfterPromote) {
4212 // skip test if not yet octopus
4213 if (_get_required_osd_release(cluster) < "octopus") {
4214 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4215 }
4216
4217 // create object
4218 {
4219 bufferlist bl;
4220 bl.append("base chunk");
4221 ObjectWriteOperation op;
4222 op.write_full(bl);
4223 ASSERT_EQ(0, ioctx.operate("foo", &op));
4224 }
4225 {
4226 bufferlist bl;
4227 bl.append("BASE CHUNK");
4228 ObjectWriteOperation op;
4229 op.write_full(bl);
4230 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
4231 }
4232
4233 // set-redirect
4234 {
4235 ObjectWriteOperation op;
4236 op.set_redirect("bar", cache_ioctx, 0);
4237 librados::AioCompletion *completion = cluster.aio_create_completion();
4238 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
4239 completion->wait_for_complete();
4240 ASSERT_EQ(0, completion->get_return_value());
4241 completion->release();
4242 }
4243
4244 // promote
4245 {
4246 ObjectWriteOperation op;
4247 op.tier_promote();
4248 librados::AioCompletion *completion = cluster.aio_create_completion();
4249 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
4250 completion->wait_for_complete();
4251 ASSERT_EQ(0, completion->get_return_value());
4252 completion->release();
4253 }
4254
4255 // write
4256 {
4257 bufferlist bl;
4258 bl.append("a");
4259 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 0));
4260 }
4261
4262 // read and verify the object (redirect)
4263 {
4264 bufferlist bl;
4265 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4266 ASSERT_EQ('a', bl[0]);
4267 }
4268
4269 // read and verify the object (redirect)
4270 {
4271 bufferlist bl;
4272 ASSERT_EQ(1, cache_ioctx.read("bar", bl, 1, 0));
4273 ASSERT_EQ('B', bl[0]);
4274 }
4275}
4276
4277TEST_F(LibRadosTwoPoolsPP, ManifestCheckRefcountWhenModification) {
4278 // skip test if not yet octopus
4279 if (_get_required_osd_release(cluster) < "octopus") {
4280 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4281 }
4282
4283 bufferlist inbl;
4284 ASSERT_EQ(0, cluster.mon_command(
4285 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
4286 inbl, NULL, NULL));
4287 cluster.wait_for_latest_osdmap();
4288
4289 // create object
4290 {
4291 bufferlist bl;
4292 bl.append("there hiHI");
4293 ObjectWriteOperation op;
4294 op.write_full(bl);
4295 ASSERT_EQ(0, ioctx.operate("foo", &op));
4296 }
4297
4298 string er_fp_oid, hi_fp_oid, HI_fp_oid, ai_fp_oid, bi_fp_oid,
4299 Er_fp_oid, Hi_fp_oid, Si_fp_oid;
4300
4301 // get fp_oid
4302 er_fp_oid = get_fp_oid("er", "sha1");
4303 hi_fp_oid = get_fp_oid("hi", "sha1");
4304 HI_fp_oid = get_fp_oid("HI", "sha1");
4305 ai_fp_oid = get_fp_oid("ai", "sha1");
4306 bi_fp_oid = get_fp_oid("bi", "sha1");
4307 Er_fp_oid = get_fp_oid("Er", "sha1");
4308 Hi_fp_oid = get_fp_oid("Hi", "sha1");
4309 Si_fp_oid = get_fp_oid("Si", "sha1");
4310
4311 // write
4312 {
4313 ObjectWriteOperation op;
4314 bufferlist bl;
4315 bl.append("er");
4316 op.write_full(bl);
4317 ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
4318 }
4319 // write
4320 {
4321 ObjectWriteOperation op;
4322 bufferlist bl;
4323 bl.append("hi");
4324 op.write_full(bl);
4325 ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
4326 }
4327 // write
4328 {
4329 ObjectWriteOperation op;
4330 bufferlist bl;
4331 bl.append("HI");
4332 op.write_full(bl);
4333 ASSERT_EQ(0, cache_ioctx.operate(HI_fp_oid, &op));
4334 }
4335 // write
4336 {
4337 ObjectWriteOperation op;
4338 bufferlist bl;
4339 bl.append("ai");
4340 op.write_full(bl);
4341 ASSERT_EQ(0, cache_ioctx.operate(ai_fp_oid, &op));
4342 }
4343 // write
4344 {
4345 ObjectWriteOperation op;
4346 bufferlist bl;
4347 bl.append("bi");
4348 op.write_full(bl);
4349 ASSERT_EQ(0, cache_ioctx.operate(bi_fp_oid, &op));
4350 }
4351 // write
4352 {
4353 ObjectWriteOperation op;
4354 bufferlist bl;
4355 bl.append("Er");
4356 op.write_full(bl);
4357 ASSERT_EQ(0, cache_ioctx.operate(Er_fp_oid, &op));
4358 }
4359 // write
4360 {
4361 ObjectWriteOperation op;
4362 bufferlist bl;
4363 bl.append("Hi");
4364 op.write_full(bl);
4365 ASSERT_EQ(0, cache_ioctx.operate(Hi_fp_oid, &op));
4366 }
4367 // write
4368 {
4369 ObjectWriteOperation op;
4370 bufferlist bl;
4371 bl.append("Si");
4372 op.write_full(bl);
4373 ASSERT_EQ(0, cache_ioctx.operate(Si_fp_oid, &op));
4374 }
4375
4376 // set-chunk (dedup)
4377 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
4378 // set-chunk (dedup)
4379 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, hi_fp_oid, "foo");
4380 // set-chunk (dedup)
4381 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, HI_fp_oid, "foo");
4382
4383 // foo head: [er] [hi] [HI]
4384
4385 // create a snapshot, clone
4386 vector<uint64_t> my_snaps(1);
4387 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4388 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4389 my_snaps));
4390
4391
4392 // foo snap[0]: [er] [hi] [HI]
4393 // foo head : [er] [ai] [HI]
4394 // create a clone
4395 {
4396 bufferlist bl;
4397 bl.append("a");
4398 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
4399 }
4400 // write
4401 {
4402 bufferlist bl;
4403 bl.append("a");
4404 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
4405 }
4406
4407 // set-chunk (dedup)
4408 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ai_fp_oid, "foo");
4409
4410 // foo snap[0]: [er] [hi] [HI]
4411 // foo head : [er] [bi] [HI]
4412 // create a clone
4413 {
4414 bufferlist bl;
4415 bl.append("b");
4416 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
4417 }
4418 // write
4419 {
4420 bufferlist bl;
4421 bl.append("b");
4422 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
4423 }
4424
4425 // set-chunk (dedup)
4426 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, bi_fp_oid, "foo");
4427
4428 sleep(10);
4429
4430 // check chunk's refcount
4431 // [ai]'s refcount should be 0
4432 {
4433 bufferlist t;
4434 SHA1 sha1_gen;
4435 int size = strlen("ai");
4436 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
4437 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4438 sha1_gen.Update((const unsigned char *)"ai", size);
4439 sha1_gen.Final(fingerprint);
4440 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4441 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
4442 }
4443
4444 // foo snap[0]: [er] [hi] [HI]
4445 // foo head : [Er] [Hi] [Si]
4446 // create a clone
4447 {
4448 bufferlist bl;
4449 bl.append("thEre HiSi");
4450 ObjectWriteOperation op;
4451 op.write_full(bl);
4452 ASSERT_EQ(0, ioctx.operate("foo", &op));
4453 }
4454 // write
4455 {
4456 bufferlist bl;
4457 bl.append("thEre HiSi");
4458 ObjectWriteOperation op;
4459 op.write_full(bl);
4460 ASSERT_EQ(0, ioctx.operate("foo", &op));
4461 }
4462
4463 // set-chunk (dedup)
4464 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, Er_fp_oid, "foo");
4465 // set-chunk (dedup)
4466 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, Hi_fp_oid, "foo");
4467 // set-chunk (dedup)
4468 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, Si_fp_oid, "foo");
4469
4470 // foo snap[0]: [er] [hi] [HI]
4471 // foo head : [ER] [HI] [SI]
4472 // write
4473 {
4474 bufferlist bl;
4475 bl.append("thERe HISI");
4476 ObjectWriteOperation op;
4477 op.write_full(bl);
4478 ASSERT_EQ(0, ioctx.operate("foo", &op));
4479 }
4480
4481 sleep(10);
4482
4483 // check chunk's refcount
4484 // [Er]'s refcount should be 0
4485 {
4486 bufferlist t;
4487 SHA1 sha1_gen;
4488 int size = strlen("Er");
4489 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
4490 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
4491 sha1_gen.Update((const unsigned char *)"Er", size);
4492 sha1_gen.Final(fingerprint);
4493 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
4494 is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
4495 }
4496}
4497
4498TEST_F(LibRadosTwoPoolsPP, ManifestSnapIncCount) {
4499 // skip test if not yet octopus
4500 if (_get_required_osd_release(cluster) < "octopus") {
4501 cout << "cluster is not yet octopus, skipping test" << std::endl;
4502 return;
4503 }
4504
4505 // create object
4506 {
4507 bufferlist bl;
4508 bl.append("there hiHI");
4509 ObjectWriteOperation op;
4510 op.write_full(bl);
4511 ASSERT_EQ(0, ioctx.operate("foo", &op));
4512 }
4513 {
4514 bufferlist bl;
4515 bl.append("there hiHI");
4516 ObjectWriteOperation op;
4517 op.write_full(bl);
4518 ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
4519 }
4520 {
4521 bufferlist bl;
4522 bl.append("there hiHI");
4523 ObjectWriteOperation op;
4524 op.write_full(bl);
4525 ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
4526 }
4527 {
4528 bufferlist bl;
4529 bl.append("there hiHI");
4530 ObjectWriteOperation op;
4531 op.write_full(bl);
4532 ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
4533 }
4534 {
4535 bufferlist bl;
4536 bl.append("there hiHI");
4537 ObjectWriteOperation op;
4538 op.write_full(bl);
4539 ASSERT_EQ(0, cache_ioctx.operate("chunk4", &op));
4540 }
4541
4542 // wait for maps to settle
4543 cluster.wait_for_latest_osdmap();
4544
4545 // create a snapshot, clone
4546 vector<uint64_t> my_snaps(1);
4547 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4548 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4549 my_snaps));
4550
4551 {
4552 bufferlist bl;
4553 bl.append("there hiHI");
4554 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4555 }
4556
4557 my_snaps.resize(2);
4558 my_snaps[1] = my_snaps[0];
4559 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4560 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4561 my_snaps));
4562
4563 {
4564 bufferlist bl;
4565 bl.append("there hiHI");
4566 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4567 }
4568
4569 // set-chunk
4570 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
4571 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4572 // foo snap[1]:
4573 // foo snap[0]:
4574 // foo head : [chunk1] [chunk4]
4575
4576 ioctx.snap_set_read(my_snaps[1]);
4577 // set-chunk
4578 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
4579 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4580 // foo snap[1]: [chunk2] [chunk4]
4581 // foo snap[0]:
4582 // foo head : [chunk1] [chunk4]
4583
4584 ioctx.snap_set_read(my_snaps[0]);
4585 // set-chunk
4586 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
4587 // foo snap[1]: [chunk2] [chunk4]
4588 // foo snap[0]: [chunk2]
4589 // foo head : [chunk1] [chunk4]
4590
4591 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk3", "foo");
4592 // foo snap[1]: [chunk2] [chunk4]
4593 // foo snap[0]: [chunk3] [chunk2]
4594 // foo head : [chunk1] [chunk4]
4595 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4596 // foo snap[1]: [chunk2] [chunk4]
4597 // foo snap[0]: [chunk3] [chunk2] [chunk4]
4598 // foo head : [chunk1] [chunk4]
4599
4600 // check chunk's refcount
4601 check_fp_oid_refcount(cache_ioctx, "chunk1", 1u, "");
4602
4603 // check chunk's refcount
4604 check_fp_oid_refcount(cache_ioctx, "chunk2", 1u, "");
4605
4606 // check chunk's refcount
4607 check_fp_oid_refcount(cache_ioctx, "chunk3", 1u, "");
4608 sleep(10);
4609
4610 // check chunk's refcount
4611 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk4", 1);
4612}
4613
4614TEST_F(LibRadosTwoPoolsPP, ManifestEvict) {
4615 // skip test if not yet octopus
4616 if (_get_required_osd_release(cluster) < "octopus") {
4617 cout << "cluster is not yet octopus, skipping test" << std::endl;
4618 return;
4619 }
4620
4621 // create object
4622 {
4623 bufferlist bl;
4624 bl.append("there hiHI");
4625 ObjectWriteOperation op;
4626 op.write_full(bl);
4627 ASSERT_EQ(0, ioctx.operate("foo", &op));
4628 }
4629 {
4630 bufferlist bl;
4631 bl.append("there hiHI");
4632 ObjectWriteOperation op;
4633 op.write_full(bl);
4634 ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
4635 }
4636 {
4637 bufferlist bl;
4638 bl.append("there hiHI");
4639 ObjectWriteOperation op;
4640 op.write_full(bl);
4641 ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
4642 }
4643 {
4644 bufferlist bl;
4645 bl.append("there hiHI");
4646 ObjectWriteOperation op;
4647 op.write_full(bl);
4648 ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
4649 }
4650 {
4651 bufferlist bl;
4652 bl.append("there hiHI");
4653 ObjectWriteOperation op;
4654 op.write_full(bl);
4655 ASSERT_EQ(0, cache_ioctx.operate("chunk4", &op));
4656 }
4657
4658 // wait for maps to settle
4659 cluster.wait_for_latest_osdmap();
4660
4661 // create a snapshot, clone
4662 vector<uint64_t> my_snaps(1);
4663 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4664 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4665 my_snaps));
4666
4667 {
4668 bufferlist bl;
4669 bl.append("there hiHI");
4670 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4671 }
4672
4673 my_snaps.resize(2);
4674 my_snaps[1] = my_snaps[0];
4675 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4676 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4677 my_snaps));
4678
4679 {
4680 bufferlist bl;
4681 bl.append("there hiHI");
4682 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4683 }
4684
4685 // set-chunk
4686 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
4687 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4688 // foo snap[1]:
4689 // foo snap[0]:
4690 // foo head : [chunk1] [chunk4]
4691
4692 ioctx.snap_set_read(my_snaps[1]);
4693 // set-chunk
4694 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
4695 // foo snap[1]: [ chunk2 ]
4696 // foo snap[0]:
4697 // foo head : [chunk1] [chunk4]
4698
4699 ioctx.snap_set_read(my_snaps[0]);
4700 // set-chunk
4701 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
4702 // foo snap[1]: [ chunk2 ]
4703 // foo snap[0]: [chunk2]
4704 // foo head : [chunk1] [chunk4]
4705
4706 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk3", "foo");
4707 // foo snap[1]: [ chunk2 ]
4708 // foo snap[0]: [chunk3] [chunk2]
4709 // foo head : [chunk1] [chunk4]
4710 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
4711 // foo snap[1]: [ chunk2 ]
4712 // foo snap[0]: [chunk3] [chunk2] [chunk4]
4713 // foo head : [chunk1] [chunk4]
4714 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, "chunk4", "foo");
4715 // foo snap[1]: [ chunk2 ]
4716 // foo snap[0]: [chunk4] [chunk3] [chunk2] [chunk4]
4717 // foo head : [chunk1] [chunk4]
4718 manifest_set_chunk(cluster, cache_ioctx, ioctx, 4, 2, "chunk1", "foo");
4719 // foo snap[1]: [ chunk2 ]
4720 // foo snap[0]: [chunk4] [chunk3] [chunk1] [chunk2] [chunk4]
4721 // foo head : [chunk1] [chunk4]
4722
4723 {
4724 ObjectReadOperation op, stat_op;
4725 uint64_t size;
4726 op.tier_evict();
4727 librados::AioCompletion *completion = cluster.aio_create_completion();
4728 ASSERT_EQ(0, ioctx.aio_operate(
4729 "foo", completion, &op,
4730 librados::OPERATION_IGNORE_OVERLAY, NULL));
4731 completion->wait_for_complete();
4732 ASSERT_EQ(0, completion->get_return_value());
4733
4734 stat_op.stat(&size, NULL, NULL);
4735 ASSERT_EQ(0, ioctx.operate("foo", &stat_op, NULL));
4736 ASSERT_EQ(10, size);
4737 }
4738
4739 ioctx.snap_set_read(librados::SNAP_HEAD);
4740 {
4741 ObjectReadOperation op, stat_op;
4742 uint64_t size;
4743 op.tier_evict();
4744 librados::AioCompletion *completion = cluster.aio_create_completion();
4745 ASSERT_EQ(0, ioctx.aio_operate(
4746 "foo", completion, &op,
4747 librados::OPERATION_IGNORE_OVERLAY, NULL));
4748 completion->wait_for_complete();
4749 ASSERT_EQ(0, completion->get_return_value());
4750
4751 stat_op.stat(&size, NULL, NULL);
4752 ASSERT_EQ(0, ioctx.operate("foo", &stat_op, NULL));
4753 ASSERT_EQ(strlen("there hiHI"), size);
4754 }
4755
4756}
4757
20effc67
TL		 4758TEST_F(LibRadosTwoPoolsPP, ManifestEvictPromote) {
4759 // skip test if not yet octopus
4760 if (_get_required_osd_release(cluster) < "octopus") {
4761 cout << "cluster is not yet octopus, skipping test" << std::endl;
4762 return;
4763 }
4764
4765 // create object
4766 {
4767 bufferlist bl;
4768 bl.append("there hiHI");
4769 ObjectWriteOperation op;
4770 op.write_full(bl);
4771 ASSERT_EQ(0, ioctx.operate("foo", &op));
4772 }
4773 {
4774 bufferlist bl;
4775 bl.append("EREHT hiHI");
4776 ObjectWriteOperation op;
4777 op.write_full(bl);
4778 ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
4779 }
4780 {
4781 bufferlist bl;
4782 bl.append("there hiHI");
4783 ObjectWriteOperation op;
4784 op.write_full(bl);
4785 ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
4786 }
4787 {
4788 bufferlist bl;
4789 bl.append("THERE HIHI");
4790 ObjectWriteOperation op;
4791 op.write_full(bl);
4792 ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
4793 }
4794
4795 // wait for maps to settle
4796 cluster.wait_for_latest_osdmap();
4797
4798 // create a snapshot, clone
4799 vector<uint64_t> my_snaps(1);
4800 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4801 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4802 my_snaps));
4803
4804 {
4805 bufferlist bl;
4806 bl.append("there");
4807 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
4808 }
4809
4810 // set-chunk
4811 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, "chunk1", "foo");
4812 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk2", "foo");
4813 // foo snap[0]:
4814 // foo head : [chunk1] [chunk2]
4815
4816 ioctx.snap_set_read(my_snaps[0]);
4817 // set-chunk
4818 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk3", "foo");
4819 // foo snap[0]: [ chunk3 ]
4820 // foo head : [chunk1] [chunk2]
4821
4822
4823 {
4824 ObjectReadOperation op, stat_op;
4825 uint64_t size;
4826 op.tier_evict();
4827 librados::AioCompletion *completion = cluster.aio_create_completion();
4828 ASSERT_EQ(0, ioctx.aio_operate(
4829 "foo", completion, &op,
4830 librados::OPERATION_IGNORE_OVERLAY, NULL));
4831 completion->wait_for_complete();
4832 ASSERT_EQ(0, completion->get_return_value());
4833
4834 stat_op.stat(&size, NULL, NULL);
4835 ASSERT_EQ(0, ioctx.operate("foo", &stat_op, NULL));
4836 ASSERT_EQ(10, size);
4837
4838 }
4839 {
4840 bufferlist bl;
4841 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4842 ASSERT_EQ('T', bl[0]);
4843 }
4844
4845 ioctx.snap_set_read(librados::SNAP_HEAD);
4846 {
4847 bufferlist bl;
4848 ASSERT_EQ(10, ioctx.read("foo", bl, 10, 0));
4849 ASSERT_EQ('H', bl[8]);
4850 }
4851}
4852
f67539c2
TL		 4853
4854TEST_F(LibRadosTwoPoolsPP, ManifestSnapSizeMismatch) {
4855 // skip test if not yet octopus
4856 if (_get_required_osd_release(cluster) < "octopus") {
4857 cout << "cluster is not yet octopus, skipping test" << std::endl;
4858 return;
4859 }
4860
4861 // create object
4862 {
4863 bufferlist bl;
4864 bl.append("there hiHI");
4865 ObjectWriteOperation op;
4866 op.write_full(bl);
4867 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
4868 }
4869 {
4870 bufferlist bl;
4871 bl.append("there hiHI");
4872 ObjectWriteOperation op;
4873 op.write_full(bl);
4874 ASSERT_EQ(0, ioctx.operate("chunk1", &op));
4875 }
4876 {
4877 bufferlist bl;
4878 bl.append("there HIHI");
4879 ObjectWriteOperation op;
4880 op.write_full(bl);
4881 ASSERT_EQ(0, ioctx.operate("chunk2", &op));
4882 }
4883
4884 // wait for maps to settle
4885 cluster.wait_for_latest_osdmap();
4886
4887 // create a snapshot, clone
4888 vector<uint64_t> my_snaps(1);
4889 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
4890 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4891 my_snaps));
4892
4893 {
4894 bufferlist bl;
4895 bl.append("There hiHI");
4896 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
4897 }
4898
4899 my_snaps.resize(2);
4900 my_snaps[1] = my_snaps[0];
4901 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
4902 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4903 my_snaps));
4904
4905 {
4906 bufferlist bl;
4907 bl.append("tHere hiHI");
4908 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
4909 }
4910
4911 // set-chunk
4912 manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "chunk1", "foo");
4913
4914 cache_ioctx.snap_set_read(my_snaps[1]);
4915
4916 // set-chunk
4917 manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "chunk2", "foo");
4918
4919 // evict
4920 {
4921 ObjectReadOperation op, stat_op;
4922 op.tier_evict();
4923 librados::AioCompletion *completion = cluster.aio_create_completion();
4924 ASSERT_EQ(0, cache_ioctx.aio_operate(
4925 "foo", completion, &op,
4926 librados::OPERATION_IGNORE_OVERLAY, NULL));
4927 completion->wait_for_complete();
4928 ASSERT_EQ(0, completion->get_return_value());
4929 }
4930
4931 uint32_t hash;
4932 ASSERT_EQ(0, cache_ioctx.get_object_pg_hash_position2("foo", &hash));
4933
4934 // scrub
4935 {
4936 for (int tries = 0; tries < 5; ++tries) {
4937 bufferlist inbl;
4938 ostringstream ss;
4939 ss << "{\"prefix\": \"pg deep-scrub\", \"pgid\": \""
4940 << cache_ioctx.get_id() << "."
4941 << std::hex << hash
4942 << "\"}";
4943 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
4944 if (r == -ENOENT ||
4945 r == -EAGAIN) {
4946 sleep(5);
4947 continue;
4948 }
4949 ASSERT_EQ(0, r);
4950 break;
4951 }
4952 cout << "waiting for scrubs..." << std::endl;
4953 sleep(20);
4954 cout << "done waiting" << std::endl;
4955 }
4956
4957 {
4958 bufferlist bl;
4959 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
4960 ASSERT_EQ('t', bl[0]);
4961 }
4962}
4963
4964#include <common/CDC.h>
4965TEST_F(LibRadosTwoPoolsPP, DedupFlushRead) {
4966 // skip test if not yet octopus
4967 if (_get_required_osd_release(cluster) < "octopus") {
4968 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
4969 }
4970
4971 bufferlist inbl;
4972 ASSERT_EQ(0, cluster.mon_command(
4973 set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
4974 inbl, NULL, NULL));
4975 ASSERT_EQ(0, cluster.mon_command(
4976 set_pool_str(cache_pool_name, "dedup_tier", pool_name),
4977 inbl, NULL, NULL));
4978 ASSERT_EQ(0, cluster.mon_command(
4979 set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
4980 inbl, NULL, NULL));
4981 ASSERT_EQ(0, cluster.mon_command(
4982 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
4983 inbl, NULL, NULL));
4984
4985 // wait for maps to settle
4986 cluster.wait_for_latest_osdmap();
4987
4988 // create object
4989 bufferlist gbl;
4990 {
4991 generate_buffer(1024*8, &gbl);
4992 ObjectWriteOperation op;
4993 op.write_full(gbl);
4994 ASSERT_EQ(0, cache_ioctx.operate("foo-chunk", &op));
4995 }
4996 {
4997 bufferlist bl;
4998 bl.append("DDse chunk");
4999 ObjectWriteOperation op;
5000 op.write_full(bl);
5001 ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
5002 }
5003
5004 // set-chunk to set manifest object
5005 {
5006 ObjectReadOperation op;
5007 op.set_chunk(0, 2, ioctx, "bar-chunk", 0,
5008 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
5009 librados::AioCompletion *completion = cluster.aio_create_completion();
5010 ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op,
5011 librados::OPERATION_IGNORE_CACHE, NULL));
5012 completion->wait_for_complete();
5013 ASSERT_EQ(0, completion->get_return_value());
5014 completion->release();
5015 }
5016 // flush
5017 {
5018 ObjectReadOperation op;
5019 op.tier_flush();
5020 librados::AioCompletion *completion = cluster.aio_create_completion();
5021 ASSERT_EQ(0, cache_ioctx.aio_operate(
5022 "foo-chunk", completion, &op,
5023 librados::OPERATION_IGNORE_CACHE, NULL));
5024 completion->wait_for_complete();
5025 ASSERT_EQ(0, completion->get_return_value());
5026 completion->release();
5027 }
5028
5029 std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
5030 vector<pair<uint64_t, uint64_t>> chunks;
5031 bufferlist chunk;
5032 cdc->calc_chunks(gbl, &chunks);
5033 chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
5034 string tgt_oid;
5035 {
5036 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5037 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5038 SHA1 sha1_gen;
5039 int size = chunk.length();
5040 sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
5041 sha1_gen.Final(fingerprint);
5042 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5043 tgt_oid = string(p_str);
5044 }
5045
5046 // read and verify the chunked object
5047 {
5048 bufferlist test_bl;
5049 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
5050 ASSERT_EQ(test_bl[1], chunk[1]);
5051 }
5052
5053 ASSERT_EQ(0, cluster.mon_command(
5054 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 512),
5055 inbl, NULL, NULL));
5056 cluster.wait_for_latest_osdmap();
5057
5058 // make a dirty chunks
5059 {
5060 bufferlist bl;
5061 bl.append("hi");
5062 ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
5063 }
5064
5065 // flush
5066 {
5067 ObjectReadOperation op;
5068 op.tier_flush();
5069 librados::AioCompletion *completion = cluster.aio_create_completion();
5070 ASSERT_EQ(0, cache_ioctx.aio_operate(
5071 "foo-chunk", completion, &op,
5072 librados::OPERATION_IGNORE_CACHE, NULL));
5073 completion->wait_for_complete();
5074 ASSERT_EQ(0, completion->get_return_value());
5075 completion->release();
5076 }
5077
5078 cdc = CDC::create("fastcdc", cbits(512)-1);
5079 chunks.clear();
5080 cdc->calc_chunks(gbl, &chunks);
5081 bufferlist chunk_512;
5082 chunk_512.substr_of(gbl, chunks[3].first, chunks[3].second);
5083 {
5084 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5085 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5086 SHA1 sha1_gen;
5087 int size = chunk_512.length();
5088 sha1_gen.Update((const unsigned char *)chunk_512.c_str(), size);
5089 sha1_gen.Final(fingerprint);
5090 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5091 tgt_oid = string(p_str);
5092 }
5093
5094 // read and verify the chunked object
5095 {
5096 bufferlist test_bl;
5097 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
5098 ASSERT_EQ(test_bl[1], chunk_512[1]);
5099 }
5100
5101 ASSERT_EQ(0, cluster.mon_command(
5102 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 16384),
5103 inbl, NULL, NULL));
5104 cluster.wait_for_latest_osdmap();
5105
5106 // make a dirty chunks
5107 {
5108 bufferlist bl;
5109 bl.append("hi");
5110 ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
5111 gbl.begin(0).copy_in(bl.length(), bl);
5112 }
5113 // flush
5114 {
5115 ObjectReadOperation op;
5116 op.tier_flush();
5117 librados::AioCompletion *completion = cluster.aio_create_completion();
5118 ASSERT_EQ(0, cache_ioctx.aio_operate(
5119 "foo-chunk", completion, &op,
5120 librados::OPERATION_IGNORE_CACHE, NULL));
5121 completion->wait_for_complete();
5122 ASSERT_EQ(0, completion->get_return_value());
5123 completion->release();
5124 }
5125
5126 cdc = CDC::create("fastcdc", cbits(16384)-1);
5127 chunks.clear();
5128 cdc->calc_chunks(gbl, &chunks);
5129 bufferlist chunk_16384;
5130 chunk_16384.substr_of(gbl, chunks[0].first, chunks[0].second);
5131 {
5132 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5133 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5134 SHA1 sha1_gen;
5135 int size = chunk_16384.length();
5136 sha1_gen.Update((const unsigned char *)chunk_16384.c_str(), size);
5137 sha1_gen.Final(fingerprint);
5138 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5139 tgt_oid = string(p_str);
5140 }
5141 // read and verify the chunked object
5142 {
5143 bufferlist test_bl;
5144 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
5145 ASSERT_EQ(test_bl[0], chunk_16384[0]);
5146 }
5147
5148 // less than object size
5149 ASSERT_EQ(0, cluster.mon_command(
5150 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
5151 inbl, NULL, NULL));
5152 cluster.wait_for_latest_osdmap();
5153
5154 // make a dirty chunks
5155 // a chunk_info is deleted by write, which converts the manifest object to non-manifest object
5156 {
5157 bufferlist bl;
5158 bl.append("hi");
5159 ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
5160 }
5161
5162 // reset set-chunk
5163 {
5164 bufferlist bl;
5165 bl.append("DDse chunk");
5166 ObjectWriteOperation op;
5167 op.write_full(bl);
5168 ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
5169 }
5170 // set-chunk to set manifest object
5171 {
5172 ObjectReadOperation op;
5173 op.set_chunk(0, 2, ioctx, "bar-chunk", 0,
5174 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
5175 librados::AioCompletion *completion = cluster.aio_create_completion();
5176 ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op,
5177 librados::OPERATION_IGNORE_CACHE, NULL));
5178 completion->wait_for_complete();
5179 ASSERT_EQ(0, completion->get_return_value());
5180 completion->release();
5181 }
5182 // flush
5183 {
5184 ObjectReadOperation op;
5185 op.tier_flush();
5186 librados::AioCompletion *completion = cluster.aio_create_completion();
5187 ASSERT_EQ(0, cache_ioctx.aio_operate(
5188 "foo-chunk", completion, &op,
5189 librados::OPERATION_IGNORE_CACHE, NULL));
5190 completion->wait_for_complete();
5191 ASSERT_EQ(0, completion->get_return_value());
5192 completion->release();
5193 }
5194
5195 cdc = CDC::create("fastcdc", cbits(1024)-1);
5196 chunks.clear();
5197 cdc->calc_chunks(gbl, &chunks);
5198 bufferlist small_chunk;
5199 small_chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
5200 {
5201 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5202 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5203 SHA1 sha1_gen;
5204 int size = small_chunk.length();
5205 sha1_gen.Update((const unsigned char *)small_chunk.c_str(), size);
5206 sha1_gen.Final(fingerprint);
5207 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5208 tgt_oid = string(p_str);
5209 }
5210 // read and verify the chunked object
5211 {
5212 bufferlist test_bl;
5213 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
5214 ASSERT_EQ(test_bl[0], small_chunk[0]);
5215 }
5216
5217}
5218
5219TEST_F(LibRadosTwoPoolsPP, ManifestFlushSnap) {
5220 // skip test if not yet octopus
5221 if (_get_required_osd_release(cluster) < "octopus") {
5222 cout << "cluster is not yet octopus, skipping test" << std::endl;
5223 return;
5224 }
5225
5226 bufferlist inbl;
5227 ASSERT_EQ(0, cluster.mon_command(
5228 set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
5229 inbl, NULL, NULL));
5230 ASSERT_EQ(0, cluster.mon_command(
5231 set_pool_str(cache_pool_name, "dedup_tier", pool_name),
5232 inbl, NULL, NULL));
5233 ASSERT_EQ(0, cluster.mon_command(
5234 set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
5235 inbl, NULL, NULL));
5236 ASSERT_EQ(0, cluster.mon_command(
5237 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
5238 inbl, NULL, NULL));
5239
5240 // wait for maps to settle
5241 cluster.wait_for_latest_osdmap();
5242
5243 // create object
5244 bufferlist gbl;
5245 {
5246 //bufferlist bl;
5247 //bl.append("there hi");
5248 generate_buffer(1024*8, &gbl);
5249 ObjectWriteOperation op;
5250 op.write_full(gbl);
5251 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
5252 }
5253 {
5254 bufferlist bl;
5255 bl.append("there hi");
5256 ObjectWriteOperation op;
5257 op.write_full(bl);
5258 ASSERT_EQ(0, ioctx.operate("bar", &op));
5259 }
5260
5261 // set-chunk (dedup)
5262 manifest_set_chunk(cluster, ioctx, cache_ioctx, 2, 2, "bar", "foo");
20effc67 5263 manifest_set_chunk(cluster, ioctx, cache_ioctx, 6, 2, "bar", "foo");
f67539c2
TL		 5264
5265 // create a snapshot, clone
5266 vector<uint64_t> my_snaps(1);
5267 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
5268 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5269 my_snaps));
5270
5271 // make a dirty chunks
5272 {
5273 bufferlist bl;
5274 bl.append("Thbbe");
5275 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5276 }
5277
5278 // and another
5279 my_snaps.resize(2);
5280 my_snaps[1] = my_snaps[0];
5281 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
5282 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5283 my_snaps));
5284
5285 // make a dirty chunks
5286 {
5287 bufferlist bl;
5288 bl.append("Thcce");
5289 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5290 }
5291
5292 // flush on head (should fail)
5293 cache_ioctx.snap_set_read(librados::SNAP_HEAD);
5294 // flush
5295 {
5296 ObjectReadOperation op;
5297 op.tier_flush();
5298 librados::AioCompletion *completion = cluster.aio_create_completion();
5299 ASSERT_EQ(0, cache_ioctx.aio_operate(
5300 "foo", completion, &op,
5301 librados::OPERATION_IGNORE_CACHE, NULL));
5302 completion->wait_for_complete();
5303 ASSERT_EQ(-EBUSY, completion->get_return_value());
5304 completion->release();
5305 }
5306
5307 // flush on recent snap (should fail)
5308 cache_ioctx.snap_set_read(my_snaps[0]);
5309 {
5310 ObjectReadOperation op;
5311 op.tier_flush();
5312 librados::AioCompletion *completion = cluster.aio_create_completion();
5313 ASSERT_EQ(0, cache_ioctx.aio_operate(
5314 "foo", completion, &op,
5315 librados::OPERATION_IGNORE_CACHE, NULL));
5316 completion->wait_for_complete();
5317 ASSERT_EQ(-EBUSY, completion->get_return_value());
5318 completion->release();
5319 }
5320
5321 // flush on oldest snap
5322 cache_ioctx.snap_set_read(my_snaps[1]);
5323 {
5324 ObjectReadOperation op;
5325 op.tier_flush();
5326 librados::AioCompletion *completion = cluster.aio_create_completion();
5327 ASSERT_EQ(0, cache_ioctx.aio_operate(
5328 "foo", completion, &op,
5329 librados::OPERATION_IGNORE_CACHE, NULL));
5330 completion->wait_for_complete();
5331 ASSERT_EQ(0, completion->get_return_value());
5332 completion->release();
5333 }
5334
5335 // flush on oldest snap
5336 cache_ioctx.snap_set_read(my_snaps[0]);
5337 {
5338 ObjectReadOperation op;
5339 op.tier_flush();
5340 librados::AioCompletion *completion = cluster.aio_create_completion();
5341 ASSERT_EQ(0, cache_ioctx.aio_operate(
5342 "foo", completion, &op,
5343 librados::OPERATION_IGNORE_CACHE, NULL));
5344 completion->wait_for_complete();
5345 ASSERT_EQ(0, completion->get_return_value());
5346 completion->release();
5347 }
5348
5349 // flush on oldest snap
5350 cache_ioctx.snap_set_read(librados::SNAP_HEAD);
5351 {
5352 ObjectReadOperation op;
5353 op.tier_flush();
5354 librados::AioCompletion *completion = cluster.aio_create_completion();
5355 ASSERT_EQ(0, cache_ioctx.aio_operate(
5356 "foo", completion, &op,
5357 librados::OPERATION_IGNORE_CACHE, NULL));
5358 completion->wait_for_complete();
5359 ASSERT_EQ(0, completion->get_return_value());
5360 completion->release();
5361 }
5362
5363 // check chunk's refcount
5364 std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
5365 vector<pair<uint64_t, uint64_t>> chunks;
5366 bufferlist chunk;
5367 cdc->calc_chunks(gbl, &chunks);
5368 chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
5369 string tgt_oid;
5370 {
5371 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5372 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5373 SHA1 sha1_gen;
5374 int size = chunk.length();
5375 sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
5376 sha1_gen.Final(fingerprint);
5377 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5378 tgt_oid = string(p_str);
5379 }
5380 // read and verify the chunked object
5381 {
5382 bufferlist test_bl;
5383 ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
5384 ASSERT_EQ(test_bl[1], chunk[1]);
5385 }
5386
5387 cache_ioctx.snap_set_read(librados::SNAP_HEAD);
5388 {
5389 bufferlist bl;
5390 ASSERT_EQ(4, cache_ioctx.read("foo", bl, 4, 0));
5391 ASSERT_EQ('c', bl[2]);
5392 }
5393
5394 cache_ioctx.snap_set_read(my_snaps[0]);
5395 {
5396 bufferlist bl;
5397 ASSERT_EQ(4, cache_ioctx.read("foo", bl, 4, 0));
5398 ASSERT_EQ('b', bl[2]);
5399 }
5400}
5401
5402TEST_F(LibRadosTwoPoolsPP, ManifestFlushDupCount) {
5403 // skip test if not yet octopus
5404 if (_get_required_osd_release(cluster) < "octopus") {
5405 cout << "cluster is not yet octopus, skipping test" << std::endl;
5406 return;
5407 }
5408
5409 bufferlist inbl;
5410 ASSERT_EQ(0, cluster.mon_command(
5411 set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
5412 inbl, NULL, NULL));
5413 ASSERT_EQ(0, cluster.mon_command(
5414 set_pool_str(cache_pool_name, "dedup_tier", pool_name),
5415 inbl, NULL, NULL));
5416 ASSERT_EQ(0, cluster.mon_command(
5417 set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
5418 inbl, NULL, NULL));
5419 ASSERT_EQ(0, cluster.mon_command(
5420 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
5421 inbl, NULL, NULL));
5422
5423 // create object
5424 bufferlist gbl;
5425 {
5426 //bufferlist bl;
5427 generate_buffer(1024*8, &gbl);
5428 ObjectWriteOperation op;
5429 op.write_full(gbl);
5430 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
5431 }
5432 {
5433 bufferlist bl;
5434 bl.append("there hiHI");
5435 ObjectWriteOperation op;
5436 op.write_full(bl);
5437 ASSERT_EQ(0, ioctx.operate("bar", &op));
9f95a23c
TL		 5438 }
5439
9f95a23c
TL		 5440 // wait for maps to settle
5441 cluster.wait_for_latest_osdmap();
5442
f67539c2
TL		 5443 // set-chunk to set manifest object
5444 {
5445 ObjectReadOperation op;
5446 op.set_chunk(0, 2, ioctx, "bar", 0,
5447 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
5448 librados::AioCompletion *completion = cluster.aio_create_completion();
5449 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
5450 librados::OPERATION_IGNORE_CACHE, NULL));
5451 completion->wait_for_complete();
5452 ASSERT_EQ(0, completion->get_return_value());
5453 completion->release();
5454 }
5455
5456 // create a snapshot, clone
5457 vector<uint64_t> my_snaps(1);
5458 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
5459 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5460 my_snaps));
5461
5462 // make a dirty chunks
5463 {
5464 bufferlist bl;
5465 bl.append("Thbbe hi");
5466 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5467 }
5468
5469 // and another
5470 my_snaps.resize(2);
5471 my_snaps[1] = my_snaps[0];
5472 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
5473 ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5474 my_snaps));
5475
5476 // make a dirty chunks
5477 {
5478 bufferlist bl;
5479 bl.append("Thcce hi");
5480 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5481 }
5482
5483 //flush on oldest snap
5484 cache_ioctx.snap_set_read(my_snaps[1]);
5485 // flush
5486 {
5487 ObjectReadOperation op;
5488 op.tier_flush();
5489 librados::AioCompletion *completion = cluster.aio_create_completion();
5490 ASSERT_EQ(0, cache_ioctx.aio_operate(
5491 "foo", completion, &op,
5492 librados::OPERATION_IGNORE_CACHE, NULL));
5493 completion->wait_for_complete();
5494 ASSERT_EQ(0, completion->get_return_value());
5495 completion->release();
5496 }
5497
5498 // flush on oldest snap
5499 cache_ioctx.snap_set_read(my_snaps[0]);
5500 // flush
5501 {
5502 ObjectReadOperation op;
5503 op.tier_flush();
5504 librados::AioCompletion *completion = cluster.aio_create_completion();
5505 ASSERT_EQ(0, cache_ioctx.aio_operate(
5506 "foo", completion, &op,
5507 librados::OPERATION_IGNORE_CACHE, NULL));
5508 completion->wait_for_complete();
5509 ASSERT_EQ(0, completion->get_return_value());
5510 completion->release();
5511 }
5512
5513 cache_ioctx.snap_set_read(librados::SNAP_HEAD);
5514 // flush
5515 {
5516 ObjectReadOperation op;
5517 op.tier_flush();
5518 librados::AioCompletion *completion = cluster.aio_create_completion();
5519 ASSERT_EQ(0, cache_ioctx.aio_operate(
5520 "foo", completion, &op,
5521 librados::OPERATION_IGNORE_CACHE, NULL));
5522 completion->wait_for_complete();
5523 ASSERT_EQ(0, completion->get_return_value());
5524 completion->release();
5525 }
5526
5527 std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
5528 vector<pair<uint64_t, uint64_t>> chunks;
5529 bufferlist chunk;
5530 cdc->calc_chunks(gbl, &chunks);
5531 chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
5532 string tgt_oid;
5533 // check chunk's refcount
5534 {
5535 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5536 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5537 bufferlist t;
5538 SHA1 sha1_gen;
5539 int size = chunk.length();
5540 sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
5541 sha1_gen.Final(fingerprint);
5542 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5543 tgt_oid = string(p_str);
5544 ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
5545 chunk_refs_t refs;
5546 try {
5547 auto iter = t.cbegin();
5548 decode(refs, iter);
5549 } catch (buffer::error& err) {
5550 ASSERT_TRUE(0);
5551 }
a4b75251 5552 ASSERT_LE(1u, refs.count());
f67539c2
TL		 5553 }
5554
5555 bufferlist chunk2;
5556 chunk2.substr_of(gbl, chunks[0].first, chunks[0].second);
5557 // check chunk's refcount
5558 {
5559 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5560 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5561 bufferlist t;
5562 SHA1 sha1_gen;
5563 int size = chunk2.length();
5564 sha1_gen.Update((const unsigned char *)chunk2.c_str(), size);
5565 sha1_gen.Final(fingerprint);
5566 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5567 tgt_oid = string(p_str);
5568 ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
5569 chunk_refs_t refs;
5570 try {
5571 auto iter = t.cbegin();
5572 decode(refs, iter);
5573 } catch (buffer::error& err) {
5574 ASSERT_TRUE(0);
5575 }
a4b75251 5576 ASSERT_LE(1u, refs.count());
f67539c2
TL		 5577 }
5578
5579 // make a dirty chunks
5580 {
5581 bufferlist bl;
5582 bl.append("ThDDe hi");
5583 ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
5584 }
5585
5586 // flush
9f95a23c 5587 {
f67539c2
TL		 5588 ObjectReadOperation op;
5589 op.tier_flush();
9f95a23c 5590 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2
TL		 5591 ASSERT_EQ(0, cache_ioctx.aio_operate(
5592 "foo", completion, &op,
5593 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c
TL		 5594 completion->wait_for_complete();
5595 ASSERT_EQ(0, completion->get_return_value());
5596 completion->release();
5597 }
f67539c2
TL		 5598
5599 bufferlist tmp;
5600 tmp.append("Thcce hi");
5601 gbl.begin(0).copy_in(tmp.length(), tmp);
5602 bufferlist chunk3;
5603 cdc->calc_chunks(gbl, &chunks);
5604 chunk3.substr_of(gbl, chunks[0].first, chunks[0].second);
5605 // check chunk's refcount
5606 {
5607 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
5608 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
5609 bufferlist t;
5610 SHA1 sha1_gen;
5611 int size = chunk2.length();
5612 sha1_gen.Update((const unsigned char *)chunk2.c_str(), size);
5613 sha1_gen.Final(fingerprint);
5614 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
5615 is_intended_refcount_state(cache_ioctx, "foo", ioctx, p_str, 0);
5616 }
5617}
5618
5619TEST_F(LibRadosTwoPoolsPP, TierFlushDuringFlush) {
5620 // skip test if not yet octopus
5621 if (_get_required_osd_release(cluster) < "octopus") {
5622 cout << "cluster is not yet octopus, skipping test" << std::endl;
5623 return;
5624 }
5625
5626 bufferlist inbl;
5627
5628 // create a new pool
5629 std::string temp_pool_name = get_temp_pool_name() + "-test-flush";
5630 ASSERT_EQ(0, cluster.pool_create(temp_pool_name.c_str()));
5631
5632 ASSERT_EQ(0, cluster.mon_command(
5633 set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
5634 inbl, NULL, NULL));
5635 ASSERT_EQ(0, cluster.mon_command(
5636 set_pool_str(cache_pool_name, "dedup_tier", temp_pool_name),
5637 inbl, NULL, NULL));
5638 ASSERT_EQ(0, cluster.mon_command(
5639 set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
5640 inbl, NULL, NULL));
5641 ASSERT_EQ(0, cluster.mon_command(
5642 set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
5643 inbl, NULL, NULL));
5644
5645 // create object
5646 bufferlist gbl;
5647 {
5648 //bufferlist bl;
5649 generate_buffer(1024*8, &gbl);
5650 ObjectWriteOperation op;
5651 op.write_full(gbl);
5652 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
5653 }
9f95a23c 5654 {
f67539c2
TL		 5655 bufferlist bl;
5656 bl.append("there hiHI");
9f95a23c 5657 ObjectWriteOperation op;
f67539c2
TL		 5658 op.write_full(bl);
5659 ASSERT_EQ(0, ioctx.operate("bar", &op));
5660 }
5661
5662 // set-chunk to set manifest object
5663 {
5664 ObjectReadOperation op;
5665 op.set_chunk(0, 2, ioctx, "bar", 0,
5666 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
9f95a23c 5667 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2
TL		 5668 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
5669 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c
TL		 5670 completion->wait_for_complete();
5671 ASSERT_EQ(0, completion->get_return_value());
5672 completion->release();
5673 }
f67539c2
TL		 5674
5675 // delete temp pool, so flushing chunk will fail
5676 ASSERT_EQ(0, s_cluster.pool_delete(temp_pool_name.c_str()));
5677
5678 // flush to check if proper error is returned
5679 {
5680 ObjectReadOperation op;
5681 op.tier_flush();
5682 librados::AioCompletion *completion = cluster.aio_create_completion();
5683 ASSERT_EQ(0, cache_ioctx.aio_operate(
5684 "foo", completion, &op,
5685 librados::OPERATION_IGNORE_CACHE, NULL));
5686 completion->wait_for_complete();
5687 ASSERT_EQ(-ENOENT, completion->get_return_value());
5688 completion->release();
5689 }
5690
5691}
5692
5693TEST_F(LibRadosTwoPoolsPP, ManifestSnapHasChunk) {
5694 // skip test if not yet octopus
5695 if (_get_required_osd_release(cluster) < "octopus") {
5696 cout << "cluster is not yet octopus, skipping test" << std::endl;
5697 return;
5698 }
5699
5700 bufferlist inbl;
5701 ASSERT_EQ(0, cluster.mon_command(
5702 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
5703 inbl, NULL, NULL));
5704 cluster.wait_for_latest_osdmap();
5705
5706 // create object
9f95a23c
TL		 5707 {
5708 bufferlist bl;
f67539c2 5709 bl.append("there HIHI");
9f95a23c 5710 ObjectWriteOperation op;
f67539c2
TL		 5711 op.write_full(bl);
5712 ASSERT_EQ(0, ioctx.operate("foo", &op));
9f95a23c 5713 }
f67539c2
TL		 5714
5715 string er_fp_oid, hi_fp_oid, HI_fp_oid, ai_fp_oid, bi_fp_oid,
5716 Er_fp_oid, Hi_fp_oid, SI_fp_oid;
5717
5718 // get fp_oid
5719 er_fp_oid = get_fp_oid("er", "sha1");
5720 hi_fp_oid = get_fp_oid("hi", "sha1");
5721 HI_fp_oid = get_fp_oid("HI", "sha1");
5722 ai_fp_oid = get_fp_oid("ai", "sha1");
5723 bi_fp_oid = get_fp_oid("bi", "sha1");
5724 Er_fp_oid = get_fp_oid("Er", "sha1");
5725 Hi_fp_oid = get_fp_oid("Hi", "sha1");
5726 SI_fp_oid = get_fp_oid("SI", "sha1");
5727
5728 // write
9f95a23c
TL		 5729 {
5730 ObjectWriteOperation op;
f67539c2
TL		 5731 bufferlist bl;
5732 bl.append("er");
5733 op.write_full(bl);
5734 ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
9f95a23c 5735 }
f67539c2
TL		 5736 // write
5737 {
5738 ObjectWriteOperation op;
5739 bufferlist bl;
5740 bl.append("hi");
5741 op.write_full(bl);
5742 ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
5743 }
5744 // write
5745 {
5746 ObjectWriteOperation op;
5747 bufferlist bl;
5748 bl.append("HI");
5749 op.write_full(bl);
5750 ASSERT_EQ(0, cache_ioctx.operate(HI_fp_oid, &op));
5751 }
5752 // write
5753 {
5754 ObjectWriteOperation op;
5755 bufferlist bl;
5756 bl.append("ai");
5757 op.write_full(bl);
5758 ASSERT_EQ(0, cache_ioctx.operate(ai_fp_oid, &op));
5759 }
5760 // write
5761 {
5762 ObjectWriteOperation op;
5763 bufferlist bl;
5764 bl.append("bi");
5765 op.write_full(bl);
5766 ASSERT_EQ(0, cache_ioctx.operate(bi_fp_oid, &op));
5767 }
5768 // write
5769 {
5770 ObjectWriteOperation op;
5771 bufferlist bl;
5772 bl.append("Er");
5773 op.write_full(bl);
5774 ASSERT_EQ(0, cache_ioctx.operate(Er_fp_oid, &op));
5775 }
5776 // write
9f95a23c 5777 {
f67539c2
TL		 5778 ObjectWriteOperation op;
5779 bufferlist bl;
5780 bl.append("Hi");
5781 op.write_full(bl);
5782 ASSERT_EQ(0, cache_ioctx.operate(Hi_fp_oid, &op));
5783 }
5784 // write
5785 {
5786 ObjectWriteOperation op;
9f95a23c 5787 bufferlist bl;
f67539c2
TL		 5788 bl.append("SI");
5789 op.write_full(bl);
5790 ASSERT_EQ(0, cache_ioctx.operate(SI_fp_oid, &op));
9f95a23c
TL		 5791 }
5792
f67539c2
TL		 5793 // set-chunk (dedup)
5794 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, HI_fp_oid, "foo");
5795 // set-chunk (dedup)
5796 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, HI_fp_oid, "foo");
9f95a23c 5797
f67539c2
TL		 5798 // foo head: [hi] [HI]
5799
5800 // create a snapshot, clone
5801 vector<uint64_t> my_snaps(1);
5802 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5803 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5804 my_snaps));
5805
5806
5807 // create a clone
5808 {
5809 bufferlist bl;
5810 bl.append("a");
5811 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
5812 }
5813 // write
5814 {
5815 bufferlist bl;
5816 bl.append("a");
5817 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
5818 }
5819 // write
5820 {
5821 bufferlist bl;
5822 bl.append("S");
5823 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 8));
5824 }
5825
5826 // foo snap[0]: [hi] [HI]
5827 // foo head : [er] [ai] [SI]
5828
5829 // set-chunk (dedup)
5830 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
5831 // set-chunk (dedup)
5832 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ai_fp_oid, "foo");
5833 // set-chunk (dedup)
5834 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, SI_fp_oid, "foo");
5835
5836 my_snaps.resize(2);
5837 my_snaps[1] = my_snaps[0];
5838 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5839 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5840 my_snaps));
5841
5842 // create a clone
5843 {
5844 bufferlist bl;
5845 bl.append("b");
5846 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
5847 }
5848 // write
5849 {
5850 bufferlist bl;
5851 bl.append("b");
5852 ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
5853 }
5854
5855 // foo snap[1]: [HI] [HI]
5856 // foo snap[0]: [er] [ai] [SI]
5857 // foo head : [er] [bi] [SI]
5858
5859 // set-chunk (dedup)
5860 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, bi_fp_oid, "foo");
5861
5862 {
5863 ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", SI_fp_oid));
5864 ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", er_fp_oid));
5865 ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", ai_fp_oid));
5866 ASSERT_EQ(2, cls_cas_references_chunk(ioctx, "foo", HI_fp_oid));
5867 ASSERT_EQ(-ENOLINK, cls_cas_references_chunk(ioctx, "foo", Hi_fp_oid));
5868 }
9f95a23c
TL		 5869}
5870
20effc67
TL		 5871TEST_F(LibRadosTwoPoolsPP, ManifestRollback) {
5872 // skip test if not yet pacific
5873 if (_get_required_osd_release(cluster) < "pacific") {
5874 cout << "cluster is not yet pacific, skipping test" << std::endl;
5875 return;
5876 }
5877
5878 // create object
5879 {
5880 bufferlist bl;
5881 bl.append("CDere hiHI");
5882 ObjectWriteOperation op;
5883 op.write_full(bl);
5884 ASSERT_EQ(0, ioctx.operate("foo", &op));
5885 }
5886 {
5887 bufferlist bl;
5888 bl.append("ABere hiHI");
5889 ObjectWriteOperation op;
5890 op.write_full(bl);
5891 ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
5892 }
5893 {
5894 bufferlist bl;
5895 bl.append("CDere hiHI");
5896 ObjectWriteOperation op;
5897 op.write_full(bl);
5898 ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
5899 }
5900 {
5901 bufferlist bl;
5902 bl.append("EFere hiHI");
5903 ObjectWriteOperation op;
5904 op.write_full(bl);
5905 ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
5906 }
5907
5908 // wait for maps to settle
5909 cluster.wait_for_latest_osdmap();
5910
5911 // create a snapshot, clone
5912 vector<uint64_t> my_snaps(1);
5913 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5914 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5915 my_snaps));
5916
5917 {
5918 bufferlist bl;
5919 bl.append("there hiHI");
5920 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
5921 }
5922
5923 my_snaps.resize(2);
5924 my_snaps[1] = my_snaps[0];
5925 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5926 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5927 my_snaps));
5928
5929 {
5930 bufferlist bl;
5931 bl.append("thABe hiEF");
5932 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
5933 }
5934
5935 // set-chunk
5936 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
5937 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk3", "foo");
5938 // foo snap[1]:
5939 // foo snap[0]:
5940 // foo head : [chunk1] [chunk3]
5941
5942 ioctx.snap_set_read(my_snaps[1]);
5943 // set-chunk
5944 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
5945 // foo snap[1]: [ chunk2 ]
5946 // foo snap[0]:
5947 // foo head : [chunk1] [chunk3]
5948
5949 // foo snap[1]: [ chunk2 ]
5950 // foo snap[0]:
5951 // foo head : [chunk1] [chunk3]
5952
5953 ASSERT_EQ(0, ioctx.selfmanaged_snap_rollback("foo", my_snaps[0]));
5954
5955 ioctx.snap_set_read(librados::SNAP_HEAD);
5956 {
5957 bufferlist bl;
5958 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5959 ASSERT_EQ('t', bl[0]);
5960 }
5961
5962 ASSERT_EQ(0, ioctx.selfmanaged_snap_rollback("foo", my_snaps[1]));
5963
5964 {
5965 bufferlist bl;
5966 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5967 ASSERT_EQ('C', bl[0]);
5968 }
5969
5970}
5971
5972TEST_F(LibRadosTwoPoolsPP, ManifestRollbackRefcount) {
5973 // skip test if not yet pacific
5974 if (_get_required_osd_release(cluster) < "pacific") {
5975 cout << "cluster is not yet pacific, skipping test" << std::endl;
5976 return;
5977 }
5978
5979 // create object
5980 {
5981 bufferlist bl;
5982 bl.append("CDere hiHI");
5983 ObjectWriteOperation op;
5984 op.write_full(bl);
5985 ASSERT_EQ(0, ioctx.operate("foo", &op));
5986 }
5987 {
5988 bufferlist bl;
5989 bl.append("ABere hiHI");
5990 ObjectWriteOperation op;
5991 op.write_full(bl);
5992 ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
5993 }
5994 {
5995 bufferlist bl;
5996 bl.append("CDere hiHI");
5997 ObjectWriteOperation op;
5998 op.write_full(bl);
5999 ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
6000 }
6001 {
6002 bufferlist bl;
6003 bl.append("EFere hiHI");
6004 ObjectWriteOperation op;
6005 op.write_full(bl);
6006 ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
6007 }
6008 {
6009 bufferlist bl;
6010 bl.append("DDDDD hiHI");
6011 ObjectWriteOperation op;
6012 op.write_full(bl);
6013 ASSERT_EQ(0, cache_ioctx.operate("chunk4", &op));
6014 }
6015 {
6016 bufferlist bl;
6017 bl.append("EEEEE hiHI");
6018 ObjectWriteOperation op;
6019 op.write_full(bl);
6020 ASSERT_EQ(0, cache_ioctx.operate("chunk5", &op));
6021 }
6022
6023 // wait for maps to settle
6024 cluster.wait_for_latest_osdmap();
6025
6026 // create a snapshot, clone
6027 vector<uint64_t> my_snaps(1);
6028 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
6029 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
6030 my_snaps));
6031
6032 {
6033 bufferlist bl;
6034 bl.append("there hiHI");
6035 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
6036 }
6037
6038 my_snaps.resize(2);
6039 my_snaps[1] = my_snaps[0];
6040 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
6041 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
6042 my_snaps));
6043
6044 {
6045 bufferlist bl;
6046 bl.append("thABe hiEF");
6047 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
6048 }
6049
6050 // set-chunk
6051 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
6052 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk3", "foo");
6053 // foo snap[1]:
6054 // foo snap[0]:
6055 // foo head : [chunk1] [chunk3]
6056
6057 ioctx.snap_set_read(my_snaps[1]);
6058 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk4", "foo");
6059 manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk5", "foo");
6060 // foo snap[1]: [chunk4] [chunk5]
6061 // foo snap[0]:
6062 // foo head : [chunk1] [chunk3]
6063
6064 ioctx.snap_set_read(my_snaps[0]);
6065 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
6066 // foo snap[1]: [chunk4] [chunk5]
6067 // foo snap[0]: [ chunk2 ]
6068 // foo head : [chunk1] [chunk3]
6069
6070 ASSERT_EQ(0, ioctx.selfmanaged_snap_rollback("foo", my_snaps[1]));
6071 // foo snap[1]: [chunk4] [chunk5]
6072 // foo snap[0]: [ chunk2 ]
6073 // foo head : [chunk4] [chunk5] <-- will contain these contents
6074
6075 sleep(10);
6076 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk1", 0);
6077 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk3", 0);
6078
6079 ioctx.selfmanaged_snap_remove(my_snaps[1]);
6080 sleep(10);
6081 // foo snap[1]:
6082 // foo snap[0]: [ chunk2 ]
6083 // foo head : [chunk4] [chunk5]
6084 ioctx.snap_set_read(librados::SNAP_HEAD);
6085 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk4", 1);
6086 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk5", 1);
6087
6088 {
6089 bufferlist bl;
6090 bl.append("thABe hiEF");
6091 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
6092 }
6093 // foo snap[1]:
6094 // foo snap[0]: [ chunk2 ]
6095 // foo head :
6096 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk1", 0);
6097 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk3", 0);
6098 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk4", 0);
6099 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk5", 0);
6100 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk2", 1);
6101}
6102
6103TEST_F(LibRadosTwoPoolsPP, ManifestEvictRollback) {
6104 // skip test if not yet pacific
6105 if (_get_required_osd_release(cluster) < "pacific") {
6106 cout << "cluster is not yet pacific, skipping test" << std::endl;
6107 return;
6108 }
6109
6110 // create object
6111 {
6112 bufferlist bl;
6113 bl.append("CDere hiHI");
6114 ObjectWriteOperation op;
6115 op.write_full(bl);
6116 ASSERT_EQ(0, ioctx.operate("foo", &op));
6117 }
6118 {
6119 bufferlist bl;
6120 bl.append("ABere hiHI");
6121 ObjectWriteOperation op;
6122 op.write_full(bl);
6123 ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
6124 }
6125 {
6126 bufferlist bl;
6127 bl.append("CDere hiHI");
6128 ObjectWriteOperation op;
6129 op.write_full(bl);
6130 ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
6131 }
6132 {
6133 bufferlist bl;
6134 bl.append("EFere hiHI");
6135 ObjectWriteOperation op;
6136 op.write_full(bl);
6137 ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
6138 }
6139
6140 // wait for maps to settle
6141 cluster.wait_for_latest_osdmap();
6142
6143 // create a snapshot, clone
6144 vector<uint64_t> my_snaps(1);
6145 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
6146 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
6147 my_snaps));
6148
6149 {
6150 bufferlist bl;
6151 bl.append("there hiHI");
6152 ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
6153 }
6154
6155
6156 // set-chunk
6157 manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
6158 manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk3", "foo");
6159 // foo snap[0]:
6160 // foo head : [chunk1] [chunk3]
6161
6162 ioctx.snap_set_read(my_snaps[0]);
6163 manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
6164 // foo snap[0]: [ chunk2 ]
6165 // foo head : [chunk1] [chunk3]
6166
6167 sleep(10);
6168 ioctx.snap_set_read(librados::SNAP_HEAD);
6169 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk1", 1);
6170 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk3", 1);
6171
6172
6173 ioctx.snap_set_read(my_snaps[0]);
6174 // evict--this makes the chunk missing state
6175 {
6176 ObjectReadOperation op, stat_op;
6177 op.tier_evict();
6178 librados::AioCompletion *completion = cluster.aio_create_completion();
6179 ASSERT_EQ(0, ioctx.aio_operate(
6180 "foo", completion, &op,
6181 librados::OPERATION_IGNORE_OVERLAY, NULL));
6182 completion->wait_for_complete();
6183 ASSERT_EQ(0, completion->get_return_value());
6184 }
6185
6186 // rollback to my_snaps[0]
6187 ASSERT_EQ(0, ioctx.selfmanaged_snap_rollback("foo", my_snaps[0]));
6188
6189 ioctx.snap_set_read(librados::SNAP_HEAD);
6190 {
6191 bufferlist bl;
6192 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6193 ASSERT_EQ('C', bl[0]);
6194 }
6195
6196 is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk2", 1);
6197}
6198
7c673cae
FG		 6199class LibRadosTwoPoolsECPP : public RadosTestECPP
6200{
6201public:
6202 LibRadosTwoPoolsECPP() {};
6203 ~LibRadosTwoPoolsECPP() override {};
6204protected:
6205 static void SetUpTestCase() {
6206 pool_name = get_temp_pool_name();
6207 ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
6208 }
6209 static void TearDownTestCase() {
6210 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
6211 }
6212 static std::string cache_pool_name;
6213
6214 void SetUp() override {
6215 cache_pool_name = get_temp_pool_name();
6216 ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
6217 RadosTestECPP::SetUp();
c07f9fc5 6218
7c673cae 6219 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 6220 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 6221 cache_ioctx.set_namespace(nspace);
6222 }
6223 void TearDown() override {
6224 // flush + evict cache
6225 flush_evict_all(cluster, cache_ioctx);
6226
6227 bufferlist inbl;
6228 // tear down tiers
6229 ASSERT_EQ(0, cluster.mon_command(
6230 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
6231 "\"}",
6232 inbl, NULL, NULL));
6233 ASSERT_EQ(0, cluster.mon_command(
6234 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
6235 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
6236 inbl, NULL, NULL));
6237
6238 // wait for maps to settle before next test
6239 cluster.wait_for_latest_osdmap();
6240
6241 RadosTestECPP::TearDown();
6242
6243 cleanup_default_namespace(cache_ioctx);
6244 cleanup_namespace(cache_ioctx, nspace);
6245
6246 cache_ioctx.close();
6247 ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
6248 }
6249
6250 librados::IoCtx cache_ioctx;
6251};
6252
6253std::string LibRadosTwoPoolsECPP::cache_pool_name;
6254
6255TEST_F(LibRadosTierECPP, Dirty) {
6256 {
6257 ObjectWriteOperation op;
6258 op.undirty();
6259 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
6260 }
6261 {
6262 ObjectWriteOperation op;
6263 op.create(true);
6264 ASSERT_EQ(0, ioctx.operate("foo", &op));
6265 }
6266 {
6267 bool dirty = false;
6268 int r = -1;
6269 ObjectReadOperation op;
6270 op.is_dirty(&dirty, &r);
6271 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
6272 ASSERT_TRUE(dirty);
6273 ASSERT_EQ(0, r);
6274 }
6275 {
6276 ObjectWriteOperation op;
6277 op.undirty();
6278 ASSERT_EQ(0, ioctx.operate("foo", &op));
6279 }
6280 {
6281 ObjectWriteOperation op;
6282 op.undirty();
6283 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
6284 }
6285 {
6286 bool dirty = false;
6287 int r = -1;
6288 ObjectReadOperation op;
6289 op.is_dirty(&dirty, &r);
6290 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
6291 ASSERT_FALSE(dirty);
6292 ASSERT_EQ(0, r);
6293 }
6294 //{
6295 // ObjectWriteOperation op;
6296 // op.truncate(0); // still a write even tho it is a no-op
6297 // ASSERT_EQ(0, ioctx.operate("foo", &op));
6298 //}
6299 //{
6300 // bool dirty = false;
6301 // int r = -1;
6302 // ObjectReadOperation op;
6303 // op.is_dirty(&dirty, &r);
6304 // ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
6305 // ASSERT_TRUE(dirty);
6306 // ASSERT_EQ(0, r);
6307 //}
6308}
6309
6310TEST_F(LibRadosTwoPoolsECPP, Overlay) {
6311 // create objects
6312 {
6313 bufferlist bl;
6314 bl.append("base");
6315 ObjectWriteOperation op;
6316 op.write_full(bl);
6317 ASSERT_EQ(0, ioctx.operate("foo", &op));
6318 }
6319 {
6320 bufferlist bl;
6321 bl.append("cache");
6322 ObjectWriteOperation op;
6323 op.write_full(bl);
6324 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
6325 }
6326
6327 // configure cache
6328 bufferlist inbl;
6329 ASSERT_EQ(0, cluster.mon_command(
6330 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6331 "\", \"tierpool\": \"" + cache_pool_name +
6332 "\", \"force_nonempty\": \"--force-nonempty\" }",
6333 inbl, NULL, NULL));
6334 ASSERT_EQ(0, cluster.mon_command(
6335 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6336 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6337 inbl, NULL, NULL));
6338
6339 // wait for maps to settle
6340 cluster.wait_for_latest_osdmap();
6341
6342 // by default, the overlay sends us to cache pool
6343 {
6344 bufferlist bl;
6345 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6346 ASSERT_EQ('c', bl[0]);
6347 }
6348 {
6349 bufferlist bl;
6350 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
6351 ASSERT_EQ('c', bl[0]);
6352 }
6353
6354 // unless we say otherwise
6355 {
6356 bufferlist bl;
6357 ObjectReadOperation op;
6358 op.read(0, 1, &bl, NULL);
6359 librados::AioCompletion *completion = cluster.aio_create_completion();
6360 ASSERT_EQ(0, ioctx.aio_operate(
6361 "foo", completion, &op,
6362 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 6363 completion->wait_for_complete();
7c673cae
FG		 6364 ASSERT_EQ(0, completion->get_return_value());
6365 completion->release();
6366 ASSERT_EQ('b', bl[0]);
6367 }
6368}
6369
6370TEST_F(LibRadosTwoPoolsECPP, Promote) {
6371 // create object
6372 {
6373 bufferlist bl;
6374 bl.append("hi there");
6375 ObjectWriteOperation op;
6376 op.write_full(bl);
6377 ASSERT_EQ(0, ioctx.operate("foo", &op));
6378 }
6379
6380 // configure cache
6381 bufferlist inbl;
6382 ASSERT_EQ(0, cluster.mon_command(
6383 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6384 "\", \"tierpool\": \"" + cache_pool_name +
6385 "\", \"force_nonempty\": \"--force-nonempty\" }",
6386 inbl, NULL, NULL));
6387 ASSERT_EQ(0, cluster.mon_command(
6388 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6389 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6390 inbl, NULL, NULL));
6391 ASSERT_EQ(0, cluster.mon_command(
6392 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6393 "\", \"mode\": \"writeback\"}",
6394 inbl, NULL, NULL));
6395
6396 // wait for maps to settle
6397 cluster.wait_for_latest_osdmap();
6398
6399 // read, trigger a promote
6400 {
6401 bufferlist bl;
6402 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6403 }
6404
6405 // read, trigger a whiteout
6406 {
6407 bufferlist bl;
6408 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
6409 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
6410 }
6411
6412 // verify the object is present in the cache tier
6413 {
6414 NObjectIterator it = cache_ioctx.nobjects_begin();
6415 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6416 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6417 ++it;
6418 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6419 ++it;
6420 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6421 }
6422}
6423
6424TEST_F(LibRadosTwoPoolsECPP, PromoteSnap) {
6425 // create object
6426 {
6427 bufferlist bl;
6428 bl.append("hi there");
6429 ObjectWriteOperation op;
6430 op.write_full(bl);
6431 ASSERT_EQ(0, ioctx.operate("foo", &op));
6432 }
6433 {
6434 bufferlist bl;
6435 bl.append("hi there");
6436 ObjectWriteOperation op;
6437 op.write_full(bl);
6438 ASSERT_EQ(0, ioctx.operate("bar", &op));
6439 }
6440 {
6441 bufferlist bl;
6442 bl.append("hi there");
6443 ObjectWriteOperation op;
6444 op.write_full(bl);
6445 ASSERT_EQ(0, ioctx.operate("baz", &op));
6446 }
6447 {
6448 bufferlist bl;
6449 bl.append("hi there");
6450 ObjectWriteOperation op;
6451 op.write_full(bl);
6452 ASSERT_EQ(0, ioctx.operate("bam", &op));
6453 }
6454
6455 // create a snapshot, clone
6456 vector<uint64_t> my_snaps(1);
6457 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
6458 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
6459 my_snaps));
6460 {
6461 bufferlist bl;
6462 bl.append("ciao!");
6463 ObjectWriteOperation op;
6464 op.write_full(bl);
6465 ASSERT_EQ(0, ioctx.operate("foo", &op));
6466 }
6467 {
6468 bufferlist bl;
6469 bl.append("ciao!");
6470 ObjectWriteOperation op;
6471 op.write_full(bl);
6472 ASSERT_EQ(0, ioctx.operate("bar", &op));
6473 }
6474 {
6475 ObjectWriteOperation op;
6476 op.remove();
6477 ASSERT_EQ(0, ioctx.operate("baz", &op));
6478 }
6479 {
6480 bufferlist bl;
6481 bl.append("ciao!");
6482 ObjectWriteOperation op;
6483 op.write_full(bl);
6484 ASSERT_EQ(0, ioctx.operate("bam", &op));
6485 }
6486
6487 // configure cache
6488 bufferlist inbl;
6489 ASSERT_EQ(0, cluster.mon_command(
6490 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6491 "\", \"tierpool\": \"" + cache_pool_name +
6492 "\", \"force_nonempty\": \"--force-nonempty\" }",
6493 inbl, NULL, NULL));
6494 ASSERT_EQ(0, cluster.mon_command(
6495 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6496 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6497 inbl, NULL, NULL));
6498 ASSERT_EQ(0, cluster.mon_command(
6499 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6500 "\", \"mode\": \"writeback\"}",
6501 inbl, NULL, NULL));
6502
6503 // wait for maps to settle
6504 cluster.wait_for_latest_osdmap();
6505
6506 // read, trigger a promote on the head
6507 {
6508 bufferlist bl;
6509 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6510 ASSERT_EQ('c', bl[0]);
6511 }
6512 {
6513 bufferlist bl;
6514 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
6515 ASSERT_EQ('c', bl[0]);
6516 }
6517
6518 ioctx.snap_set_read(my_snaps[0]);
6519
6520 // stop and scrub this pg (to make sure scrub can handle missing
6521 // clones in the cache tier)
6522 // This test requires cache tier and base tier to have the same pg_num/pgp_num
6523 {
6524 for (int tries = 0; tries < 5; ++tries) {
6525 IoCtx cache_ioctx;
6526 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
6527 uint32_t hash;
6528 ASSERT_EQ(0, ioctx.get_object_pg_hash_position2("foo", &hash));
6529 ostringstream ss;
6530 ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
6531 << cache_ioctx.get_id() << "."
6532 << hash
6533 << "\"}";
6534 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
6535 if (r == -EAGAIN ||
6536 r == -ENOENT) { // in case mgr osdmap is a bit stale
6537 sleep(5);
6538 continue;
6539 }
6540 ASSERT_EQ(0, r);
6541 break;
6542 }
6543 // give it a few seconds to go. this is sloppy but is usually enough time
6544 cout << "waiting for scrub..." << std::endl;
6545 sleep(15);
6546 cout << "done waiting" << std::endl;
6547 }
6548
6549 // read foo snap
6550 {
6551 bufferlist bl;
6552 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6553 ASSERT_EQ('h', bl[0]);
6554 }
6555
6556 // read bar snap
6557 {
6558 bufferlist bl;
6559 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
6560 ASSERT_EQ('h', bl[0]);
6561 }
6562
6563 // read baz snap
6564 {
6565 bufferlist bl;
6566 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
6567 ASSERT_EQ('h', bl[0]);
6568 }
6569
6570 ioctx.snap_set_read(librados::SNAP_HEAD);
6571
6572 // read foo
6573 {
6574 bufferlist bl;
6575 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6576 ASSERT_EQ('c', bl[0]);
6577 }
6578
6579 // read bar
6580 {
6581 bufferlist bl;
6582 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
6583 ASSERT_EQ('c', bl[0]);
6584 }
6585
6586 // read baz
6587 {
6588 bufferlist bl;
6589 ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
6590 }
6591
6592 // cleanup
6593 ioctx.selfmanaged_snap_remove(my_snaps[0]);
6594}
6595
6596TEST_F(LibRadosTwoPoolsECPP, PromoteSnapTrimRace) {
6597 // create object
6598 {
6599 bufferlist bl;
6600 bl.append("hi there");
6601 ObjectWriteOperation op;
6602 op.write_full(bl);
6603 ASSERT_EQ(0, ioctx.operate("foo", &op));
6604 }
6605
6606 // create a snapshot, clone
6607 vector<uint64_t> my_snaps(1);
6608 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
6609 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
6610 my_snaps));
6611 {
6612 bufferlist bl;
6613 bl.append("ciao!");
6614 ObjectWriteOperation op;
6615 op.write_full(bl);
6616 ASSERT_EQ(0, ioctx.operate("foo", &op));
6617 }
6618
6619 // configure cache
6620 bufferlist inbl;
6621 ASSERT_EQ(0, cluster.mon_command(
6622 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6623 "\", \"tierpool\": \"" + cache_pool_name +
6624 "\", \"force_nonempty\": \"--force-nonempty\" }",
6625 inbl, NULL, NULL));
6626 ASSERT_EQ(0, cluster.mon_command(
6627 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6628 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6629 inbl, NULL, NULL));
6630 ASSERT_EQ(0, cluster.mon_command(
6631 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6632 "\", \"mode\": \"writeback\"}",
6633 inbl, NULL, NULL));
6634
6635 // wait for maps to settle
6636 cluster.wait_for_latest_osdmap();
6637
6638 // delete the snap
6639 ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
6640
6641 ioctx.snap_set_read(my_snaps[0]);
6642
9f95a23c
TL		 6643 // read foo snap. the OSD may or may not realize that this snap has
6644 // been logically deleted; either response is valid.
7c673cae
FG		 6645 {
6646 bufferlist bl;
9f95a23c
TL		 6647 int r = ioctx.read("foo", bl, 1, 0);
6648 ASSERT_TRUE(r == 1 || r == -ENOENT);
7c673cae
FG		 6649 }
6650
6651 // cleanup
6652 ioctx.selfmanaged_snap_remove(my_snaps[0]);
6653}
6654
6655TEST_F(LibRadosTwoPoolsECPP, Whiteout) {
6656 // create object
6657 {
6658 bufferlist bl;
6659 bl.append("hi there");
6660 ObjectWriteOperation op;
6661 op.write_full(bl);
6662 ASSERT_EQ(0, ioctx.operate("foo", &op));
6663 }
6664
6665 // configure cache
6666 bufferlist inbl;
6667 ASSERT_EQ(0, cluster.mon_command(
6668 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6669 "\", \"tierpool\": \"" + cache_pool_name +
6670 "\", \"force_nonempty\": \"--force-nonempty\" }",
6671 inbl, NULL, NULL));
6672 ASSERT_EQ(0, cluster.mon_command(
6673 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6674 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6675 inbl, NULL, NULL));
6676 ASSERT_EQ(0, cluster.mon_command(
6677 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6678 "\", \"mode\": \"writeback\"}",
6679 inbl, NULL, NULL));
6680
6681 // wait for maps to settle
6682 cluster.wait_for_latest_osdmap();
6683
6684 // create some whiteouts, verify they behave
6685 {
6686 ObjectWriteOperation op;
6687 op.assert_exists();
6688 op.remove();
6689 ASSERT_EQ(0, ioctx.operate("foo", &op));
6690 }
6691
6692 {
6693 ObjectWriteOperation op;
6694 op.assert_exists();
6695 op.remove();
6696 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
6697 }
6698 {
6699 ObjectWriteOperation op;
6700 op.assert_exists();
6701 op.remove();
6702 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
6703 }
6704
6705 // verify the whiteouts are there in the cache tier
6706 {
6707 NObjectIterator it = cache_ioctx.nobjects_begin();
6708 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6709 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6710 ++it;
6711 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6712 ++it;
6713 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6714 }
6715
6716 // delete a whiteout and verify it goes away
6717 ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
6718 {
6719 ObjectWriteOperation op;
6720 op.remove();
6721 librados::AioCompletion *completion = cluster.aio_create_completion();
6722 ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
6723 librados::OPERATION_IGNORE_CACHE));
9f95a23c 6724 completion->wait_for_complete();
7c673cae
FG		 6725 ASSERT_EQ(0, completion->get_return_value());
6726 completion->release();
6727
6728 NObjectIterator it = cache_ioctx.nobjects_begin();
6729 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6730 ASSERT_TRUE(it->get_oid() == string("foo"));
6731 ++it;
6732 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6733 }
6734
6735 // recreate an object and verify we can read it
6736 {
6737 bufferlist bl;
6738 bl.append("hi there");
6739 ObjectWriteOperation op;
6740 op.write_full(bl);
6741 ASSERT_EQ(0, ioctx.operate("foo", &op));
6742 }
6743 {
6744 bufferlist bl;
6745 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6746 ASSERT_EQ('h', bl[0]);
6747 }
6748}
6749
6750TEST_F(LibRadosTwoPoolsECPP, Evict) {
6751 // create object
6752 {
6753 bufferlist bl;
6754 bl.append("hi there");
6755 ObjectWriteOperation op;
6756 op.write_full(bl);
6757 ASSERT_EQ(0, ioctx.operate("foo", &op));
6758 }
6759
6760 // configure cache
6761 bufferlist inbl;
6762 ASSERT_EQ(0, cluster.mon_command(
6763 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6764 "\", \"tierpool\": \"" + cache_pool_name +
6765 "\", \"force_nonempty\": \"--force-nonempty\" }",
6766 inbl, NULL, NULL));
6767 ASSERT_EQ(0, cluster.mon_command(
6768 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6769 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6770 inbl, NULL, NULL));
6771 ASSERT_EQ(0, cluster.mon_command(
6772 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6773 "\", \"mode\": \"writeback\"}",
6774 inbl, NULL, NULL));
6775
6776 // wait for maps to settle
6777 cluster.wait_for_latest_osdmap();
6778
6779 // read, trigger a promote
6780 {
6781 bufferlist bl;
6782 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6783 }
6784
6785 // read, trigger a whiteout, and a dirty object
6786 {
6787 bufferlist bl;
6788 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
6789 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
6790 ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
6791 }
6792
6793 // verify the object is present in the cache tier
6794 {
6795 NObjectIterator it = cache_ioctx.nobjects_begin();
6796 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6797 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6798 ++it;
6799 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
6800 ++it;
6801 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6802 }
6803
6804 // pin
6805 {
6806 ObjectWriteOperation op;
6807 op.cache_pin();
6808 librados::AioCompletion *completion = cluster.aio_create_completion();
6809 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6810 completion->wait_for_complete();
7c673cae
FG		 6811 ASSERT_EQ(0, completion->get_return_value());
6812 completion->release();
6813 }
6814
6815 // evict the pinned object with -EPERM
6816 {
6817 ObjectReadOperation op;
6818 op.cache_evict();
6819 librados::AioCompletion *completion = cluster.aio_create_completion();
6820 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
6821 librados::OPERATION_IGNORE_CACHE,
6822 NULL));
9f95a23c 6823 completion->wait_for_complete();
7c673cae
FG		 6824 ASSERT_EQ(-EPERM, completion->get_return_value());
6825 completion->release();
6826 }
6827
6828 // unpin
6829 {
6830 ObjectWriteOperation op;
6831 op.cache_unpin();
6832 librados::AioCompletion *completion = cluster.aio_create_completion();
6833 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6834 completion->wait_for_complete();
7c673cae
FG		 6835 ASSERT_EQ(0, completion->get_return_value());
6836 completion->release();
6837 }
6838
6839 // flush
6840 {
6841 ObjectReadOperation op;
6842 op.cache_flush();
6843 librados::AioCompletion *completion = cluster.aio_create_completion();
6844 ASSERT_EQ(0, cache_ioctx.aio_operate(
6845 "foo", completion, &op,
6846 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 6847 completion->wait_for_complete();
7c673cae
FG		 6848 ASSERT_EQ(0, completion->get_return_value());
6849 completion->release();
6850 }
6851
6852 // verify clean
6853 {
6854 bool dirty = false;
6855 int r = -1;
6856 ObjectReadOperation op;
6857 op.is_dirty(&dirty, &r);
6858 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
6859 ASSERT_FALSE(dirty);
6860 ASSERT_EQ(0, r);
6861 }
6862
6863 // evict
6864 {
6865 ObjectReadOperation op;
6866 op.cache_evict();
6867 librados::AioCompletion *completion = cluster.aio_create_completion();
6868 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
6869 librados::OPERATION_IGNORE_CACHE,
6870 NULL));
9f95a23c 6871 completion->wait_for_complete();
7c673cae
FG		 6872 ASSERT_EQ(0, completion->get_return_value());
6873 completion->release();
6874 }
6875 {
6876 ObjectReadOperation op;
6877 op.cache_evict();
6878 librados::AioCompletion *completion = cluster.aio_create_completion();
6879 ASSERT_EQ(0, cache_ioctx.aio_operate(
6880 "foo", completion, &op,
6881 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6882 completion->wait_for_complete();
7c673cae
FG		 6883 ASSERT_EQ(0, completion->get_return_value());
6884 completion->release();
6885 }
6886 {
6887 ObjectReadOperation op;
6888 op.cache_evict();
6889 librados::AioCompletion *completion = cluster.aio_create_completion();
6890 ASSERT_EQ(0, cache_ioctx.aio_operate(
6891 "bar", completion, &op,
6892 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 6893 completion->wait_for_complete();
7c673cae
FG		 6894 ASSERT_EQ(-EBUSY, completion->get_return_value());
6895 completion->release();
6896 }
6897}
6898
6899TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
6900 // create object
6901 {
6902 bufferlist bl;
6903 bl.append("hi there");
6904 ObjectWriteOperation op;
6905 op.write_full(bl);
6906 ASSERT_EQ(0, ioctx.operate("foo", &op));
6907 }
6908 {
6909 bufferlist bl;
6910 bl.append("hi there");
6911 ObjectWriteOperation op;
6912 op.write_full(bl);
6913 ASSERT_EQ(0, ioctx.operate("bar", &op));
6914 }
6915 {
6916 bufferlist bl;
6917 bl.append("hi there");
6918 ObjectWriteOperation op;
6919 op.write_full(bl);
6920 ASSERT_EQ(0, ioctx.operate("baz", &op));
6921 }
6922 {
6923 bufferlist bl;
6924 bl.append("hi there");
6925 ObjectWriteOperation op;
6926 op.write_full(bl);
6927 ASSERT_EQ(0, ioctx.operate("bam", &op));
6928 }
6929
6930 // create a snapshot, clone
6931 vector<uint64_t> my_snaps(1);
6932 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
6933 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
6934 my_snaps));
6935 {
6936 bufferlist bl;
6937 bl.append("ciao!");
6938 ObjectWriteOperation op;
6939 op.write_full(bl);
6940 ASSERT_EQ(0, ioctx.operate("foo", &op));
6941 }
6942 {
6943 bufferlist bl;
6944 bl.append("ciao!");
6945 ObjectWriteOperation op;
6946 op.write_full(bl);
6947 ASSERT_EQ(0, ioctx.operate("bar", &op));
6948 }
6949 {
6950 ObjectWriteOperation op;
6951 op.remove();
6952 ASSERT_EQ(0, ioctx.operate("baz", &op));
6953 }
6954 {
6955 bufferlist bl;
6956 bl.append("ciao!");
6957 ObjectWriteOperation op;
6958 op.write_full(bl);
6959 ASSERT_EQ(0, ioctx.operate("bam", &op));
6960 }
6961
6962 // configure cache
6963 bufferlist inbl;
6964 ASSERT_EQ(0, cluster.mon_command(
6965 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6966 "\", \"tierpool\": \"" + cache_pool_name +
6967 "\", \"force_nonempty\": \"--force-nonempty\" }",
6968 inbl, NULL, NULL));
6969 ASSERT_EQ(0, cluster.mon_command(
6970 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6971 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6972 inbl, NULL, NULL));
6973 ASSERT_EQ(0, cluster.mon_command(
6974 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6975 "\", \"mode\": \"writeback\"}",
6976 inbl, NULL, NULL));
6977
6978 // wait for maps to settle
6979 cluster.wait_for_latest_osdmap();
6980
6981 // read, trigger a promote on the head
6982 {
6983 bufferlist bl;
6984 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6985 ASSERT_EQ('c', bl[0]);
6986 }
6987 {
6988 bufferlist bl;
6989 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
6990 ASSERT_EQ('c', bl[0]);
6991 }
6992
6993 // evict bam
6994 {
6995 ObjectReadOperation op;
6996 op.cache_evict();
6997 librados::AioCompletion *completion = cluster.aio_create_completion();
6998 ASSERT_EQ(0, cache_ioctx.aio_operate(
6999 "bam", completion, &op,
7000 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7001 completion->wait_for_complete();
7c673cae
FG		 7002 ASSERT_EQ(0, completion->get_return_value());
7003 completion->release();
7004 }
7005 {
7006 bufferlist bl;
7007 ObjectReadOperation op;
7008 op.read(1, 0, &bl, NULL);
7009 librados::AioCompletion *completion = cluster.aio_create_completion();
7010 ASSERT_EQ(0, cache_ioctx.aio_operate(
7011 "bam", completion, &op,
7012 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7013 completion->wait_for_complete();
7c673cae
FG		 7014 ASSERT_EQ(-ENOENT, completion->get_return_value());
7015 completion->release();
7016 }
7017
7018 // read foo snap
7019 ioctx.snap_set_read(my_snaps[0]);
7020 {
7021 bufferlist bl;
7022 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7023 ASSERT_EQ('h', bl[0]);
7024 }
7025
7026 // evict foo snap
7027 {
7028 ObjectReadOperation op;
7029 op.cache_evict();
7030 librados::AioCompletion *completion = cluster.aio_create_completion();
7031 ASSERT_EQ(0, ioctx.aio_operate(
7032 "foo", completion, &op,
7033 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7034 completion->wait_for_complete();
7c673cae
FG		 7035 ASSERT_EQ(0, completion->get_return_value());
7036 completion->release();
7037 }
7038 // snap is gone...
7039 {
7040 bufferlist bl;
7041 ObjectReadOperation op;
7042 op.read(1, 0, &bl, NULL);
7043 librados::AioCompletion *completion = cluster.aio_create_completion();
7044 ASSERT_EQ(0, ioctx.aio_operate(
7045 "foo", completion, &op,
7046 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7047 completion->wait_for_complete();
7c673cae
FG		 7048 ASSERT_EQ(-ENOENT, completion->get_return_value());
7049 completion->release();
7050 }
7051 // head is still there...
7052 ioctx.snap_set_read(librados::SNAP_HEAD);
7053 {
7054 bufferlist bl;
7055 ObjectReadOperation op;
7056 op.read(1, 0, &bl, NULL);
7057 librados::AioCompletion *completion = cluster.aio_create_completion();
7058 ASSERT_EQ(0, ioctx.aio_operate(
7059 "foo", completion, &op,
7060 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7061 completion->wait_for_complete();
7c673cae
FG		 7062 ASSERT_EQ(0, completion->get_return_value());
7063 completion->release();
7064 }
7065
7066 // promote head + snap of bar
7067 ioctx.snap_set_read(librados::SNAP_HEAD);
7068 {
7069 bufferlist bl;
7070 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
7071 ASSERT_EQ('c', bl[0]);
7072 }
7073 ioctx.snap_set_read(my_snaps[0]);
7074 {
7075 bufferlist bl;
7076 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
7077 ASSERT_EQ('h', bl[0]);
7078 }
7079
7080 // evict bar head (fail)
7081 ioctx.snap_set_read(librados::SNAP_HEAD);
7082 {
7083 ObjectReadOperation op;
7084 op.cache_evict();
7085 librados::AioCompletion *completion = cluster.aio_create_completion();
7086 ASSERT_EQ(0, ioctx.aio_operate(
7087 "bar", completion, &op,
7088 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7089 completion->wait_for_complete();
7c673cae
FG		 7090 ASSERT_EQ(-EBUSY, completion->get_return_value());
7091 completion->release();
7092 }
7093
7094 // evict bar snap
7095 ioctx.snap_set_read(my_snaps[0]);
7096 {
7097 ObjectReadOperation op;
7098 op.cache_evict();
7099 librados::AioCompletion *completion = cluster.aio_create_completion();
7100 ASSERT_EQ(0, ioctx.aio_operate(
7101 "bar", completion, &op,
7102 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7103 completion->wait_for_complete();
7c673cae
FG		 7104 ASSERT_EQ(0, completion->get_return_value());
7105 completion->release();
7106 }
7107 // ...and then head
7108 ioctx.snap_set_read(librados::SNAP_HEAD);
7109 {
7110 bufferlist bl;
7111 ObjectReadOperation op;
7112 op.read(1, 0, &bl, NULL);
7113 librados::AioCompletion *completion = cluster.aio_create_completion();
7114 ASSERT_EQ(0, ioctx.aio_operate(
7115 "bar", completion, &op,
7116 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7117 completion->wait_for_complete();
7c673cae
FG		 7118 ASSERT_EQ(0, completion->get_return_value());
7119 completion->release();
7120 }
7121 {
7122 ObjectReadOperation op;
7123 op.cache_evict();
7124 librados::AioCompletion *completion = cluster.aio_create_completion();
7125 ASSERT_EQ(0, ioctx.aio_operate(
7126 "bar", completion, &op,
7127 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7128 completion->wait_for_complete();
7c673cae
FG		 7129 ASSERT_EQ(0, completion->get_return_value());
7130 completion->release();
7131 }
7132
7133 // cleanup
7134 ioctx.selfmanaged_snap_remove(my_snaps[0]);
7135}
7136
7137TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
7138 // configure cache
7139 bufferlist inbl;
7140 ASSERT_EQ(0, cluster.mon_command(
7141 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7142 "\", \"tierpool\": \"" + cache_pool_name +
7143 "\", \"force_nonempty\": \"--force-nonempty\" }",
7144 inbl, NULL, NULL));
7145 ASSERT_EQ(0, cluster.mon_command(
7146 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7147 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7148 inbl, NULL, NULL));
7149 ASSERT_EQ(0, cluster.mon_command(
7150 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7151 "\", \"mode\": \"writeback\"}",
7152 inbl, NULL, NULL));
7153
7154 // wait for maps to settle
7155 cluster.wait_for_latest_osdmap();
7156
7157 // create object
7158 {
7159 bufferlist bl;
7160 bl.append("hi there");
7161 ObjectWriteOperation op;
7162 op.write_full(bl);
7163 ASSERT_EQ(0, ioctx.operate("foo", &op));
7164 }
7165
7166 // verify the object is present in the cache tier
7167 {
7168 NObjectIterator it = cache_ioctx.nobjects_begin();
7169 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
7170 ASSERT_TRUE(it->get_oid() == string("foo"));
7171 ++it;
7172 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7173 }
7174
7175 // verify the object is NOT present in the base tier
7176 {
7177 NObjectIterator it = ioctx.nobjects_begin();
7178 ASSERT_TRUE(it == ioctx.nobjects_end());
7179 }
7180
7181 // verify dirty
7182 {
7183 bool dirty = false;
7184 int r = -1;
7185 ObjectReadOperation op;
7186 op.is_dirty(&dirty, &r);
7187 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
7188 ASSERT_TRUE(dirty);
7189 ASSERT_EQ(0, r);
7190 }
7191
7192 // pin
7193 {
7194 ObjectWriteOperation op;
7195 op.cache_pin();
7196 librados::AioCompletion *completion = cluster.aio_create_completion();
7197 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 7198 completion->wait_for_complete();
7c673cae
FG		 7199 ASSERT_EQ(0, completion->get_return_value());
7200 completion->release();
7201 }
7202
7203 // flush the pinned object with -EPERM
7204 {
7205 ObjectReadOperation op;
7206 op.cache_try_flush();
7207 librados::AioCompletion *completion = cluster.aio_create_completion();
7208 ASSERT_EQ(0, cache_ioctx.aio_operate(
7209 "foo", completion, &op,
7210 librados::OPERATION_IGNORE_OVERLAY |
7211 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 7212 completion->wait_for_complete();
7c673cae
FG		 7213 ASSERT_EQ(-EPERM, completion->get_return_value());
7214 completion->release();
7215 }
7216
7217 // unpin
7218 {
7219 ObjectWriteOperation op;
7220 op.cache_unpin();
7221 librados::AioCompletion *completion = cluster.aio_create_completion();
7222 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 7223 completion->wait_for_complete();
7c673cae
FG		 7224 ASSERT_EQ(0, completion->get_return_value());
7225 completion->release();
7226 }
7227
7228 // flush
7229 {
7230 ObjectReadOperation op;
7231 op.cache_try_flush();
7232 librados::AioCompletion *completion = cluster.aio_create_completion();
7233 ASSERT_EQ(0, cache_ioctx.aio_operate(
7234 "foo", completion, &op,
7235 librados::OPERATION_IGNORE_OVERLAY |
7236 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 7237 completion->wait_for_complete();
7c673cae
FG		 7238 ASSERT_EQ(0, completion->get_return_value());
7239 completion->release();
7240 }
7241
7242 // verify clean
7243 {
7244 bool dirty = false;
7245 int r = -1;
7246 ObjectReadOperation op;
7247 op.is_dirty(&dirty, &r);
7248 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
7249 ASSERT_FALSE(dirty);
7250 ASSERT_EQ(0, r);
7251 }
7252
7253 // verify in base tier
7254 {
7255 NObjectIterator it = ioctx.nobjects_begin();
7256 ASSERT_TRUE(it != ioctx.nobjects_end());
7257 ASSERT_TRUE(it->get_oid() == string("foo"));
7258 ++it;
7259 ASSERT_TRUE(it == ioctx.nobjects_end());
7260 }
7261
7262 // evict it
7263 {
7264 ObjectReadOperation op;
7265 op.cache_evict();
7266 librados::AioCompletion *completion = cluster.aio_create_completion();
7267 ASSERT_EQ(0, cache_ioctx.aio_operate(
7268 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7269 completion->wait_for_complete();
7c673cae
FG		 7270 ASSERT_EQ(0, completion->get_return_value());
7271 completion->release();
7272 }
7273
7274 // verify no longer in cache tier
7275 {
7276 NObjectIterator it = cache_ioctx.nobjects_begin();
7277 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7278 }
7279}
7280
7281TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
7282 // configure cache
7283 bufferlist inbl;
7284 ASSERT_EQ(0, cluster.mon_command(
7285 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7286 "\", \"tierpool\": \"" + cache_pool_name +
7287 "\", \"force_nonempty\": \"--force-nonempty\" }",
7288 inbl, NULL, NULL));
7289 ASSERT_EQ(0, cluster.mon_command(
7290 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7291 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7292 inbl, NULL, NULL));
7293 ASSERT_EQ(0, cluster.mon_command(
7294 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7295 "\", \"mode\": \"writeback\"}",
7296 inbl, NULL, NULL));
7297
7298 // wait for maps to settle
7299 cluster.wait_for_latest_osdmap();
7300
7301 // create object
7302 {
7303 bufferlist bl;
7304 bl.append("hi there");
7305 ObjectWriteOperation op;
7306 op.write_full(bl);
7307 ASSERT_EQ(0, ioctx.operate("foo", &op));
7308 }
7309
7310 // verify the object is present in the cache tier
7311 {
7312 NObjectIterator it = cache_ioctx.nobjects_begin();
7313 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
7314 ASSERT_TRUE(it->get_oid() == string("foo"));
7315 ++it;
7316 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7317 }
7318
7319 // verify the object is NOT present in the base tier
7320 {
7321 NObjectIterator it = ioctx.nobjects_begin();
7322 ASSERT_TRUE(it == ioctx.nobjects_end());
7323 }
7324
7325 // set omap
7326 {
7327 ObjectWriteOperation op;
7328 std::map<std::string, bufferlist> omap;
7329 omap["somekey"] = bufferlist();
7330 op.omap_set(omap);
7331 librados::AioCompletion *completion = cluster.aio_create_completion();
7332 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 7333 completion->wait_for_complete();
7c673cae
FG		 7334 ASSERT_EQ(0, completion->get_return_value());
7335 completion->release();
7336 }
7337
7338 // flush
7339 {
7340 ObjectReadOperation op;
7341 op.cache_flush();
7342 librados::AioCompletion *completion = cluster.aio_create_completion();
7343 ASSERT_EQ(0, cache_ioctx.aio_operate(
7344 "foo", completion, &op,
7345 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 7346 completion->wait_for_complete();
7c673cae
FG		 7347 ASSERT_NE(0, completion->get_return_value());
7348 completion->release();
7349 }
7350
7351 // get omap
7352 {
7353 ObjectReadOperation op;
7354 bufferlist bl;
7355 int prval = 0;
7356 std::set<std::string> keys;
7357 keys.insert("somekey");
7358 std::map<std::string, bufferlist> map;
7359
7360 op.omap_get_vals_by_keys(keys, &map, &prval);
7361 librados::AioCompletion *completion = cluster.aio_create_completion();
7362 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op, &bl));
7363 sleep(5);
7364 bool completed = completion->is_complete();
7365 if( !completed ) {
7366 cache_ioctx.aio_cancel(completion);
7367 std::cerr << "Most probably test case will hang here, please reset manually" << std::endl;
7368 ASSERT_TRUE(completed); //in fact we are locked forever at test case shutdown unless fix for http://tracker.ceph.com/issues/14511 is applied. Seems there is no workaround for that
7369 }
7370 completion->release();
7371 }
7372 // verify still not in base tier
7373 {
7374 ASSERT_TRUE(ioctx.nobjects_begin() == ioctx.nobjects_end());
7375 }
7376 // erase it
7377 {
7378 ObjectWriteOperation op;
7379 op.remove();
7380 ASSERT_EQ(0, ioctx.operate("foo", &op));
7381 }
7382 // flush whiteout
7383 {
7384 ObjectReadOperation op;
7385 op.cache_flush();
7386 librados::AioCompletion *completion = cluster.aio_create_completion();
7387 ASSERT_EQ(0, cache_ioctx.aio_operate(
7388 "foo", completion, &op,
7389 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 7390 completion->wait_for_complete();
7c673cae
FG		 7391 ASSERT_EQ(0, completion->get_return_value());
7392 completion->release();
7393 }
7394 // evict
7395 {
7396 ObjectReadOperation op;
7397 op.cache_evict();
7398 librados::AioCompletion *completion = cluster.aio_create_completion();
7399 ASSERT_EQ(0, cache_ioctx.aio_operate(
7400 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7401 completion->wait_for_complete();
7c673cae
FG		 7402 ASSERT_EQ(0, completion->get_return_value());
7403 completion->release();
7404 }
7405
7406 // verify no longer in cache tier
7407 {
7408 NObjectIterator it = cache_ioctx.nobjects_begin();
7409 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7410 }
7411 // or base tier
7412 {
7413 NObjectIterator it = ioctx.nobjects_begin();
7414 ASSERT_TRUE(it == ioctx.nobjects_end());
7415 }
7416}
7417
7418TEST_F(LibRadosTwoPoolsECPP, Flush) {
7419 // configure cache
7420 bufferlist inbl;
7421 ASSERT_EQ(0, cluster.mon_command(
7422 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7423 "\", \"tierpool\": \"" + cache_pool_name +
7424 "\", \"force_nonempty\": \"--force-nonempty\" }",
7425 inbl, NULL, NULL));
7426 ASSERT_EQ(0, cluster.mon_command(
7427 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7428 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7429 inbl, NULL, NULL));
7430 ASSERT_EQ(0, cluster.mon_command(
7431 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7432 "\", \"mode\": \"writeback\"}",
7433 inbl, NULL, NULL));
7434
7435 // wait for maps to settle
7436 cluster.wait_for_latest_osdmap();
7437
7438 uint64_t user_version = 0;
7439
7440 // create object
7441 {
7442 bufferlist bl;
7443 bl.append("hi there");
7444 ObjectWriteOperation op;
7445 op.write_full(bl);
7446 ASSERT_EQ(0, ioctx.operate("foo", &op));
7447 }
7448
7449 // verify the object is present in the cache tier
7450 {
7451 NObjectIterator it = cache_ioctx.nobjects_begin();
7452 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
7453 ASSERT_TRUE(it->get_oid() == string("foo"));
7454 ++it;
7455 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7456 }
7457
7458 // verify the object is NOT present in the base tier
7459 {
7460 NObjectIterator it = ioctx.nobjects_begin();
7461 ASSERT_TRUE(it == ioctx.nobjects_end());
7462 }
7463
7464 // verify dirty
7465 {
7466 bool dirty = false;
7467 int r = -1;
7468 ObjectReadOperation op;
7469 op.is_dirty(&dirty, &r);
7470 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
7471 ASSERT_TRUE(dirty);
7472 ASSERT_EQ(0, r);
7473 user_version = cache_ioctx.get_last_version();
7474 }
7475
7476 // pin
7477 {
7478 ObjectWriteOperation op;
7479 op.cache_pin();
7480 librados::AioCompletion *completion = cluster.aio_create_completion();
7481 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 7482 completion->wait_for_complete();
7c673cae
FG		 7483 ASSERT_EQ(0, completion->get_return_value());
7484 completion->release();
7485 }
7486
7487 // flush the pinned object with -EPERM
7488 {
7489 ObjectReadOperation op;
7490 op.cache_try_flush();
7491 librados::AioCompletion *completion = cluster.aio_create_completion();
7492 ASSERT_EQ(0, cache_ioctx.aio_operate(
7493 "foo", completion, &op,
7494 librados::OPERATION_IGNORE_OVERLAY |
7495 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 7496 completion->wait_for_complete();
7c673cae
FG		 7497 ASSERT_EQ(-EPERM, completion->get_return_value());
7498 completion->release();
7499 }
7500
7501 // unpin
7502 {
7503 ObjectWriteOperation op;
7504 op.cache_unpin();
7505 librados::AioCompletion *completion = cluster.aio_create_completion();
7506 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 7507 completion->wait_for_complete();
7c673cae
FG		 7508 ASSERT_EQ(0, completion->get_return_value());
7509 completion->release();
7510 }
7511
7512 // flush
7513 {
7514 ObjectReadOperation op;
7515 op.cache_flush();
7516 librados::AioCompletion *completion = cluster.aio_create_completion();
7517 ASSERT_EQ(0, cache_ioctx.aio_operate(
7518 "foo", completion, &op,
7519 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 7520 completion->wait_for_complete();
7c673cae
FG		 7521 ASSERT_EQ(0, completion->get_return_value());
7522 completion->release();
7523 }
7524
7525 // verify clean
7526 {
7527 bool dirty = false;
7528 int r = -1;
7529 ObjectReadOperation op;
7530 op.is_dirty(&dirty, &r);
7531 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
7532 ASSERT_FALSE(dirty);
7533 ASSERT_EQ(0, r);
7534 }
7535
7536 // verify in base tier
7537 {
7538 NObjectIterator it = ioctx.nobjects_begin();
7539 ASSERT_TRUE(it != ioctx.nobjects_end());
7540 ASSERT_TRUE(it->get_oid() == string("foo"));
7541 ++it;
7542 ASSERT_TRUE(it == ioctx.nobjects_end());
7543 }
7544
7545 // evict it
7546 {
7547 ObjectReadOperation op;
7548 op.cache_evict();
7549 librados::AioCompletion *completion = cluster.aio_create_completion();
7550 ASSERT_EQ(0, cache_ioctx.aio_operate(
7551 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7552 completion->wait_for_complete();
7c673cae
FG		 7553 ASSERT_EQ(0, completion->get_return_value());
7554 completion->release();
7555 }
7556
7557 // verify no longer in cache tier
7558 {
7559 NObjectIterator it = cache_ioctx.nobjects_begin();
7560 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7561 }
7562
7563 // read it again and verify the version is consistent
7564 {
7565 bufferlist bl;
7566 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
7567 ASSERT_EQ(user_version, cache_ioctx.get_last_version());
7568 }
7569
7570 // erase it
7571 {
7572 ObjectWriteOperation op;
7573 op.remove();
7574 ASSERT_EQ(0, ioctx.operate("foo", &op));
7575 }
7576
7577 // flush whiteout
7578 {
7579 ObjectReadOperation op;
7580 op.cache_flush();
7581 librados::AioCompletion *completion = cluster.aio_create_completion();
7582 ASSERT_EQ(0, cache_ioctx.aio_operate(
7583 "foo", completion, &op,
7584 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 7585 completion->wait_for_complete();
7c673cae
FG		 7586 ASSERT_EQ(0, completion->get_return_value());
7587 completion->release();
7588 }
7589
7590 // evict
7591 {
7592 ObjectReadOperation op;
7593 op.cache_evict();
7594 librados::AioCompletion *completion = cluster.aio_create_completion();
7595 ASSERT_EQ(0, cache_ioctx.aio_operate(
7596 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7597 completion->wait_for_complete();
7c673cae
FG		 7598 ASSERT_EQ(0, completion->get_return_value());
7599 completion->release();
7600 }
7601
7602 // verify no longer in cache tier
7603 {
7604 NObjectIterator it = cache_ioctx.nobjects_begin();
7605 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7606 }
7607 // or base tier
7608 {
7609 NObjectIterator it = ioctx.nobjects_begin();
7610 ASSERT_TRUE(it == ioctx.nobjects_end());
7611 }
7612}
7613
7614TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
7615 // configure cache
7616 bufferlist inbl;
7617 ASSERT_EQ(0, cluster.mon_command(
7618 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7619 "\", \"tierpool\": \"" + cache_pool_name +
7620 "\", \"force_nonempty\": \"--force-nonempty\" }",
7621 inbl, NULL, NULL));
7622 ASSERT_EQ(0, cluster.mon_command(
7623 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7624 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7625 inbl, NULL, NULL));
7626 ASSERT_EQ(0, cluster.mon_command(
7627 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7628 "\", \"mode\": \"writeback\"}",
7629 inbl, NULL, NULL));
7630
7631 // wait for maps to settle
7632 cluster.wait_for_latest_osdmap();
7633
7634 // create object
7635 {
7636 bufferlist bl;
7637 bl.append("a");
7638 ObjectWriteOperation op;
7639 op.write_full(bl);
7640 ASSERT_EQ(0, ioctx.operate("foo", &op));
7641 }
7642
7643 // create a snapshot, clone
7644 vector<uint64_t> my_snaps(1);
7645 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
7646 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
7647 my_snaps));
7648 {
7649 bufferlist bl;
7650 bl.append("b");
7651 ObjectWriteOperation op;
7652 op.write_full(bl);
7653 ASSERT_EQ(0, ioctx.operate("foo", &op));
7654 }
7655
7656 // and another
7657 my_snaps.resize(2);
7658 my_snaps[1] = my_snaps[0];
7659 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
7660 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
7661 my_snaps));
7662 {
7663 bufferlist bl;
7664 bl.append("c");
7665 ObjectWriteOperation op;
7666 op.write_full(bl);
7667 ASSERT_EQ(0, ioctx.operate("foo", &op));
7668 }
7669
7670 // verify the object is present in the cache tier
7671 {
7672 NObjectIterator it = cache_ioctx.nobjects_begin();
7673 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
7674 ASSERT_TRUE(it->get_oid() == string("foo"));
7675 ++it;
7676 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
7677 }
7678
7679 // verify the object is NOT present in the base tier
7680 {
7681 NObjectIterator it = ioctx.nobjects_begin();
7682 ASSERT_TRUE(it == ioctx.nobjects_end());
7683 }
7684
7685 // flush on head (should fail)
7686 ioctx.snap_set_read(librados::SNAP_HEAD);
7687 {
7688 ObjectReadOperation op;
7689 op.cache_flush();
7690 librados::AioCompletion *completion = cluster.aio_create_completion();
7691 ASSERT_EQ(0, ioctx.aio_operate(
7692 "foo", completion, &op,
7693 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7694 completion->wait_for_complete();
7c673cae
FG		 7695 ASSERT_EQ(-EBUSY, completion->get_return_value());
7696 completion->release();
7697 }
7698 // flush on recent snap (should fail)
7699 ioctx.snap_set_read(my_snaps[0]);
7700 {
7701 ObjectReadOperation op;
7702 op.cache_flush();
7703 librados::AioCompletion *completion = cluster.aio_create_completion();
7704 ASSERT_EQ(0, ioctx.aio_operate(
7705 "foo", completion, &op,
7706 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7707 completion->wait_for_complete();
7c673cae
FG		 7708 ASSERT_EQ(-EBUSY, completion->get_return_value());
7709 completion->release();
7710 }
7711 // flush on oldest snap
7712 ioctx.snap_set_read(my_snaps[1]);
7713 {
7714 ObjectReadOperation op;
7715 op.cache_flush();
7716 librados::AioCompletion *completion = cluster.aio_create_completion();
7717 ASSERT_EQ(0, ioctx.aio_operate(
7718 "foo", completion, &op,
7719 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7720 completion->wait_for_complete();
7c673cae
FG		 7721 ASSERT_EQ(0, completion->get_return_value());
7722 completion->release();
7723 }
7724 // flush on next oldest snap
7725 ioctx.snap_set_read(my_snaps[0]);
7726 {
7727 ObjectReadOperation op;
7728 op.cache_flush();
7729 librados::AioCompletion *completion = cluster.aio_create_completion();
7730 ASSERT_EQ(0, ioctx.aio_operate(
7731 "foo", completion, &op,
7732 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7733 completion->wait_for_complete();
7c673cae
FG		 7734 ASSERT_EQ(0, completion->get_return_value());
7735 completion->release();
7736 }
7737 // flush on head
7738 ioctx.snap_set_read(librados::SNAP_HEAD);
7739 {
7740 ObjectReadOperation op;
7741 op.cache_flush();
7742 librados::AioCompletion *completion = cluster.aio_create_completion();
7743 ASSERT_EQ(0, ioctx.aio_operate(
7744 "foo", completion, &op,
7745 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 7746 completion->wait_for_complete();
7c673cae
FG		 7747 ASSERT_EQ(0, completion->get_return_value());
7748 completion->release();
7749 }
7750
7751 // verify i can read the snaps from the cache pool
7752 ioctx.snap_set_read(librados::SNAP_HEAD);
7753 {
7754 bufferlist bl;
7755 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7756 ASSERT_EQ('c', bl[0]);
7757 }
7758 ioctx.snap_set_read(my_snaps[0]);
7759 {
7760 bufferlist bl;
7761 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7762 ASSERT_EQ('b', bl[0]);
7763 }
7764 ioctx.snap_set_read(my_snaps[1]);
7765 {
7766 bufferlist bl;
7767 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7768 ASSERT_EQ('a', bl[0]);
7769 }
7770
7771 // tear down tiers
7772 ASSERT_EQ(0, cluster.mon_command(
7773 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
7774 "\"}",
7775 inbl, NULL, NULL));
7776
7777 // wait for maps to settle
7778 cluster.wait_for_latest_osdmap();
7779
7780 // verify i can read the snaps from the base pool
7781 ioctx.snap_set_read(librados::SNAP_HEAD);
7782 {
7783 bufferlist bl;
7784 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7785 ASSERT_EQ('c', bl[0]);
7786 }
7787 ioctx.snap_set_read(my_snaps[0]);
7788 {
7789 bufferlist bl;
7790 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7791 ASSERT_EQ('b', bl[0]);
7792 }
7793 ioctx.snap_set_read(my_snaps[1]);
7794 {
7795 bufferlist bl;
7796 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
7797 ASSERT_EQ('a', bl[0]);
7798 }
7799
7800 ASSERT_EQ(0, cluster.mon_command(
7801 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7802 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7803 inbl, NULL, NULL));
7804 cluster.wait_for_latest_osdmap();
7805
7806 // cleanup
7807 ioctx.selfmanaged_snap_remove(my_snaps[0]);
7808}
7809
7810TEST_F(LibRadosTierECPP, FlushWriteRaces) {
7811 Rados cluster;
7812 std::string pool_name = get_temp_pool_name();
7813 std::string cache_pool_name = pool_name + "-cache";
7814 ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
7815 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
7816 IoCtx cache_ioctx;
7817 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 7818 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 7819 IoCtx ioctx;
7820 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
7821
7822 // configure cache
7823 bufferlist inbl;
7824 ASSERT_EQ(0, cluster.mon_command(
7825 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7826 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
7827 inbl, NULL, NULL));
7828 ASSERT_EQ(0, cluster.mon_command(
7829 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7830 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7831 inbl, NULL, NULL));
7832 ASSERT_EQ(0, cluster.mon_command(
7833 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7834 "\", \"mode\": \"writeback\"}",
7835 inbl, NULL, NULL));
7836
7837 // wait for maps to settle
7838 cluster.wait_for_latest_osdmap();
7839
7840 // create/dirty object
7841 bufferlist bl;
7842 bl.append("hi there");
7843 {
7844 ObjectWriteOperation op;
7845 op.write_full(bl);
7846 ASSERT_EQ(0, ioctx.operate("foo", &op));
7847 }
7848
7849 // flush + write
7850 {
7851 ObjectReadOperation op;
7852 op.cache_flush();
7853 librados::AioCompletion *completion = cluster.aio_create_completion();
7854 ASSERT_EQ(0, cache_ioctx.aio_operate(
7855 "foo", completion, &op,
7856 librados::OPERATION_IGNORE_OVERLAY, NULL));
7857
7858 ObjectWriteOperation op2;
7859 op2.write_full(bl);
7860 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7861 ASSERT_EQ(0, ioctx.aio_operate(
7862 "foo", completion2, &op2, 0));
7863
9f95a23c
TL		 7864 completion->wait_for_complete();
7865 completion2->wait_for_complete();
7c673cae
FG		 7866 ASSERT_EQ(0, completion->get_return_value());
7867 ASSERT_EQ(0, completion2->get_return_value());
7868 completion->release();
7869 completion2->release();
7870 }
7871
7872 int tries = 1000;
7873 do {
7874 // create/dirty object
7875 {
7876 bufferlist bl;
7877 bl.append("hi there");
7878 ObjectWriteOperation op;
7879 op.write_full(bl);
7880 ASSERT_EQ(0, ioctx.operate("foo", &op));
7881 }
7882
7883 // try-flush + write
7884 {
7885 ObjectReadOperation op;
7886 op.cache_try_flush();
7887 librados::AioCompletion *completion = cluster.aio_create_completion();
7888 ASSERT_EQ(0, cache_ioctx.aio_operate(
7889 "foo", completion, &op,
7890 librados::OPERATION_IGNORE_OVERLAY |
7891 librados::OPERATION_SKIPRWLOCKS, NULL));
7892
7893 ObjectWriteOperation op2;
7894 op2.write_full(bl);
7895 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7896 ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
7897
9f95a23c
TL		 7898 completion->wait_for_complete();
7899 completion2->wait_for_complete();
7c673cae
FG		 7900 int r = completion->get_return_value();
7901 ASSERT_TRUE(r == -EBUSY || r == 0);
7902 ASSERT_EQ(0, completion2->get_return_value());
7903 completion->release();
7904 completion2->release();
7905 if (r == -EBUSY)
7906 break;
7907 cout << "didn't get EBUSY, trying again" << std::endl;
7908 }
7909 ASSERT_TRUE(--tries);
7910 } while (true);
7911
7912 // tear down tiers
7913 ASSERT_EQ(0, cluster.mon_command(
7914 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
7915 "\"}",
7916 inbl, NULL, NULL));
7917 ASSERT_EQ(0, cluster.mon_command(
7918 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
7919 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
7920 inbl, NULL, NULL));
7921
7922 // wait for maps to settle before next test
7923 cluster.wait_for_latest_osdmap();
7924
7925 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
7926 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
7927}
7928
7929TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
7930 // configure cache
7931 bufferlist inbl;
7932 ASSERT_EQ(0, cluster.mon_command(
7933 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
7934 "\", \"tierpool\": \"" + cache_pool_name +
7935 "\", \"force_nonempty\": \"--force-nonempty\" }",
7936 inbl, NULL, NULL));
7937 ASSERT_EQ(0, cluster.mon_command(
7938 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
7939 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
7940 inbl, NULL, NULL));
7941 ASSERT_EQ(0, cluster.mon_command(
7942 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
7943 "\", \"mode\": \"writeback\"}",
7944 inbl, NULL, NULL));
7945
7946 // wait for maps to settle
7947 cluster.wait_for_latest_osdmap();
7948
7949 // create/dirty object
7950 {
7951 bufferlist bl;
7952 bl.append("hi there");
7953 ObjectWriteOperation op;
7954 op.write_full(bl);
7955 ASSERT_EQ(0, ioctx.operate("foo", &op));
7956 }
7957
7958 // flush + flush
7959 {
7960 ObjectReadOperation op;
7961 op.cache_flush();
7962 librados::AioCompletion *completion = cluster.aio_create_completion();
7963 ASSERT_EQ(0, cache_ioctx.aio_operate(
7964 "foo", completion, &op,
7965 librados::OPERATION_IGNORE_OVERLAY, NULL));
7966
7967 ObjectReadOperation op2;
7968 op2.cache_flush();
7969 librados::AioCompletion *completion2 = cluster.aio_create_completion();
7970 ASSERT_EQ(0, cache_ioctx.aio_operate(
7971 "foo", completion2, &op2,
7972 librados::OPERATION_IGNORE_OVERLAY, NULL));
7973
9f95a23c
TL		 7974 completion->wait_for_complete();
7975 completion2->wait_for_complete();
7c673cae
FG		 7976 ASSERT_EQ(0, completion->get_return_value());
7977 ASSERT_EQ(0, completion2->get_return_value());
7978 completion->release();
7979 completion2->release();
7980 }
7981
7982 // create/dirty object
7983 {
7984 bufferlist bl;
7985 bl.append("hi there");
7986 ObjectWriteOperation op;
7987 op.write_full(bl);
7988 ASSERT_EQ(0, ioctx.operate("foo", &op));
7989 }
7990
7991 // flush + try-flush
7992 {
7993 ObjectReadOperation op;
7994 op.cache_flush();
7995 librados::AioCompletion *completion = cluster.aio_create_completion();
7996 ASSERT_EQ(0, cache_ioctx.aio_operate(
7997 "foo", completion, &op,
7998 librados::OPERATION_IGNORE_OVERLAY, NULL));
7999
8000 ObjectReadOperation op2;
8001 op2.cache_try_flush();
8002 librados::AioCompletion *completion2 = cluster.aio_create_completion();
8003 ASSERT_EQ(0, cache_ioctx.aio_operate(
8004 "foo", completion2, &op2,
8005 librados::OPERATION_IGNORE_OVERLAY |
8006 librados::OPERATION_SKIPRWLOCKS, NULL));
8007
9f95a23c
TL		 8008 completion->wait_for_complete();
8009 completion2->wait_for_complete();
7c673cae
FG		 8010 ASSERT_EQ(0, completion->get_return_value());
8011 ASSERT_EQ(0, completion2->get_return_value());
8012 completion->release();
8013 completion2->release();
8014 }
8015
8016 // create/dirty object
8017 int tries = 1000;
8018 do {
8019 {
8020 bufferlist bl;
8021 bl.append("hi there");
8022 ObjectWriteOperation op;
8023 op.write_full(bl);
8024 ASSERT_EQ(0, ioctx.operate("foo", &op));
8025 }
8026
8027 // try-flush + flush
8028 // (flush will not piggyback on try-flush)
8029 {
8030 ObjectReadOperation op;
8031 op.cache_try_flush();
8032 librados::AioCompletion *completion = cluster.aio_create_completion();
8033 ASSERT_EQ(0, cache_ioctx.aio_operate(
8034 "foo", completion, &op,
8035 librados::OPERATION_IGNORE_OVERLAY |
8036 librados::OPERATION_SKIPRWLOCKS, NULL));
8037
8038 ObjectReadOperation op2;
8039 op2.cache_flush();
8040 librados::AioCompletion *completion2 = cluster.aio_create_completion();
8041 ASSERT_EQ(0, cache_ioctx.aio_operate(
8042 "foo", completion2, &op2,
8043 librados::OPERATION_IGNORE_OVERLAY, NULL));
8044
9f95a23c
TL		 8045 completion->wait_for_complete();
8046 completion2->wait_for_complete();
7c673cae
FG		 8047 int r = completion->get_return_value();
8048 ASSERT_TRUE(r == -EBUSY || r == 0);
8049 ASSERT_EQ(0, completion2->get_return_value());
8050 completion->release();
8051 completion2->release();
8052 if (r == -EBUSY)
8053 break;
8054 cout << "didn't get EBUSY, trying again" << std::endl;
8055 }
8056 ASSERT_TRUE(--tries);
8057 } while (true);
8058
8059 // create/dirty object
8060 {
8061 bufferlist bl;
8062 bl.append("hi there");
8063 ObjectWriteOperation op;
8064 op.write_full(bl);
8065 ASSERT_EQ(0, ioctx.operate("foo", &op));
8066 }
8067
8068 // try-flush + try-flush
8069 {
8070 ObjectReadOperation op;
8071 op.cache_try_flush();
8072 librados::AioCompletion *completion = cluster.aio_create_completion();
8073 ASSERT_EQ(0, cache_ioctx.aio_operate(
8074 "foo", completion, &op,
8075 librados::OPERATION_IGNORE_OVERLAY |
8076 librados::OPERATION_SKIPRWLOCKS, NULL));
8077
8078 ObjectReadOperation op2;
8079 op2.cache_try_flush();
8080 librados::AioCompletion *completion2 = cluster.aio_create_completion();
8081 ASSERT_EQ(0, cache_ioctx.aio_operate(
8082 "foo", completion2, &op2,
8083 librados::OPERATION_IGNORE_OVERLAY |
8084 librados::OPERATION_SKIPRWLOCKS, NULL));
8085
9f95a23c
TL		 8086 completion->wait_for_complete();
8087 completion2->wait_for_complete();
7c673cae
FG		 8088 ASSERT_EQ(0, completion->get_return_value());
8089 ASSERT_EQ(0, completion2->get_return_value());
8090 completion->release();
8091 completion2->release();
8092 }
8093}
8094
8095TEST_F(LibRadosTwoPoolsECPP, TryFlushReadRace) {
8096 // configure cache
8097 bufferlist inbl;
8098 ASSERT_EQ(0, cluster.mon_command(
8099 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8100 "\", \"tierpool\": \"" + cache_pool_name +
8101 "\", \"force_nonempty\": \"--force-nonempty\" }",
8102 inbl, NULL, NULL));
8103 ASSERT_EQ(0, cluster.mon_command(
8104 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8105 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8106 inbl, NULL, NULL));
8107 ASSERT_EQ(0, cluster.mon_command(
8108 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8109 "\", \"mode\": \"writeback\"}",
8110 inbl, NULL, NULL));
8111
8112 // wait for maps to settle
8113 cluster.wait_for_latest_osdmap();
8114
8115 // create/dirty object
8116 {
8117 bufferlist bl;
8118 bl.append("hi there");
8119 bufferptr bp(4000000); // make it big!
8120 bp.zero();
8121 bl.append(bp);
8122 ObjectWriteOperation op;
8123 op.write_full(bl);
8124 ASSERT_EQ(0, ioctx.operate("foo", &op));
8125 }
8126
8127 // start a continuous stream of reads
8128 read_ioctx = &ioctx;
9f95a23c 8129 test_lock.lock();
7c673cae
FG		 8130 for (int i = 0; i < max_reads; ++i) {
8131 start_flush_read();
8132 num_reads++;
8133 }
9f95a23c 8134 test_lock.unlock();
7c673cae
FG		 8135
8136 // try-flush
8137 ObjectReadOperation op;
8138 op.cache_try_flush();
8139 librados::AioCompletion *completion = cluster.aio_create_completion();
8140 ASSERT_EQ(0, cache_ioctx.aio_operate(
8141 "foo", completion, &op,
8142 librados::OPERATION_IGNORE_OVERLAY |
8143 librados::OPERATION_SKIPRWLOCKS, NULL));
8144
9f95a23c 8145 completion->wait_for_complete();
7c673cae
FG		 8146 ASSERT_EQ(0, completion->get_return_value());
8147 completion->release();
8148
8149 // stop reads
9f95a23c
TL		 8150 std::unique_lock locker{test_lock};
8151 max_reads = 0;
8152 cond.wait(locker, [] { return num_reads == 0;});
7c673cae
FG		 8153}
8154
8155TEST_F(LibRadosTierECPP, CallForcesPromote) {
8156 Rados cluster;
8157 std::string pool_name = get_temp_pool_name();
8158 std::string cache_pool_name = pool_name + "-cache";
8159 ASSERT_EQ("", create_one_ec_pool_pp(pool_name, cluster));
8160 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
8161 IoCtx cache_ioctx;
8162 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 8163 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 8164 IoCtx ioctx;
8165 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
8166
8167 // configure cache
8168 bufferlist inbl;
8169 ASSERT_EQ(0, cluster.mon_command(
8170 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8171 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8172 inbl, NULL, NULL));
8173 ASSERT_EQ(0, cluster.mon_command(
8174 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8175 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8176 inbl, NULL, NULL));
8177 ASSERT_EQ(0, cluster.mon_command(
8178 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8179 "\", \"mode\": \"writeback\"}",
8180 inbl, NULL, NULL));
8181
8182 // set things up such that the op would normally be proxied
8183 ASSERT_EQ(0, cluster.mon_command(
8184 set_pool_str(cache_pool_name, "hit_set_count", 2),
8185 inbl, NULL, NULL));
8186 ASSERT_EQ(0, cluster.mon_command(
8187 set_pool_str(cache_pool_name, "hit_set_period", 600),
8188 inbl, NULL, NULL));
8189 ASSERT_EQ(0, cluster.mon_command(
8190 set_pool_str(cache_pool_name, "hit_set_type",
8191 "explicit_object"),
8192 inbl, NULL, NULL));
8193 ASSERT_EQ(0, cluster.mon_command(
8194 set_pool_str(cache_pool_name, "min_read_recency_for_promote",
8195 "4"),
8196 inbl, NULL, NULL));
8197
8198 // wait for maps to settle
8199 cluster.wait_for_latest_osdmap();
8200
8201 // create/dirty object
8202 bufferlist bl;
8203 bl.append("hi there");
8204 {
8205 ObjectWriteOperation op;
8206 op.write_full(bl);
8207 ASSERT_EQ(0, ioctx.operate("foo", &op));
8208 }
8209
8210 // flush
8211 {
8212 ObjectReadOperation op;
8213 op.cache_flush();
8214 librados::AioCompletion *completion = cluster.aio_create_completion();
8215 ASSERT_EQ(0, cache_ioctx.aio_operate(
8216 "foo", completion, &op,
8217 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 8218 completion->wait_for_complete();
7c673cae
FG		 8219 ASSERT_EQ(0, completion->get_return_value());
8220 completion->release();
8221 }
8222
8223 // evict
8224 {
8225 ObjectReadOperation op;
8226 op.cache_evict();
8227 librados::AioCompletion *completion = cluster.aio_create_completion();
8228 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
8229 librados::OPERATION_IGNORE_CACHE,
8230 NULL));
9f95a23c 8231 completion->wait_for_complete();
7c673cae
FG		 8232 ASSERT_EQ(0, completion->get_return_value());
8233 completion->release();
8234 }
8235
8236 // call
8237 {
8238 ObjectReadOperation op;
8239 bufferlist bl;
8240 op.exec("rbd", "get_id", bl);
8241 bufferlist out;
8242 // should get EIO (not an rbd object), not -EOPNOTSUPP (we didn't promote)
8243 ASSERT_EQ(-5, ioctx.operate("foo", &op, &out));
8244 }
8245
8246 // make sure foo is back in the cache tier
8247 {
8248 NObjectIterator it = cache_ioctx.nobjects_begin();
8249 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
8250 ASSERT_TRUE(it->get_oid() == string("foo"));
8251 ++it;
8252 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
8253 }
8254
8255 // tear down tiers
8256 ASSERT_EQ(0, cluster.mon_command(
8257 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
8258 "\"}",
8259 inbl, NULL, NULL));
8260 ASSERT_EQ(0, cluster.mon_command(
8261 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8262 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8263 inbl, NULL, NULL));
8264
8265 // wait for maps to settle before next test
8266 cluster.wait_for_latest_osdmap();
8267
8268 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
9f95a23c 8269 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, cluster));
7c673cae
FG		 8270}
8271
8272TEST_F(LibRadosTierECPP, HitSetNone) {
8273 {
8274 list< pair<time_t,time_t> > ls;
8275 AioCompletion *c = librados::Rados::aio_create_completion();
8276 ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
8277 c->wait_for_complete();
8278 ASSERT_EQ(0, c->get_return_value());
8279 ASSERT_TRUE(ls.empty());
8280 c->release();
8281 }
8282 {
8283 bufferlist bl;
8284 AioCompletion *c = librados::Rados::aio_create_completion();
8285 ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
8286 c->wait_for_complete();
8287 ASSERT_EQ(-ENOENT, c->get_return_value());
8288 c->release();
8289 }
8290}
8291
8292TEST_F(LibRadosTwoPoolsECPP, HitSetRead) {
8293 // make it a tier
8294 bufferlist inbl;
8295 ASSERT_EQ(0, cluster.mon_command(
8296 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8297 "\", \"tierpool\": \"" + cache_pool_name +
8298 "\", \"force_nonempty\": \"--force-nonempty\" }",
8299 inbl, NULL, NULL));
8300
8301 // enable hitset tracking for this pool
8302 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
8303 inbl, NULL, NULL));
8304 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
8305 inbl, NULL, NULL));
8306 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
8307 "explicit_object"),
8308 inbl, NULL, NULL));
8309
8310 // wait for maps to settle
8311 cluster.wait_for_latest_osdmap();
8312
8313 cache_ioctx.set_namespace("");
8314
8315 // keep reading until we see our object appear in the HitSet
8316 utime_t start = ceph_clock_now();
8317 utime_t hard_stop = start + utime_t(600, 0);
8318
8319 while (true) {
8320 utime_t now = ceph_clock_now();
8321 ASSERT_TRUE(now < hard_stop);
8322
8323 string name = "foo";
8324 uint32_t hash;
8325 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
8326 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
8327 cluster.pool_lookup(cache_pool_name.c_str()), "");
8328
8329 bufferlist bl;
8330 ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
8331
8332 bufferlist hbl;
8333 AioCompletion *c = librados::Rados::aio_create_completion();
8334 ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
8335 c->wait_for_complete();
8336 c->release();
8337
8338 if (hbl.length()) {
11fdf7f2 8339 auto p = hbl.cbegin();
7c673cae 8340 HitSet hs;
11fdf7f2 8341 decode(hs, p);
7c673cae
FG		 8342 if (hs.contains(oid)) {
8343 cout << "ok, hit_set contains " << oid << std::endl;
8344 break;
8345 }
8346 cout << "hmm, not in HitSet yet" << std::endl;
8347 } else {
8348 cout << "hmm, no HitSet yet" << std::endl;
8349 }
8350
8351 sleep(1);
8352 }
8353}
8354
8355// disable this test until hitset-get reliably works on EC pools
8356#if 0
8357TEST_F(LibRadosTierECPP, HitSetWrite) {
8358 int num_pg = _get_pg_num(cluster, pool_name);
11fdf7f2 8359 ceph_assert(num_pg > 0);
7c673cae
FG		 8360
8361 // enable hitset tracking for this pool
8362 bufferlist inbl;
8363 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 8),
8364 inbl, NULL, NULL));
8365 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", 600),
8366 inbl, NULL, NULL));
8367 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type",
8368 "explicit_hash"),
8369 inbl, NULL, NULL));
8370
8371 // wait for maps to settle
8372 cluster.wait_for_latest_osdmap();
8373
8374 ioctx.set_namespace("");
8375
8376 // do a bunch of writes
8377 for (int i=0; i<1000; ++i) {
8378 bufferlist bl;
8379 bl.append("a");
8380 ASSERT_EQ(0, ioctx.write(stringify(i), bl, 1, 0));
8381 }
8382
8383 // get HitSets
8384 std::map<int,HitSet> hitsets;
8385 for (int i=0; i<num_pg; ++i) {
8386 list< pair<time_t,time_t> > ls;
8387 AioCompletion *c = librados::Rados::aio_create_completion();
8388 ASSERT_EQ(0, ioctx.hit_set_list(i, c, &ls));
8389 c->wait_for_complete();
8390 c->release();
8391 std::cout << "pg " << i << " ls " << ls << std::endl;
8392 ASSERT_FALSE(ls.empty());
8393
8394 // get the latest
8395 c = librados::Rados::aio_create_completion();
8396 bufferlist bl;
8397 ASSERT_EQ(0, ioctx.hit_set_get(i, c, ls.back().first, &bl));
8398 c->wait_for_complete();
8399 c->release();
8400
8401 //std::cout << "bl len is " << bl.length() << "\n";
8402 //bl.hexdump(std::cout);
8403 //std::cout << std::endl;
8404
11fdf7f2
TL		 8405 auto p = bl.cbegin();
8406 decode(hitsets[i], p);
7c673cae
FG		 8407
8408 // cope with racing splits by refreshing pg_num
8409 if (i == num_pg - 1)
8410 num_pg = _get_pg_num(cluster, pool_name);
8411 }
8412
8413 for (int i=0; i<1000; ++i) {
8414 string n = stringify(i);
8415 uint32_t hash = ioctx.get_object_hash_position(n);
8416 hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
8417 cluster.pool_lookup(pool_name.c_str()), "");
8418 std::cout << "checking for " << oid << std::endl;
8419 bool found = false;
8420 for (int p=0; p<num_pg; ++p) {
8421 if (hitsets[p].contains(oid)) {
8422 found = true;
8423 break;
8424 }
8425 }
8426 ASSERT_TRUE(found);
8427 }
8428}
8429#endif
8430
8431TEST_F(LibRadosTwoPoolsECPP, HitSetTrim) {
8432 unsigned count = 3;
8433 unsigned period = 3;
8434
8435 // make it a tier
8436 bufferlist inbl;
8437 ASSERT_EQ(0, cluster.mon_command(
8438 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8439 "\", \"tierpool\": \"" + cache_pool_name +
8440 "\", \"force_nonempty\": \"--force-nonempty\" }",
8441 inbl, NULL, NULL));
8442
8443 // enable hitset tracking for this pool
8444 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
8445 inbl, NULL, NULL));
8446 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
8447 inbl, NULL, NULL));
8448 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
8449 inbl, NULL, NULL));
8450 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
8451 inbl, NULL, NULL));
8452
8453 // wait for maps to settle
8454 cluster.wait_for_latest_osdmap();
8455
8456 cache_ioctx.set_namespace("");
8457
8458 // do a bunch of writes and make sure the hitsets rotate
8459 utime_t start = ceph_clock_now();
8460 utime_t hard_stop = start + utime_t(count * period * 50, 0);
8461
8462 time_t first = 0;
8463 int bsize = alignment;
8464 char *buf = (char *)new char[bsize];
8465 memset(buf, 'f', bsize);
8466
8467 while (true) {
8468 string name = "foo";
8469 uint32_t hash;
8470 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
8471 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
8472
8473 bufferlist bl;
8474 bl.append(buf, bsize);
8475 ASSERT_EQ(0, cache_ioctx.append("foo", bl, bsize));
8476
8477 list<pair<time_t, time_t> > ls;
8478 AioCompletion *c = librados::Rados::aio_create_completion();
8479 ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
8480 c->wait_for_complete();
8481 c->release();
8482
8483 cout << " got ls " << ls << std::endl;
8484 if (!ls.empty()) {
8485 if (!first) {
8486 first = ls.front().first;
8487 cout << "first is " << first << std::endl;
8488 } else {
8489 if (ls.front().first != first) {
8490 cout << "first now " << ls.front().first << ", trimmed" << std::endl;
8491 break;
8492 }
8493 }
8494 }
8495
8496 utime_t now = ceph_clock_now();
8497 ASSERT_TRUE(now < hard_stop);
8498
8499 sleep(1);
8500 }
8501 delete[] buf;
8502}
8503
8504TEST_F(LibRadosTwoPoolsECPP, PromoteOn2ndRead) {
8505 // create object
8506 for (int i=0; i<20; ++i) {
8507 bufferlist bl;
8508 bl.append("hi there");
8509 ObjectWriteOperation op;
8510 op.write_full(bl);
8511 ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
8512 }
8513
8514 // configure cache
8515 bufferlist inbl;
8516 ASSERT_EQ(0, cluster.mon_command(
8517 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8518 "\", \"tierpool\": \"" + cache_pool_name +
8519 "\", \"force_nonempty\": \"--force-nonempty\" }",
8520 inbl, NULL, NULL));
8521 ASSERT_EQ(0, cluster.mon_command(
8522 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8523 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8524 inbl, NULL, NULL));
8525 ASSERT_EQ(0, cluster.mon_command(
8526 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8527 "\", \"mode\": \"writeback\"}",
8528 inbl, NULL, NULL));
8529
8530 // enable hitset tracking for this pool
8531 ASSERT_EQ(0, cluster.mon_command(
8532 set_pool_str(cache_pool_name, "hit_set_count", 2),
8533 inbl, NULL, NULL));
8534 ASSERT_EQ(0, cluster.mon_command(
8535 set_pool_str(cache_pool_name, "hit_set_period", 600),
8536 inbl, NULL, NULL));
8537 ASSERT_EQ(0, cluster.mon_command(
8538 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
8539 inbl, NULL, NULL));
8540 ASSERT_EQ(0, cluster.mon_command(
8541 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
8542 inbl, NULL, NULL));
8543 ASSERT_EQ(0, cluster.mon_command(
8544 set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
8545 inbl, NULL, NULL));
8546 ASSERT_EQ(0, cluster.mon_command(
8547 set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
8548 inbl, NULL, NULL));
8549
8550 // wait for maps to settle
8551 cluster.wait_for_latest_osdmap();
8552
8553 int fake = 0; // set this to non-zero to test spurious promotion,
8554 // e.g. from thrashing
8555 int attempt = 0;
8556 string obj;
8557 while (true) {
8558 // 1st read, don't trigger a promote
8559 obj = "foo" + stringify(attempt);
8560 cout << obj << std::endl;
8561 {
8562 bufferlist bl;
8563 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
8564 if (--fake >= 0) {
8565 sleep(1);
8566 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
8567 sleep(1);
8568 }
8569 }
8570
8571 // verify the object is NOT present in the cache tier
8572 {
8573 bool found = false;
8574 NObjectIterator it = cache_ioctx.nobjects_begin();
8575 while (it != cache_ioctx.nobjects_end()) {
8576 cout << " see " << it->get_oid() << std::endl;
8577 if (it->get_oid() == string(obj.c_str())) {
8578 found = true;
8579 break;
8580 }
8581 ++it;
8582 }
8583 if (!found)
8584 break;
8585 }
8586
8587 ++attempt;
8588 ASSERT_LE(attempt, 20);
8589 cout << "hrm, object is present in cache on attempt " << attempt
8590 << ", retrying" << std::endl;
8591 }
8592
8593 // Read until the object is present in the cache tier
8594 cout << "verifying " << obj << " is eventually promoted" << std::endl;
8595 while (true) {
8596 bufferlist bl;
8597 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
8598
8599 bool there = false;
8600 NObjectIterator it = cache_ioctx.nobjects_begin();
8601 while (it != cache_ioctx.nobjects_end()) {
8602 if (it->get_oid() == string(obj.c_str())) {
8603 there = true;
8604 break;
8605 }
8606 ++it;
8607 }
8608 if (there)
8609 break;
8610
8611 sleep(1);
8612 }
8613
8614 // tear down tiers
8615 ASSERT_EQ(0, cluster.mon_command(
8616 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
8617 "\"}",
8618 inbl, NULL, NULL));
8619 ASSERT_EQ(0, cluster.mon_command(
8620 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8621 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8622 inbl, NULL, NULL));
8623
8624 // wait for maps to settle before next test
8625 cluster.wait_for_latest_osdmap();
8626}
8627
8628TEST_F(LibRadosTwoPoolsECPP, ProxyRead) {
8629 // create object
8630 {
8631 bufferlist bl;
8632 bl.append("hi there");
8633 ObjectWriteOperation op;
8634 op.write_full(bl);
8635 ASSERT_EQ(0, ioctx.operate("foo", &op));
8636 }
8637
8638 // configure cache
8639 bufferlist inbl;
8640 ASSERT_EQ(0, cluster.mon_command(
8641 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8642 "\", \"tierpool\": \"" + cache_pool_name +
8643 "\", \"force_nonempty\": \"--force-nonempty\" }",
8644 inbl, NULL, NULL));
8645 ASSERT_EQ(0, cluster.mon_command(
8646 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8647 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8648 inbl, NULL, NULL));
8649 ASSERT_EQ(0, cluster.mon_command(
8650 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8651 "\", \"mode\": \"readproxy\"}",
8652 inbl, NULL, NULL));
8653
8654 // wait for maps to settle
8655 cluster.wait_for_latest_osdmap();
8656
8657 // read and verify the object
8658 {
8659 bufferlist bl;
8660 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
8661 ASSERT_EQ('h', bl[0]);
8662 }
8663
8664 // Verify 10 times the object is NOT present in the cache tier
8665 uint32_t i = 0;
8666 while (i++ < 10) {
8667 NObjectIterator it = cache_ioctx.nobjects_begin();
8668 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
8669 sleep(1);
8670 }
8671
8672 // tear down tiers
8673 ASSERT_EQ(0, cluster.mon_command(
8674 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
8675 "\"}",
8676 inbl, NULL, NULL));
8677 ASSERT_EQ(0, cluster.mon_command(
8678 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8679 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8680 inbl, NULL, NULL));
8681
8682 // wait for maps to settle before next test
8683 cluster.wait_for_latest_osdmap();
8684}
8685
8686TEST_F(LibRadosTwoPoolsECPP, CachePin) {
8687 // create object
8688 {
8689 bufferlist bl;
8690 bl.append("hi there");
8691 ObjectWriteOperation op;
8692 op.write_full(bl);
8693 ASSERT_EQ(0, ioctx.operate("foo", &op));
8694 }
8695 {
8696 bufferlist bl;
8697 bl.append("hi there");
8698 ObjectWriteOperation op;
8699 op.write_full(bl);
8700 ASSERT_EQ(0, ioctx.operate("bar", &op));
8701 }
8702 {
8703 bufferlist bl;
8704 bl.append("hi there");
8705 ObjectWriteOperation op;
8706 op.write_full(bl);
8707 ASSERT_EQ(0, ioctx.operate("baz", &op));
8708 }
8709 {
8710 bufferlist bl;
8711 bl.append("hi there");
8712 ObjectWriteOperation op;
8713 op.write_full(bl);
8714 ASSERT_EQ(0, ioctx.operate("bam", &op));
8715 }
8716
8717 // configure cache
8718 bufferlist inbl;
8719 ASSERT_EQ(0, cluster.mon_command(
8720 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8721 "\", \"tierpool\": \"" + cache_pool_name +
8722 "\", \"force_nonempty\": \"--force-nonempty\" }",
8723 inbl, NULL, NULL));
8724 ASSERT_EQ(0, cluster.mon_command(
8725 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
8726 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
8727 inbl, NULL, NULL));
8728 ASSERT_EQ(0, cluster.mon_command(
8729 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
8730 "\", \"mode\": \"writeback\"}",
8731 inbl, NULL, NULL));
8732
8733 // wait for maps to settle
8734 cluster.wait_for_latest_osdmap();
8735
8736 // read, trigger promote
8737 {
8738 bufferlist bl;
8739 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
8740 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
8741 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
8742 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
8743 }
8744
8745 // verify the objects are present in the cache tier
8746 {
8747 NObjectIterator it = cache_ioctx.nobjects_begin();
8748 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
8749 for (uint32_t i = 0; i < 4; i++) {
8750 ASSERT_TRUE(it->get_oid() == string("foo") ||
8751 it->get_oid() == string("bar") ||
8752 it->get_oid() == string("baz") ||
8753 it->get_oid() == string("bam"));
8754 ++it;
8755 }
8756 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
8757 }
8758
8759 // pin objects
8760 {
8761 ObjectWriteOperation op;
8762 op.cache_pin();
8763 librados::AioCompletion *completion = cluster.aio_create_completion();
8764 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 8765 completion->wait_for_complete();
7c673cae
FG		 8766 ASSERT_EQ(0, completion->get_return_value());
8767 completion->release();
8768 }
8769 {
8770 ObjectWriteOperation op;
8771 op.cache_pin();
8772 librados::AioCompletion *completion = cluster.aio_create_completion();
8773 ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
9f95a23c 8774 completion->wait_for_complete();
7c673cae
FG		 8775 ASSERT_EQ(0, completion->get_return_value());
8776 completion->release();
8777 }
8778
8779 // enable agent
8780 ASSERT_EQ(0, cluster.mon_command(
8781 set_pool_str(cache_pool_name, "hit_set_count", 2),
8782 inbl, NULL, NULL));
8783 ASSERT_EQ(0, cluster.mon_command(
8784 set_pool_str(cache_pool_name, "hit_set_period", 600),
8785 inbl, NULL, NULL));
8786 ASSERT_EQ(0, cluster.mon_command(
8787 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
8788 inbl, NULL, NULL));
8789 ASSERT_EQ(0, cluster.mon_command(
8790 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
8791 inbl, NULL, NULL));
8792 ASSERT_EQ(0, cluster.mon_command(
8793 set_pool_str(cache_pool_name, "target_max_objects", 1),
8794 inbl, NULL, NULL));
8795
8796 sleep(10);
8797
8798 // Verify the pinned object 'foo' is not flushed/evicted
8799 uint32_t count = 0;
8800 while (true) {
8801 bufferlist bl;
8802 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
8803
8804 count = 0;
8805 NObjectIterator it = cache_ioctx.nobjects_begin();
8806 while (it != cache_ioctx.nobjects_end()) {
8807 ASSERT_TRUE(it->get_oid() == string("foo") ||
8808 it->get_oid() == string("bar") ||
8809 it->get_oid() == string("baz") ||
8810 it->get_oid() == string("bam"));
8811 ++count;
8812 ++it;
8813 }
8814 if (count == 2) {
8815 ASSERT_TRUE(it->get_oid() == string("foo") ||
8816 it->get_oid() == string("baz"));
8817 break;
8818 }
8819
8820 sleep(1);
8821 }
8822
8823 // tear down tiers
8824 ASSERT_EQ(0, cluster.mon_command(
8825 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
8826 "\"}",
8827 inbl, NULL, NULL));
8828 ASSERT_EQ(0, cluster.mon_command(
8829 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8830 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8831 inbl, NULL, NULL));
8832
8833 // wait for maps to settle before next test
8834 cluster.wait_for_latest_osdmap();
8835}
31f18b77 8836TEST_F(LibRadosTwoPoolsECPP, SetRedirectRead) {
31f18b77
FG		 8837 // create object
8838 {
8839 bufferlist bl;
8840 bl.append("hi there");
8841 ObjectWriteOperation op;
8842 op.write_full(bl);
8843 ASSERT_EQ(0, ioctx.operate("foo", &op));
8844 }
8845 {
8846 bufferlist bl;
8847 bl.append("there");
8848 ObjectWriteOperation op;
8849 op.write_full(bl);
8850 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
8851 }
8852
8853 // configure tier
8854 bufferlist inbl;
8855 ASSERT_EQ(0, cluster.mon_command(
8856 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
8857 "\", \"tierpool\": \"" + cache_pool_name +
8858 "\", \"force_nonempty\": \"--force-nonempty\" }",
8859 inbl, NULL, NULL));
8860
8861 // wait for maps to settle
8862 cluster.wait_for_latest_osdmap();
8863
8864 {
8865 ObjectWriteOperation op;
8866 op.set_redirect("bar", cache_ioctx, 0);
8867 librados::AioCompletion *completion = cluster.aio_create_completion();
8868 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 8869 completion->wait_for_complete();
31f18b77
FG		 8870 ASSERT_EQ(0, completion->get_return_value());
8871 completion->release();
8872 }
8873 // read and verify the object
8874 {
8875 bufferlist bl;
8876 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
8877 ASSERT_EQ('t', bl[0]);
8878 }
8879
8880 ASSERT_EQ(0, cluster.mon_command(
8881 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
8882 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
8883 inbl, NULL, NULL));
8884
8885 // wait for maps to settle before next test
8886 cluster.wait_for_latest_osdmap();
8887}
11fdf7f2
TL		 8888
8889TEST_F(LibRadosTwoPoolsECPP, SetChunkRead) {
8890 // note: require >= mimic
8891
11fdf7f2 8892 {
f67539c2
TL		 8893 bufferlist bl;
8894 bl.append("there hi");
11fdf7f2 8895 ObjectWriteOperation op;
f67539c2
TL		 8896 op.write_full(bl);
8897 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
11fdf7f2 8898 }
f67539c2 8899
11fdf7f2
TL		 8900 {
8901 bufferlist bl;
f67539c2 8902 bl.append("There hi");
11fdf7f2
TL		 8903 ObjectWriteOperation op;
8904 op.write_full(bl);
f67539c2 8905 ASSERT_EQ(0, ioctx.operate("bar", &op));
11fdf7f2
TL		 8906 }
8907
11fdf7f2
TL		 8908 // wait for maps to settle
8909 cluster.wait_for_latest_osdmap();
8910
8911 // set_chunk
f67539c2
TL		 8912 manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 4, "bar", "foo");
8913
8914 // promote
11fdf7f2
TL		 8915 {
8916 ObjectWriteOperation op;
f67539c2 8917 op.tier_promote();
11fdf7f2 8918 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2 8919 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 8920 completion->wait_for_complete();
11fdf7f2
TL		 8921 ASSERT_EQ(0, completion->get_return_value());
8922 completion->release();
8923 }
8924
11fdf7f2
TL		 8925 // read and verify the object
8926 {
8927 bufferlist bl;
f67539c2 8928 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
11fdf7f2
TL		 8929 ASSERT_EQ('T', bl[0]);
8930 }
8931
11fdf7f2
TL		 8932 // wait for maps to settle before next test
8933 cluster.wait_for_latest_osdmap();
8934}
8935
8936TEST_F(LibRadosTwoPoolsECPP, ManifestPromoteRead) {
8937 // note: require >= mimic
8938
8939 // create object
8940 {
8941 bufferlist bl;
f67539c2 8942 bl.append("hiaa there");
11fdf7f2
TL		 8943 ObjectWriteOperation op;
8944 op.write_full(bl);
f67539c2 8945 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
11fdf7f2
TL		 8946 }
8947 {
f67539c2
TL		 8948 bufferlist bl;
8949 bl.append("base chunk");
11fdf7f2 8950 ObjectWriteOperation op;
f67539c2
TL		 8951 op.write_full(bl);
8952 ASSERT_EQ(0, cache_ioctx.operate("foo-chunk", &op));
11fdf7f2
TL		 8953 }
8954 {
8955 bufferlist bl;
f67539c2 8956 bl.append("HIaa there");
11fdf7f2
TL		 8957 ObjectWriteOperation op;
8958 op.write_full(bl);
f67539c2 8959 ASSERT_EQ(0, ioctx.operate("bar", &op));
11fdf7f2
TL		 8960 }
8961 {
8962 bufferlist bl;
8963 bl.append("BASE CHUNK");
8964 ObjectWriteOperation op;
8965 op.write_full(bl);
f67539c2 8966 ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
11fdf7f2
TL		 8967 }
8968
11fdf7f2
TL		 8969 // set-redirect
8970 {
8971 ObjectWriteOperation op;
f67539c2 8972 op.set_redirect("bar", ioctx, 0);
11fdf7f2 8973 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2 8974 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 8975 completion->wait_for_complete();
11fdf7f2
TL		 8976 ASSERT_EQ(0, completion->get_return_value());
8977 completion->release();
8978 }
8979 // set-chunk
f67539c2 8980 manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "bar-chunk", "foo-chunk");
11fdf7f2
TL		 8981 // promote
8982 {
8983 ObjectWriteOperation op;
8984 op.tier_promote();
8985 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2 8986 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 8987 completion->wait_for_complete();
11fdf7f2
TL		 8988 ASSERT_EQ(0, completion->get_return_value());
8989 completion->release();
8990 }
8991 // read and verify the object (redirect)
8992 {
8993 bufferlist bl;
f67539c2 8994 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
11fdf7f2
TL		 8995 ASSERT_EQ('H', bl[0]);
8996 }
8997 // promote
8998 {
8999 ObjectWriteOperation op;
9000 op.tier_promote();
9001 librados::AioCompletion *completion = cluster.aio_create_completion();
f67539c2 9002 ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 9003 completion->wait_for_complete();
11fdf7f2
TL		 9004 ASSERT_EQ(0, completion->get_return_value());
9005 completion->release();
9006 }
9007 // read and verify the object
9008 {
9009 bufferlist bl;
f67539c2 9010 ASSERT_EQ(1, cache_ioctx.read("foo-chunk", bl, 1, 0));
11fdf7f2
TL		 9011 ASSERT_EQ('B', bl[0]);
9012 }
9013
11fdf7f2
TL		 9014 // wait for maps to settle before next test
9015 cluster.wait_for_latest_osdmap();
9016}
9017
f67539c2
TL		 9018TEST_F(LibRadosTwoPoolsECPP, TrySetDedupTier) {
9019 // note: require >= mimic
9020
9021 bufferlist inbl;
9022 ASSERT_EQ(-EOPNOTSUPP, cluster.mon_command(
9023 set_pool_str(pool_name, "dedup_tier", cache_pool_name),
9024 inbl, NULL, NULL));
9025}
9026
11fdf7f2
TL		 9027TEST_F(LibRadosTwoPoolsPP, PropagateBaseTierError) {
9028 // write object to base tier
9029 bufferlist omap_bl;
9030 encode(static_cast<uint32_t>(0U), omap_bl);
9031
9032 ObjectWriteOperation op1;
9033 op1.omap_set({{"somekey", omap_bl}});
9034 ASSERT_EQ(0, ioctx.operate("propagate-base-tier-error", &op1));
9035
9036 // configure cache
9037 bufferlist inbl;
9038 ASSERT_EQ(0, cluster.mon_command(
9039 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
9040 "\", \"tierpool\": \"" + cache_pool_name +
9041 "\", \"force_nonempty\": \"--force-nonempty\" }",
9042 inbl, NULL, NULL));
9043 ASSERT_EQ(0, cluster.mon_command(
9044 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
9045 "\", \"mode\": \"writeback\"}",
9046 inbl, NULL, NULL));
9047 ASSERT_EQ(0, cluster.mon_command(
9048 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
9049 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
9050 inbl, NULL, NULL));
9051
9052 ASSERT_EQ(0, cluster.mon_command(
9053 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
9054 inbl, NULL, NULL));
9055 ASSERT_EQ(0, cluster.mon_command(
9056 set_pool_str(cache_pool_name, "hit_set_count", 1),
9057 inbl, NULL, NULL));
9058 ASSERT_EQ(0, cluster.mon_command(
9059 set_pool_str(cache_pool_name, "hit_set_period", 600),
9060 inbl, NULL, NULL));
9061 ASSERT_EQ(0, cluster.mon_command(
9062 set_pool_str(cache_pool_name, "target_max_objects", 250),
9063 inbl, NULL, NULL));
9064
9065 // wait for maps to settle
9066 cluster.wait_for_latest_osdmap();
9067
9068 // guarded op should fail so expect error to propagate to cache tier
9069 bufferlist test_omap_bl;
9070 encode(static_cast<uint32_t>(1U), test_omap_bl);
9071
9072 ObjectWriteOperation op2;
9073 op2.omap_cmp({{"somekey", {test_omap_bl, CEPH_OSD_CMPXATTR_OP_EQ}}}, nullptr);
9074 op2.omap_set({{"somekey", test_omap_bl}});
9075
9076 ASSERT_EQ(-ECANCELED, ioctx.operate("propagate-base-tier-error", &op2));
9077}
9f95a23c
TL		 9078
9079TEST_F(LibRadosTwoPoolsPP, HelloWriteReturn) {
9080 // configure cache
9081 bufferlist inbl;
9082 ASSERT_EQ(0, cluster.mon_command(
9083 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
9084 "\", \"tierpool\": \"" + cache_pool_name +
9085 "\", \"force_nonempty\": \"--force-nonempty\" }",
9086 inbl, NULL, NULL));
9087 ASSERT_EQ(0, cluster.mon_command(
9088 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
9089 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
9090 inbl, NULL, NULL));
9091 ASSERT_EQ(0, cluster.mon_command(
9092 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
9093 "\", \"mode\": \"writeback\"}",
9094 inbl, NULL, NULL));
9095
9096 // set things up such that the op would normally be proxied
9097 ASSERT_EQ(0, cluster.mon_command(
9098 set_pool_str(cache_pool_name, "hit_set_count", 2),
9099 inbl, NULL, NULL));
9100 ASSERT_EQ(0, cluster.mon_command(
9101 set_pool_str(cache_pool_name, "hit_set_period", 600),
9102 inbl, NULL, NULL));
9103 ASSERT_EQ(0, cluster.mon_command(
9104 set_pool_str(cache_pool_name, "hit_set_type",
9105 "explicit_object"),
9106 inbl, NULL, NULL));
9107 ASSERT_EQ(0, cluster.mon_command(
9108 set_pool_str(cache_pool_name, "min_read_recency_for_promote",
9109 "10000"),
9110 inbl, NULL, NULL));
9111
9112 // wait for maps to settle
9113 cluster.wait_for_latest_osdmap();
9114
9115 // this *will* return data due to the RETURNVEC flag
9116 {
9117 bufferlist in, out;
9118 int rval;
9119 ObjectWriteOperation o;
9120 o.exec("hello", "write_return_data", in, &out, &rval);
9121 librados::AioCompletion *completion = cluster.aio_create_completion();
9122 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
9123 librados::OPERATION_RETURNVEC));
9124 completion->wait_for_complete();
9125 ASSERT_EQ(42, completion->get_return_value());
9126 ASSERT_EQ(42, rval);
9127 out.hexdump(std::cout);
9128 ASSERT_EQ("you might see this", std::string(out.c_str(), out.length()));
9129 }
9130
9131 // this will overflow because the return data is too big
9132 {
9133 bufferlist in, out;
9134 int rval;
9135 ObjectWriteOperation o;
9136 o.exec("hello", "write_too_much_return_data", in, &out, &rval);
9137 librados::AioCompletion *completion = cluster.aio_create_completion();
9138 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
9139 librados::OPERATION_RETURNVEC));
9140 completion->wait_for_complete();
9141 ASSERT_EQ(-EOVERFLOW, completion->get_return_value());
9142 ASSERT_EQ(-EOVERFLOW, rval);
9143 ASSERT_EQ("", std::string(out.c_str(), out.length()));
9144 }
9145}
Ceph